From 0f241f313473b599840d3e17bcd02ef4c9795764 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:46 +0100 Subject: [PATCH 01/24] feat(dev): og dev build rebuilds the three committed asset tables og dev build cn-intervals, callability and sv-rule-regions read the pinned downloads from a folder and write bin/cn_loci.tsv's interval rows, assets/callability_genes.tsv and assets/sv_rule_regions.tsv, byte for byte as the Python builders did. The GTF is read as bytes with a hand scanner of its attributes, the cut sets are merged once per chromosome and searched by binary search, and og never downloads. The SV rule builder now keeps the first GENCODE record per HGNC id and skips _PAR_Y copies, as the callability builder does; the committed table does not change. Bead: pgp-bs9u.35 --- rust/Cargo.lock | 1 + rust/crates/og-dev/Cargo.toml | 3 +- rust/crates/og-dev/src/build/callability.rs | 628 ++++++++++ rust/crates/og-dev/src/build/cn_intervals.rs | 1162 ++++++++++++++++++ rust/crates/og-dev/src/build/gtf.rs | 133 ++ rust/crates/og-dev/src/build/mod.rs | 298 +++++ rust/crates/og-dev/src/build/sv_rules.rs | 649 ++++++++++ rust/crates/og-dev/src/lib.rs | 6 +- rust/og/src/main.rs | 1 + 9 files changed, 2879 insertions(+), 2 deletions(-) create mode 100644 rust/crates/og-dev/src/build/callability.rs create mode 100644 rust/crates/og-dev/src/build/cn_intervals.rs create mode 100644 rust/crates/og-dev/src/build/gtf.rs create mode 100644 rust/crates/og-dev/src/build/mod.rs create mode 100644 rust/crates/og-dev/src/build/sv_rules.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d91a18ca..44bf8f81a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -666,6 +666,7 @@ version = "0.1.0" dependencies = [ "flate2", "indexmap", + "og-intervals", "og-io", "og-nf", "png", diff --git a/rust/crates/og-dev/Cargo.toml b/rust/crates/og-dev/Cargo.toml index 20bcd7e4c..91e36986b 100644 --- a/rust/crates/og-dev/Cargo.toml +++ b/rust/crates/og-dev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-dev" -description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only" +description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only, build" version.workspace = true edition.workspace = true license.workspace = true @@ -10,6 +10,7 @@ publish.workspace = true pycompat.workspace = true og-io.workspace = true og-nf.workspace = true +og-intervals.workspace = true flate2.workspace = true indexmap.workspace = true png = "0.18" diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs new file mode 100644 index 000000000..d7ce4dece --- /dev/null +++ b/rust/crates/og-dev/src/build/callability.rs @@ -0,0 +1,628 @@ +//! `og dev build callability`: `assets/callability_genes.tsv`, the genes +//! where short reads are often not placed with confidence, so a "no +//! finding" there needs a reading of its own (step 16b measures each one). +//! +//! A gene is a GENCODE 50 gene with an HGNC id, on chr1-22, X or Y unless it +//! is ALT-only, and protein-coding unless it is a CMRG gene. It is listed +//! with every reason that applies: +//! +//! - `CMRG`: one of GIAB's 273 challenging medically relevant genes, +//! matched by symbol, or by the protein-coding gene that covers most of +//! its CMRG span when GENCODE has renamed it; +//! - `segdup`: half or more of the gene body in a segmental duplication; +//! - `false_duplication`: the gene body overlaps a false duplication of +//! GRCh38 (either copy); +//! - `collapsed`: the gene body overlaps a collapsed duplication; +//! - `ALT-only`: the gene is on an ALT scaffold and nowhere on chr1-22, X, +//! Y or M; +//! +//! and the step that calls it with a method of its own: `parascopy` for the +//! genes of `assets/paralog_loci.tsv`, `pypgx` for the bam genes of +//! `assets/pgx_genes.tsv` callable without ALT contigs, `cn_loci:` +//! for a gene a gene row of `bin/cn_loci.tsv` names; `.` for none. +//! +//! `--work` holds the seven downloads `scripts/ci/build-callability.sh` +//! names. The GTF is read once, line by line; each interval file is merged +//! once per chromosome, so a gene's coverage is a binary search. + +use super::gtf::Record; +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, each_raw_line, int}; +use og_intervals::sets::Set; +use og_io::Utf8; +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; +use std::io::Write; +use std::path::Path; + +/// The reasons, in the order a row lists them. +const REASONS: [&str; 5] = [ + "CMRG", + "segdup", + "false_duplication", + "collapsed", + "ALT-only", +]; +const CMRG: u8 = 1; +const SEGDUP: u8 = 1 << 1; +const FALSE_DUP: u8 = 1 << 2; +const COLLAPSED: u8 = 1 << 3; +const ALT_ONLY: u8 = 1 << 4; + +/// A gene of GENCODE on chr1-22, X or Y: the first record of its HGNC id. +struct Gene { + symbol: String, + hgnc: String, + chrom: usize, + start: i64, + end: i64, + coding: bool, +} + +/// What the GTF gives. +#[derive(Default)] +struct Genes { + /// In the GTF's order. + primary: Vec, + by_hgnc: HashMap, + /// Every gene symbol on chr1-22, X, Y or M. + on_primary: HashSet, + /// HGNC id -> (symbol, UCSC ALT scaffold name) of the first + /// protein-coding ALT copy. + alt: HashMap, +} + +/// The ALT scaffolds of NCBI's assembly report: GenBank accession -> UCSC +/// name (GENCODE names them by accession). +fn read_alt(path: &Path) -> Result, BuildError> { + let mut alt = HashMap::new(); + each_line(path, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if !line.starts_with('#') && c.len() >= 10 && c[1] == "alt-scaffold" { + alt.insert(c[4].to_owned(), c[9].to_owned()); + } + Ok(()) + })?; + Ok(alt) +} + +fn read_gtf(path: &Path, alt: &HashMap) -> Result { + let mut g = Genes::default(); + each_raw_line(path, |line| { + let Some(r) = Record::parse(line)? else { + return Ok(()); + }; + if r.feature != b"gene" { + return Ok(()); + } + let chrom = chrom_index(r.seq); + if (chrom.is_some() || r.seq == b"chrM") + && let Some(n) = r.attr("gene_name")? + { + g.on_primary.insert(n.to_owned()); + } + let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) else { + return Ok(()); + }; + let need = |k: &str| { + r.attr(k)?.ok_or_else(|| { + BuildError::Input(format!( + "GTF: a gene with an HGNC id and no {k}: {:?}", + String::from_utf8_lossy(line) + )) + }) + }; + if need("gene_id")?.ends_with("_PAR_Y") { + return Ok(()); + } + let coding = r.attr("gene_type")? == Some("protein_coding"); + match chrom { + Some(chrom) if !g.by_hgnc.contains_key(h) => { + let (start, end) = r.span()?; + g.by_hgnc.insert(h.to_owned(), g.primary.len()); + g.primary.push(Gene { + symbol: need("gene_name")?.to_owned(), + hgnc: h.to_owned(), + chrom, + start, + end, + coding, + }); + } + _ => { + if let Some(name) = std::str::from_utf8(r.seq).ok().and_then(|s| alt.get(s)) + && coding + && !g.alt.contains_key(h) + { + g.alt + .insert(h.to_owned(), (need("gene_name")?.to_owned(), name.clone())); + } + } + } + Ok(()) + })?; + Ok(g) +} + +/// A BED file as a [`Set`] per sequence. +fn read_sets(paths: &[&Path]) -> Result, BuildError> { + let mut rows: HashMap> = HashMap::new(); + for path in paths { + each_line(path, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() >= 3 + && !(line.starts_with('#') + || line.starts_with("track") + || line.starts_with("browser")) + { + let s = u64::try_from(int(c[1], "BED start")?); + let e = u64::try_from(int(c[2], "BED end")?); + let (Ok(s), Ok(e)) = (s, e) else { + return Err(BuildError::Input(format!( + "{}: a negative position: {line:?}", + path.display() + ))); + }; + rows.entry(c[0].to_owned()).or_default().push((s, e)); + } + Ok(()) + })?; + } + Ok(rows.into_iter().map(|(k, v)| (k, Set::new(v))).collect()) +} + +/// Bases of `[s, e)` on `chrom` inside `sets`. +fn covered(sets: &HashMap, chrom: usize, s: i64, e: i64) -> u64 { + let (s, e) = (u64::try_from(s).unwrap_or(0), u64::try_from(e).unwrap_or(0)); + sets.get(CHROMS[chrom]).map_or(0, |set| set.covered(s, e)) +} + +/// The rows of a tab-separated table after its comment lines, as +/// `csv.DictReader` with `QUOTE_NONE` reads them: the first line is the +/// header; each row is handed to `f` with a cell getter. +fn each_row( + path: &Path, + mut f: impl FnMut(&dyn Fn(&str) -> Result) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + let mut header: Option> = None; + each_line(path, Utf8::Strict, |line| { + if pycompat::text::strip(line, None).is_empty() || line.starts_with('#') { + return Ok(()); + } + let cells: Vec<&str> = line.split('\t').collect(); + let Some(h) = &header else { + header = Some(cells.iter().map(|c| (*c).to_owned()).collect()); + return Ok(()); + }; + let get = |k: &str| { + h.iter() + .rposition(|n| n == k) + .and_then(|i| cells.get(i)) + .map(|c| (*c).to_owned()) + .ok_or_else(|| { + BuildError::Input(format!("{}: no {k} cell in {line:?}", path.display())) + }) + }; + f(&get) + }) +} + +/// Who calls a gene with a method of its own. +struct Owners { + parascopy: HashSet, + pypgx: HashSet, + /// A name a `bin/cn_loci.tsv` gene row reports -> its locus (the first + /// row that names it). + cn: HashMap, +} + +impl Owners { + fn read(repo: &Path) -> Result { + let mut parascopy = HashSet::new(); + each_row(&repo.join("assets/paralog_loci.tsv"), |get| { + parascopy.extend(get("genes")?.split(',').map(str::to_owned)); + Ok(()) + })?; + // A gene step 32 skips on the default no-ALT reference + // (callable_without_alt no) has no caller there, and step 32 + // refuses a BAM with ALT contigs, so it is not pypgx's. + let mut pypgx = HashSet::new(); + each_row(&repo.join("assets/pgx_genes.tsv"), |get| { + if get("mode")? == "bam" && get("callable_without_alt")? != "no" { + pypgx.insert(get("gene")?); + } + Ok(()) + })?; + // A gene row reports the genes it names (its locus, as RHD, or LPA + // for LPA_KIV2, or a target, as HBA1 and HBA2), not every gene whose + // span holds one of its targets. + let mut cn: HashMap = HashMap::new(); + each_row(&repo.join("bin/cn_loci.tsv"), |get| { + if get("kind")? != "gene" { + return Ok(()); + } + let locus = get("locus")?; + let targets = get("targets")?; + let mut names = vec![locus.as_str(), first(&locus, '_')]; + for part in targets.split(';') { + let n = first(part, ':'); + names.extend([n, first(n, '_')]); + } + for n in names { + cn.entry(n.to_owned()).or_insert_with(|| locus.clone()); + } + Ok(()) + })?; + Ok(Owners { + parascopy, + pypgx, + cn, + }) + } + + fn of(&self, symbol: &str) -> String { + if self.parascopy.contains(symbol) { + "parascopy".to_owned() + } else if self.pypgx.contains(symbol) { + "pypgx".to_owned() + } else if let Some(locus) = self.cn.get(symbol) { + format!("cn_loci:{locus}") + } else { + ".".to_owned() + } + } +} + +/// The text before the first `sep`. +fn first(s: &str, sep: char) -> &str { + s.split(sep).next().unwrap_or(s) +} + +/// The reasons of each gene, by HGNC id. +fn reasons(work: &Path, genes: &Genes) -> Result, BuildError> { + let mut why: HashMap = HashMap::new(); + let mut by_symbol: HashMap<&str, usize> = HashMap::new(); + for (i, g) in genes.primary.iter().enumerate() { + by_symbol.insert(&g.symbol, i); + } + let cmrg = work.join("cmrg.bed"); + each_line(&cmrg, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 4 { + return Err(BuildError::Input(format!( + "cmrg.bed: a row with under 4 columns: {line:?}" + ))); + } + let (chrom, sym) = (c[0], c[3]); + let (s, e) = (int(c[1], "cmrg.bed start")?, int(c[2], "cmrg.bed end")?); + let i = match by_symbol.get(sym) { + Some(&i) => i, + None => { + // Renamed since CMRG v1.00: the protein-coding gene that + // covers most of its span (the larger HGNC id on a tie). + let best = genes + .primary + .iter() + .enumerate() + .filter(|(_, g)| g.coding && CHROMS[g.chrom] == chrom) + .map(|(i, g)| (e.min(g.end) - s.max(g.start), &g.hgnc, i)) + .max_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))); + match best { + Some((overlap, _, i)) if overlap > 0 => i, + _ => { + return Err(BuildError::Data(format!( + "CMRG gene {sym} ({chrom}:{s}-{e}) matches no GENCODE 50 gene" + ))); + } + } + } + }; + *why.entry(genes.primary[i].hgnc.clone()).or_default() |= CMRG; + Ok(()) + })?; + let segdups = read_sets(&[&work.join("segdups.bed.gz")])?; + let collapsed = read_sets(&[&work.join("collapsed.bed.gz")])?; + let falsedup = read_sets(&[ + &work.join("falsedup_correct.bed.gz"), + &work.join("falsedup_incorrect.bed.gz"), + ])?; + for g in genes.primary.iter().filter(|g| g.coding) { + let mut bits = 0; + // Half or more: covered >= 0.5 * length, in whole numbers. + if 2 * covered(&segdups, g.chrom, g.start, g.end) + >= u64::try_from(g.end - g.start).unwrap_or(0) + { + bits |= SEGDUP; + } + if covered(&falsedup, g.chrom, g.start, g.end) > 0 { + bits |= FALSE_DUP; + } + if covered(&collapsed, g.chrom, g.start, g.end) > 0 { + bits |= COLLAPSED; + } + if bits != 0 { + *why.entry(g.hgnc.clone()).or_default() |= bits; + } + } + for (h, (sym, _)) in &genes.alt { + if !genes.by_hgnc.contains_key(h) && !genes.on_primary.contains(sym) { + *why.entry(h.clone()).or_default() |= ALT_ONLY; + } + } + Ok(why) +} + +/// A row of the catalog: its sort key (chrom order, start, symbol, HGNC +/// id; ALT-only genes last), its cells and its reasons. +type CatalogRow<'a> = ((usize, i64, &'a str, &'a str), [String; 7], u8); + +/// The catalog's text and the summary line, from the downloads in `work` +/// and the owner tables of `repo`. +pub fn build(work: &Path, repo: &Path) -> Result<(String, String), BuildError> { + let alt = read_alt(&work.join("report.txt"))?; + let genes = read_gtf(&work.join("gencode.gtf.gz"), &alt)?; + let why = reasons(work, &genes)?; + let owners = Owners::read(repo)?; + let mut rows: Vec = Vec::with_capacity(why.len()); + for (h, &bits) in &why { + let (key, chrom, start, end, sym) = match genes.by_hgnc.get(h) { + Some(&i) => { + let g = &genes.primary[i]; + let key = (g.chrom, g.start, g.symbol.as_str(), h.as_str()); + ( + key, + CHROMS[g.chrom].to_owned(), + g.start.to_string(), + g.end.to_string(), + &g.symbol, + ) + } + None => { + let (sym, name) = &genes.alt[h]; + ( + (CHROMS.len(), 0, sym.as_str(), h.as_str()), + name.clone(), + ".".to_owned(), + ".".to_owned(), + sym, + ) + } + }; + let listed: Vec<&str> = REASONS + .iter() + .enumerate() + .filter(|(i, _)| bits & (1 << i) != 0) + .map(|(_, r)| *r) + .collect(); + let cells = [ + sym.clone(), + h.clone(), + chrom, + start, + end, + listed.join(","), + owners.of(sym), + ]; + rows.push((key, cells, bits)); + } + rows.sort_by(|a, b| a.0.cmp(&b.0)); + let mut seen: HashMap<&str, usize> = HashMap::new(); + for r in &rows { + *seen.entry(r.0.2).or_default() += 1; + } + let mut dup: Vec<&str> = seen + .into_iter() + .filter(|&(_, n)| n > 1) + .map(|(s, _)| s) + .collect(); + if !dup.is_empty() { + dup.sort_unstable(); + return Err(BuildError::Data(format!( + "gene symbols listed twice: {}", + dup.join(", ") + ))); + } + let counts: Vec = REASONS + .iter() + .enumerate() + .map(|(i, r)| { + format!( + "{r} {}", + rows.iter().filter(|x| x.2 & (1 << i) != 0).count() + ) + }) + .collect(); + let owned = rows.iter().filter(|r| r.1[6] != ".").count(); + let summary = format!( + "Genes: {} ({}); with an owner: {owned}", + rows.len(), + counts.join(", ") + ); + let mut text = String::from( + "# assets/callability_genes.tsv: the genes where short reads are often not placed with confidence, so a\n\ + # 'no finding' there needs a reading of its own. Step 16b measures each gene on chr1-22, X and Y; the\n\ + # report lists the ones it could not read. Written by scripts/ci/build-callability.sh, which says how the\n\ + # rows and the owners are chosen; rebuild it there, do not edit it by hand. start is 0-based and end\n\ + # 1-based, as in a BED file; an ALT-only gene has its ALT scaffold as chrom and '.' as start and end.\n", + ); + let _ = writeln!(text, "# {summary}."); + text.push_str("gene\thgnc_id\tchrom\tstart\tend\treasons\towner\n"); + for r in &rows { + text.push_str(&r.1.join("\t")); + text.push('\n'); + } + Ok((text, summary)) +} + +pub(super) fn cli(args: &[String], err: &mut dyn Write) -> Result { + let a = Args::parse(args, &["--work", "--repo", "--out"], &[])?; + let (work, repo, out) = (a.need("--work")?, a.need("--repo")?, a.need("--out")?); + let (text, summary) = build(Path::new(work), Path::new(repo))?; + super::write_atomic(Path::new(out), text.as_bytes())?; + let _ = writeln!(err, "{summary}"); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// A work folder and a repo folder with tiny inputs: A is a CMRG gene by + /// symbol, OLD a CMRG gene GENCODE renamed to B, C sits in a segmental + /// duplication, D over a false and a collapsed duplication, E is only on + /// an ALT scaffold, F is on an ALT scaffold and chr1, P is a CMRG gene whose + /// _PAR_Y copy comes first and must not move it to chrY, N is not protein-coding and has no reason. + fn fixture(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf) { + let (work, repo) = (dir.join("work"), dir.join("repo")); + fs::create_dir_all(&work).unwrap(); + fs::create_dir_all(repo.join("assets")).unwrap(); + fs::create_dir_all(repo.join("bin")).unwrap(); + let gene = |seq: &str, s: u32, e: u32, id: &str, ty: &str, name: &str, h: &str| { + let hg = if h.is_empty() { + String::new() + } else { + format!(" hgnc_id \"{h}\";") + }; + format!( + "{seq}\tX\tgene\t{s}\t{e}\t.\t+\t.\tgene_id \"{id}\"; gene_type \"{ty}\"; gene_name \"{name}\";{hg}\n" + ) + }; + let gtf = [ + "##format: gtf\n".to_owned(), + gene("chr1", 101, 200, "G1", "protein_coding", "A", "HGNC:1"), + "chr1\tX\texon\t101\t150\t.\t+\t.\tgene_id \"G1\";\n".to_owned(), + gene("chr1", 1001, 2000, "G2", "protein_coding", "B", "HGNC:2"), + gene("chr2", 101, 1100, "G3", "protein_coding", "C", "HGNC:3"), + gene("chr2", 5001, 6000, "G4", "protein_coding", "D", "HGNC:4"), + gene("chr2", 9001, 9100, "G5", "lncRNA", "N", "HGNC:5"), + gene( + "chrY", + 100_001, + 100_500, + "G6_PAR_Y", + "protein_coding", + "P", + "HGNC:6", + ), + gene( + "chrX", + 100_001, + 100_500, + "G6", + "protein_coding", + "P", + "HGNC:6", + ), + gene("chr1", 3001, 3100, "G7", "protein_coding", "F", "HGNC:7"), + gene("KI1.1", 1, 100, "G8", "protein_coding", "E", "HGNC:8"), + gene("KI1.1", 1, 100, "G9", "protein_coding", "F", "HGNC:9"), + gene("chrM", 1, 50, "G10", "protein_coding", "MT", ""), + ] + .concat(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gz.write_all(gtf.as_bytes()).unwrap(); + fs::write(work.join("gencode.gtf.gz"), gz.finish().unwrap()).unwrap(); + fs::write( + work.join("report.txt"), + "# report\nKI1\talt-scaffold\t1\tc\tKI1.1\t=\tx\ty\tz\tchr1_KI1v1_alt\r\n", + ) + .unwrap(); + fs::write( + work.join("cmrg.bed"), + "chr1\t100\t200\tA\nchr1\t900\t1500\tOLD\nchrX\t100000\t100500\tP\n", + ) + .unwrap(); + fs::write( + work.join("segdups.bed.gz"), + "chr2\t100\t400\nchr2\t350\t700\n", + ) + .unwrap(); + fs::write(work.join("falsedup_correct.bed.gz"), "chr2\t5999\t6001\n").unwrap(); + fs::write(work.join("falsedup_incorrect.bed.gz"), "").unwrap(); + fs::write(work.join("collapsed.bed.gz"), "#c\nchr2\t5000\t5001\n").unwrap(); + fs::write( + repo.join("assets/paralog_loci.tsv"), + "# x\nlocus\tgenes\nl\tC,Z\n", + ) + .unwrap(); + fs::write( + repo.join("assets/pgx_genes.tsv"), + "gene\tmode\tcallable_without_alt\nD\tbam\tyes\nA\tbam\tno\n", + ) + .unwrap(); + fs::write( + repo.join("bin/cn_loci.tsv"), + "locus\tkind\ttargets\nA_x\tgene\tq_t1:1-2\nY\tinterval\tA:1-2\n", + ) + .unwrap(); + (work, repo) + } + + #[test] + fn the_tiny_catalog_has_every_reason_and_owner() { + let dir = std::env::temp_dir().join(format!("og-callability-{}", std::process::id())); + let (work, repo) = fixture(&dir); + let (text, summary) = build(&work, &repo).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + let body: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect(); + assert_eq!( + body, + [ + "gene\thgnc_id\tchrom\tstart\tend\treasons\towner", + "A\tHGNC:1\tchr1\t100\t200\tCMRG\tcn_loci:A_x", + "B\tHGNC:2\tchr1\t1000\t2000\tCMRG\t.", + "C\tHGNC:3\tchr2\t100\t1100\tsegdup\tparascopy", + "D\tHGNC:4\tchr2\t5000\t6000\tfalse_duplication,collapsed\tpypgx", + "P\tHGNC:6\tchrX\t100000\t100500\tCMRG\t.", + "E\tHGNC:8\tchr1_KI1v1_alt\t.\t.\tALT-only\t.", + ] + ); + assert_eq!( + summary, + "Genes: 6 (CMRG 3, segdup 1, false_duplication 1, collapsed 1, ALT-only 1); with an owner: 3" + ); + assert!(text.contains(&format!("\n# {summary}.\n"))); + } + + #[test] + fn a_cmrg_gene_with_no_gencode_gene_stops_the_build() { + let dir = std::env::temp_dir().join(format!("og-callability-cmrg-{}", std::process::id())); + let (work, repo) = fixture(&dir); + fs::write(work.join("cmrg.bed"), "chr3\t1\t2\tGONE\n").unwrap(); + let e = build(&work, &repo).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!( + e.to_string(), + "CMRG gene GONE (chr3:1-2) matches no GENCODE 50 gene" + ); + assert!(matches!(e, BuildError::Data(_))); + } + + #[test] + fn a_symbol_listed_twice_stops_the_build() { + let dir = std::env::temp_dir().join(format!("og-callability-dup-{}", std::process::id())); + let (work, repo) = fixture(&dir); + // Two genes named A, both CMRG (the second by its span). + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gz.write_all(b"chr1\tX\tgene\t101\t200\t.\t+\t.\tgene_id \"G1\"; gene_type \"protein_coding\"; gene_name \"A\"; hgnc_id \"HGNC:1\";\nchr1\tX\tgene\t901\t1500\t.\t+\t.\tgene_id \"G2\"; gene_type \"protein_coding\"; gene_name \"A\"; hgnc_id \"HGNC:2\";\n").unwrap(); + fs::write(work.join("gencode.gtf.gz"), gz.finish().unwrap()).unwrap(); + fs::write( + work.join("cmrg.bed"), + "chr1\t100\t200\tX1\nchr1\t900\t1500\tX2\n", + ) + .unwrap(); + let e = build(&work, &repo).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!(e.to_string(), "gene symbols listed twice: A"); + } + + #[test] + fn coverage_is_union_coverage() { + let sets = HashMap::from([( + "chr2".to_owned(), + Set::new([(100, 400), (350, 700), (900, 950)]), + )]); + assert_eq!(covered(&sets, 1, 100, 1100), 650); + assert_eq!(covered(&sets, 0, 100, 1100), 0); + } +} diff --git a/rust/crates/og-dev/src/build/cn_intervals.rs b/rust/crates/og-dev/src/build/cn_intervals.rs new file mode 100644 index 000000000..8624daa9b --- /dev/null +++ b/rust/crates/og-dev/src/build/cn_intervals.rs @@ -0,0 +1,1162 @@ +//! `og dev build cn-intervals`: the interval rows of `bin/cn_loci.tsv`, +//! the copy-number checks step 35 runs with one `samtools bedcov` pass +//! (`og cn intervals`). +//! +//! The rows are: +//! +//! - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, +//! version 2.1, GRCh38 (`CLINGEN_RECURRENT_CNV_URL` in `versions.env`, +//! CC0). Its breakpoint and LCR rows (itemRgb `0,0,0`) are the segmental +//! duplications that bound the regions: they are cut out of the targets +//! and kept out of the controls. Its chrX rows are spelled `chrx` and are +//! read as chrX; any other name that is not a GRCh38 chromosome stops the +//! build; +//! - the rows the BED lacks (`HAND_ROWS`): two ClinGen curations (1p36, +//! Xq28 MECP2), five carrier deletions at their ClinVar or HbVar GRCh38 +//! spans, and the Y chromosome's AZF regions with two single-copy Y +//! controls. +//! +//! Targets are cut to pieces of at most 500 kb and lose every stretch inside +//! a breakpoint or LCR row, a GIAB v3.6 segmental duplication, an assembly +//! gap (GIAB's gaps with 15 kb on each side), a satellite or a region of a +//! model the pinned Parascopy data holds; the AZF targets keep their +//! segmental duplications, since the DAZ copies are one. Controls are two +//! 100 kb windows, one on each side of the cluster of ClinGen rows the +//! region belongs to, the nearest that stay clear of all of that and of +//! every target, within 3 Mb; a side with none (a chromosome end) is left +//! out. +//! +//! Every cut set is merged once per chromosome, so a subtraction or a +//! "clear of the cuts" test is a binary search. + +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, input_error, int}; +use og_io::Utf8; +use std::io::Write; +use std::path::Path; + +/// The line after which the catalog's rows are this builder's. +pub const MARKER: &str = + "# --- interval rows: written by scripts/ci/build-cn-intervals.sh; do not edit by hand ---"; +/// How a marker line starts, whatever builder it names. +const MARKER_START: &[u8] = b"# --- interval rows: "; + +const REGION_RGB: &str = "255,111,0"; +const MASK_RGB: &str = "0,0,0"; +/// GRCh38 chromosome lengths (the no-ALT analysis set's `.fai`), in +/// [`CHROMS`] order. +const LENGTHS: [i64; 24] = [ + 248_956_422, + 242_193_529, + 198_295_559, + 190_214_555, + 181_538_259, + 170_805_979, + 159_345_973, + 145_138_636, + 138_394_717, + 133_797_422, + 135_086_622, + 133_275_309, + 114_364_328, + 107_043_718, + 101_991_189, + 90_338_345, + 83_257_441, + 80_373_285, + 58_617_616, + 64_444_167, + 46_709_983, + 50_818_468, + 156_040_895, + 57_227_415, +]; +const PIECE: i64 = 500_000; +const FLANK: i64 = 100_000; +const GAP: i64 = 50_000; +const SEARCH: i64 = 3_000_000; +const STEP: i64 = 10_000; +const MIN_PIECE: i64 = 500; +const CHR_Y: usize = 23; + +/// A 0-based, half-open stretch. +type Span = (i64, i64); + +/// How a row's target depth is read. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Depth { + /// Depth pooled over the targets. + Pooled, + /// The targets' depths added up (copies of one sequence). + Sum, +} + +impl Depth { + fn name(self) -> &'static str { + match self { + Depth::Pooled => "pooled", + Depth::Sum => "sum", + } + } +} + +/// A row the BED lacks. +struct Hand { + locus: &'static str, + chrom: usize, + /// (label, 0-based start, end); the first label names the row. + targets: &'static [(&'static str, i64, i64)], + /// (name, start, end, label), or `None` for flanks. + controls: Option<&'static [(&'static str, i64, i64, &'static str)]>, + depth: Depth, + /// Whether segmental duplications are cut out of the targets. + cut_segdups: bool, + pmids: &'static str, +} + +const Y_CONTROLS: &[(&str, i64, i64, &str)] = &[ + ( + "ZFY", + 2_935_200, + 2_982_579, + "ZFY, single-copy control on Yp", + ), + ( + "NLGN4Y", + 14_522_572, + 14_845_654, + "NLGN4Y, single-copy control on Yq", + ), +]; + +/// The rows the ClinGen BED lacks. 1p36 (ISCA-37434) and Xq28 (ISCA-46304) +/// from ClinGen_region_curation_list_GRCh38.tsv as read on 2026-11-04; +/// del(GJB6-D13S1830), CFTRdele2,3 and the 621 bp HBB deletion at their +/// ClinVar GRCh38 spans; the 619 bp HBB deletion (HbVar 979, Orkin 1979) +/// and the Sicilian (delta-beta)0 deletion (HbVar 1035, Henthorn 1990) at +/// their HbVar spans, placed on NC_000011.10 by 50 bp on each side of the +/// NG_000007.3 breakpoints on 2026-11-04; the AZF regions by the single-copy +/// genes each deletion removes (AZFc as the four DAZ copies added up), gene +/// spans from Ensembl's REST lookup, checked on 2026-11-04. The alpha-globin +/// --SEA deletion is read by step 35's HBA table, not here. +const HAND_ROWS: &[Hand] = &[ + Hand { + locus: "1p36_terminal", + chrom: 0, + targets: &[( + "1p36 terminal region (includes GABRD), ClinGen ISCA-37434", + 898_702, + 6_229_913, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "Xq28_MECP2", + chrom: 22, + targets: &[( + "Xq28 region (includes MECP2), ClinGen ISCA-46304: the MECP2 duplication", + 154_008_528, + 154_110_279, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "GJB6_D13S1830", + chrom: 12, + targets: &[("del(GJB6-D13S1830), ClinVar 5546", 20_223_037, 20_531_806)], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "11807148", + }, + Hand { + locus: "CFTR_dele2_3", + chrom: 6, + targets: &[( + "CFTRdele2,3 (21 kb), ClinVar 66105", + 117_498_312, + 117_519_392, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "HBB_exon3_del", + chrom: 10, + targets: &[( + "HBB deletion of exon 3 (621 bp), ClinVar 3769040", + 5_225_254, + 5_225_875, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "HBB_619bp_del", + chrom: 10, + targets: &[( + "HBB 619 bp deletion (South Asian), HbVar 979", + 5_225_388, + 5_226_007, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "287080", + }, + Hand { + locus: "HBD_HBB_Sicilian_del", + chrom: 10, + targets: &[( + "Sicilian (delta-beta)0 deletion (13.4 kb, HBD to HBB 3'), HbVar 1035", + 5_219_877, + 5_233_280, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "2307466", + }, + Hand { + locus: "AZFa", + chrom: CHR_Y, + targets: &[ + ("AZFa (USP9Y and DDX3Y)", 12_537_649, 12_860_852), + ("DDX3Y", 12_904_107, 12_920_478), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Pooled, + cut_segdups: false, + pmids: "37674303", + }, + Hand { + locus: "AZFb", + chrom: CHR_Y, + targets: &[ + ("AZFb (KDM5D and EIF1AY)", 19_702_297, 19_745_148), + ("EIF1AY", 20_575_693, 20_593_154), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Pooled, + cut_segdups: false, + pmids: "37674303", + }, + Hand { + locus: "AZFc_DAZ", + chrom: CHR_Y, + targets: &[ + ( + "AZFc (DAZ1 to DAZ4 added up; gr/gr and b2/b3 leave two)", + 23_129_354, + 23_199_010, + ), + ("DAZ2", 23_219_446, 23_291_356), + ("DAZ3", 24_763_068, 24_813_492), + ("DAZ4", 24_833_842, 24_907_040), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Sum, + cut_segdups: false, + pmids: "37674303", + }, +]; + +/// A region row of the ClinGen BED. +#[derive(Debug)] +struct Region { + chrom: usize, + start: i64, + end: i64, + name: String, +} + +/// The stretches cut out of targets and controls, per chromosome. +#[derive(Default, Clone)] +struct Cuts { + /// ClinGen breakpoint and LCR rows. + mask: Vec, + segdup: Vec, + gap: Vec, + satellite: Vec, + parascopy: Vec, +} + +/// The downloads, read. +pub struct Inputs { + regions: Vec, + cuts: Vec, +} + +/// `chrx` and `chry` (the ClinGen BED's spelling) as chrX and chrY; any +/// other name that is not a GRCh38 chromosome is an error. +fn norm_chrom(c: &str) -> Result { + let fixed = match c { + "chrx" => "chrX", + "chry" => "chrY", + _ => c, + }; + chrom_index(fixed).ok_or_else(|| { + BuildError::Input(format!( + "chromosome {} is not a GRCh38 chromosome name", + py_repr(c) + )) + }) +} + +fn py_repr(s: &str) -> String { + pycompat::value::py_repr(&pycompat::json::Value::Str(s.to_owned())) + .unwrap_or_else(|_| format!("'{s}'")) +} + +/// The region rows and breakpoint rows of the ClinGen BED's lines. +fn read_clingen<'a>( + lines: impl IntoIterator, + cuts: &mut [Cuts], +) -> Result, BuildError> { + let mut regions = Vec::new(); + for line in lines { + let strip = pycompat::text::strip; + if strip(line, None).is_empty() + || line.starts_with('#') + || line.starts_with("track") + || line.starts_with("browser") + { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 9 { + return Err(BuildError::Input(format!( + "ClinGen BED: a row with {} columns: {}", + c.len(), + py_repr(strip(line, None)) + ))); + } + let chrom = norm_chrom(c[0])?; + let (start, end) = ( + int(c[1], "ClinGen BED start")?, + int(c[2], "ClinGen BED end")?, + ); + let (name, rgb) = (strip(c[3], None), strip(c[8], None)); + if end <= start { + return Err(BuildError::Input(format!( + "ClinGen BED: {name}: empty span" + ))); + } + match rgb { + REGION_RGB => regions.push(Region { + chrom, + start, + end, + name: name.to_owned(), + }), + MASK_RGB => cuts[chrom].mask.push((start, end)), + _ => { + return Err(BuildError::Input(format!( + "ClinGen BED: {name}: itemRgb {} is neither a region nor a breakpoint row", + py_repr(rgb) + ))); + } + } + } + if regions.is_empty() { + return Err(BuildError::Input("ClinGen BED: no region row".to_owned())); + } + Ok(regions) +} + +/// The rows of a BED on chr1-22, X and Y, per chromosome. +fn read_bed( + path: &Path, + put: impl Fn(&mut Cuts) -> &mut Vec, + cuts: &mut [Cuts], +) -> Result<(), BuildError> { + each_line(path, Utf8::Strict, |line| { + if line.starts_with('#') || pycompat::text::strip(line, None).is_empty() { + return Ok(()); + } + let mut c = line.split('\t'); + let (seq, s, e) = (c.next(), c.next(), c.next()); + let (Some(seq), Some(s), Some(e)) = (seq, s, e) else { + return Err(BuildError::Input(format!( + "{}: a row with under 3 columns", + path.display() + ))); + }; + if let Some(i) = chrom_index(seq) { + let span = (int(s, "BED start")?, int(e, "BED end")?); + put(&mut cuts[i]).push(span); + } + Ok(()) + }) +} + +/// The regions of Parascopy's models: `NAME CHROM:START-END ...`, 1-based. +fn read_parascopy(path: &Path, cuts: &mut [Cuts]) -> Result<(), BuildError> { + each_line(path, Utf8::Strict, |line| { + if line.starts_with('#') || pycompat::text::strip(line, None).is_empty() { + return Ok(()); + } + let bad = || { + BuildError::Input(format!( + "{}: not NAME CHROM:START-END DETAIL: {line:?}", + path.display() + )) + }; + let f = pycompat::text::split(line, None, None); + let [_, reg, _] = f[..] else { + return Err(bad()); + }; + let (c, span) = reg.split_once(':').ok_or_else(bad)?; + let (s, e) = span.split_once('-').ok_or_else(bad)?; + if let Some(i) = chrom_index(c) { + cuts[i] + .parascopy + .push((int(s, "Parascopy start")? - 1, int(e, "Parascopy end")?)); + } + Ok(()) + }) +} + +/// Sorted, with overlapping and touching spans joined. +fn merge(mut spans: Vec) -> Vec { + spans.sort_unstable(); + let mut out: Vec = Vec::with_capacity(spans.len()); + for (s, e) in spans { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + out +} + +/// The parts of `[s, e)` outside the merged `cuts`. +fn subtract(s: i64, e: i64, cuts: &[Span]) -> Vec { + let mut out = Vec::new(); + let mut pos = s; + let first = cuts.partition_point(|c| c.1 <= s); + for &(cs, ce) in &cuts[first..] { + if cs >= e { + break; + } + if cs > pos { + out.push((pos, cs)); + } + pos = pos.max(ce); + } + if pos < e { + out.push((pos, e)); + } + out +} + +/// Each span of `MIN_PIECE` bp or more cut into equal pieces of at most +/// `PIECE` bp. +fn pieces(spans: &[Span]) -> Vec { + let mut out = Vec::new(); + for &(s, e) in spans { + let len = e - s; + if len < MIN_PIECE { + continue; + } + // Ceiling divisions of positive numbers. + let n = (len + PIECE - 1) / PIECE; + let step = (len + n - 1) / n; + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); + } + out +} + +/// Whether `[s, e)` touches none of the merged `cuts`. +fn clear(s: i64, e: i64, cuts: &[Span]) -> bool { + let i = cuts.partition_point(|c| c.1 <= s); + cuts.get(i).is_none_or(|c| c.0 >= e) +} + +/// The nearest `FLANK` window `GAP` beyond `edge` on the left (`left`) or +/// right, clear of `cuts`, within `SEARCH`. +fn flank(chrom: usize, edge: i64, left: bool, cuts: &[Span]) -> Option { + for k in 0..=SEARCH / STEP { + let (s, e) = if left { + let e = edge - GAP - k * STEP; + (e - FLANK, e) + } else { + let s = edge + GAP + k * STEP; + (s, s + FLANK) + }; + if s < 0 || e > LENGTHS[chrom] { + return None; + } + if clear(s, e, cuts) { + return Some((s, e)); + } + } + None +} + +/// The row id of a ClinGen region name: its band, then the words of its +/// parentheses that are not an "includes" note. +fn locus_id(name: &str) -> String { + let band = pycompat::text::split(name, None, None) + .first() + .copied() + .unwrap_or_default(); + let b = name.as_bytes(); + let mut groups: Vec<&str> = Vec::new(); + let mut i = 0; + while i < b.len() { + if b[i] == b'(' { + let j = b[i + 1..] + .iter() + .position(|&c| c == b'(' || c == b')') + .map(|k| i + 1 + k); + if let Some(j) = j + && b[j] == b')' + { + let g = &name[i + 1..j]; + if !pycompat::text::strip(g, None).starts_with("includes") { + groups.push(g); + } + i = j + 1; + continue; + } + } + i += 1; + } + let mut words = String::new(); + let mut in_run = false; + for ch in groups.join(" ").chars() { + if ch.is_ascii_alphanumeric() || ch == '.' || ch == '-' { + words.push(ch); + in_run = false; + } else if !in_run { + words.push('_'); + in_run = true; + } + } + let words = words.trim_matches('_'); + if words.is_empty() { + band.to_owned() + } else { + format!("{band}_{words}") + } +} + +/// A name made safe for a target or control cell. +fn label(text: &str) -> String { + let t: String = text + .chars() + .map(|c| { + if matches!(c, ';' | '\t' | '\n') { + ',' + } else { + c + } + }) + .collect(); + pycompat::text::strip(&t, None).to_owned() +} + +/// A control window: (name, start, end, label). +type Control = (String, i64, i64, String); + +/// The catalog rows, as Python wrote them, in their order. +struct Builder<'a> { + inputs: &'a Inputs, + /// Per chromosome, the ClinGen rows that touch or overlap, merged + /// (each row's end taken one base further). + clusters: Vec>, + /// Per chromosome, every cut (with and without segmental duplications) + /// and the cuts a control keeps out of. + cuts_all: Vec>, + cuts_no_segdups: Vec>, + flank_cuts: Vec>, + rows: Vec, + seen: std::collections::HashMap, +} + +impl<'a> Builder<'a> { + fn new(inputs: &'a Inputs) -> Builder<'a> { + let n = CHROMS.len(); + let mut clusters = vec![Vec::new(); n]; + let mut flank_cuts = vec![Vec::new(); n]; + for r in &inputs.regions { + clusters[r.chrom].push((r.start, r.end + 1)); + flank_cuts[r.chrom].push((r.start, r.end)); + } + for h in HAND_ROWS { + flank_cuts[h.chrom].extend(h.targets.iter().map(|&(_, s, e)| (s, e))); + } + let mut cuts_all = Vec::with_capacity(n); + let mut cuts_no_segdups = Vec::with_capacity(n); + for (i, c) in inputs.cuts.iter().enumerate() { + clusters[i].extend(c.mask.iter().map(|&(s, e)| (s, e + 1))); + let all: Vec = [&c.mask, &c.segdup, &c.gap, &c.satellite, &c.parascopy] + .into_iter() + .flatten() + .copied() + .collect(); + flank_cuts[i].extend(&all); + cuts_all.push(merge(all)); + cuts_no_segdups.push(merge( + [&c.mask, &c.gap, &c.parascopy] + .into_iter() + .flatten() + .copied() + .collect(), + )); + } + Builder { + inputs, + clusters: clusters.into_iter().map(merge).collect(), + cuts_all, + cuts_no_segdups, + flank_cuts: flank_cuts.into_iter().map(merge).collect(), + rows: Vec::new(), + seen: std::collections::HashMap::new(), + } + } + + /// The two flank controls of the cluster that holds `[s, e]`. + fn flanks(&self, chrom: usize, s: i64, e: i64) -> Result, BuildError> { + let (cs, ce) = self.clusters[chrom] + .iter() + .copied() + .find(|&(a, b)| a <= s && e <= b) + .unwrap_or((s, e)); + let mut out = Vec::new(); + for (left, edge, n) in [(true, cs, "L"), (false, ce, "R")] { + if let Some((a, b)) = flank(chrom, edge, left, &self.flank_cuts[chrom]) { + let side = if left { "left" } else { "right" }; + out.push((n.to_owned(), a, b, format!("flank {side} of the region"))); + } + } + if out.is_empty() { + return Err(BuildError::Input(format!( + "{}:{s}-{e}: no flank clear of the masks within {} Mb", + CHROMS[chrom], + SEARCH / 1_000_000 + ))); + } + Ok(out) + } + + fn add( + &mut self, + locus: &str, + chrom: usize, + targets: &[(i64, i64, String)], + controls: &[Control], + depth: Depth, + pmids: &str, + ) -> Result<(), BuildError> { + if targets.is_empty() { + return Err(BuildError::Input(format!( + "{locus}: no target stretch is left outside the masks" + ))); + } + let n = self.seen.entry(locus.to_owned()).or_insert(0); + *n += 1; + let locus = if *n > 1 { + format!("{locus}_{n}") + } else { + locus.to_owned() + }; + let tg: Vec = targets + .iter() + .enumerate() + .map(|(i, (s, e, lab))| { + let tail = if lab.is_empty() { + String::new() + } else { + format!(":{lab}") + }; + format!("{locus}_t{}:{s}-{e}{tail}", i + 1) + }) + .collect(); + let ct: Vec = controls + .iter() + .map(|(n, s, e, lab)| format!("{locus}_{n}:{s}-{e}:{lab}")) + .collect(); + let y = chrom == CHR_Y; + let cells = [ + locus.as_str(), + "interval", + CHROMS[chrom], + &tg.join(";"), + &ct.join(";"), + ".", + depth.name(), + if y { "pooled" } else { "flanks" }, + "2", + "0", + ".", + ".", + ".", + if y { "4" } else { "8" }, + if y { "." } else { "1.5" }, + "round:0.3", + ".", + ".", + "bedcov", + "cn-depth", + "cnvpytor:cnvs", + pmids, + ]; + self.rows.push(cells.join("\t")); + Ok(()) + } + + /// Targets with the first piece labelled. + fn labelled(segs: &[Span], name: &str) -> Vec<(i64, i64, String)> { + segs.iter() + .enumerate() + .map(|(i, &(a, b))| (a, b, if i == 0 { label(name) } else { String::new() })) + .collect() + } + + fn build(mut self) -> Result, BuildError> { + let mut regions: Vec<&Region> = self.inputs.regions.iter().collect(); + regions.sort_by_key(|r| (r.chrom, r.start, r.end)); + for r in regions { + let segs = pieces(&subtract(r.start, r.end, &self.cuts_all[r.chrom])); + let targets = Self::labelled(&segs, &r.name); + let controls = self.flanks(r.chrom, r.start, r.end)?; + self.add( + &locus_id(&r.name), + r.chrom, + &targets, + &controls, + Depth::Pooled, + ".", + )?; + } + for h in HAND_ROWS { + let cuts = if h.cut_segdups { + &self.cuts_all[h.chrom] + } else { + &self.cuts_no_segdups[h.chrom] + }; + let mut targets = Vec::new(); + for &(name, s, e) in h.targets { + let segs = match h.depth { + Depth::Pooled => pieces(&subtract(s, e, cuts)), + Depth::Sum => vec![(s, e)], + }; + if h.depth == Depth::Sum + && !clear(s, e, &merge(self.inputs.cuts[h.chrom].parascopy.clone())) + { + return Err(BuildError::Input(format!( + "{}: {name} overlaps a Parascopy model", + h.locus + ))); + } + targets.extend(Self::labelled(&segs, name)); + } + let controls = match h.controls { + None => self.flanks(h.chrom, h.targets[0].1, h.targets[h.targets.len() - 1].2)?, + Some(c) => c + .iter() + .enumerate() + .map(|(i, &(_, s, e, lab))| (format!("c{}", i + 1), s, e, lab.to_owned())) + .collect(), + }; + self.add(h.locus, h.chrom, &targets, &controls, h.depth, h.pmids)?; + } + Ok(self.rows) + } +} + +impl Inputs { + /// The downloads: the ClinGen BED's lines, the three GIAB BEDs and the + /// Parascopy model list. + pub fn read( + clingen: &Path, + segdups: &Path, + gaps: &Path, + satellites: &Path, + parascopy: &Path, + ) -> Result { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let lines: Vec = super::lines(clingen, Utf8::Replace)? + .collect::>() + .map_err(|e| input_error(clingen, &e))?; + let regions = read_clingen(lines.iter().map(String::as_str), &mut cuts)?; + read_bed(segdups, |c| &mut c.segdup, &mut cuts)?; + read_bed(gaps, |c| &mut c.gap, &mut cuts)?; + read_bed(satellites, |c| &mut c.satellite, &mut cuts)?; + read_parascopy(parascopy, &mut cuts)?; + Ok(Inputs { regions, cuts }) + } + + /// The interval rows, without line endings, marker excluded. + pub fn rows(&self) -> Result, BuildError> { + Builder::new(self).build() + } +} + +/// The catalog with its rows from the marker line on replaced: the lines +/// before a marker line are kept (all of them when there is none). +fn splice(catalog: &[u8], rows: &[String]) -> Vec { + let mut out = Vec::with_capacity(catalog.len()); + for line in catalog.split_inclusive(|&c| c == b'\n') { + if line.starts_with(MARKER_START) && line.ends_with(b"\n") { + break; + } + out.extend_from_slice(line); + } + for line in std::iter::once(MARKER).chain(rows.iter().map(String::as_str)) { + out.extend_from_slice(line.as_bytes()); + out.push(b'\n'); + } + out +} + +/// How two catalogs differ: (lines only in `old`, lines only in `new`), +/// each line counted as often as it appears. +fn line_diff(old: &[u8], new: &[u8]) -> (usize, usize) { + let mut count: std::collections::HashMap<&[u8], i64> = std::collections::HashMap::new(); + for l in old.split_inclusive(|&c| c == b'\n') { + *count.entry(l).or_default() += 1; + } + for l in new.split_inclusive(|&c| c == b'\n') { + *count.entry(l).or_default() -= 1; + } + let only_old = count + .values() + .filter(|&&v| v > 0) + .map(|&v| v as usize) + .sum(); + let only_new = count + .values() + .filter(|&&v| v < 0) + .map(|&v| (-v) as usize) + .sum(); + (only_old, only_new) +} + +pub(super) fn cli( + args: &[String], + out: &mut dyn Write, + err: &mut dyn Write, +) -> Result { + let a = Args::parse( + args, + &[ + "--clingen", + "--segdups", + "--gaps", + "--satellites", + "--parascopy", + "--catalog", + ], + &["--check", "--stdout"], + )?; + let catalog = a.get("--catalog"); + if catalog.is_some() == a.has("--stdout") { + return Err(Failure::Usage( + "give one of --catalog FILE and --stdout".to_owned(), + )); + } + if a.has("--check") && catalog.is_none() { + return Err(Failure::Usage("--check needs --catalog".to_owned())); + } + let p = |f: &str| a.need(f).map(Path::new); + let inputs = Inputs::read( + p("--clingen")?, + p("--segdups")?, + p("--gaps")?, + p("--satellites")?, + p("--parascopy")?, + )?; + let rows = inputs.rows()?; + let Some(catalog) = catalog else { + let mut text = String::new(); + for line in std::iter::once(MARKER).chain(rows.iter().map(String::as_str)) { + text.push_str(line); + text.push('\n'); + } + let _ = out.write_all(text.as_bytes()); + return Ok(0); + }; + let path = Path::new(catalog); + let old = std::fs::read(path).map_err(|e| input_error(path, &e))?; + let new = splice(&old, &rows); + if a.has("--check") { + if new != old { + let (only_old, only_new) = line_diff(&old, &new); + let how = if only_old == 0 && only_new == 0 { + "the same lines in another order".to_owned() + } else { + format!("{only_old} line(s) only in the file, {only_new} only in the rebuild") + }; + let _ = writeln!( + err, + "{catalog} is not what the pinned downloads give ({how}); rerun scripts/ci/build-cn-intervals.sh" + ); + return Ok(1); + } + let _ = writeln!( + out, + "{catalog}: the {} interval rows are what the pinned downloads give", + rows.len() + ); + return Ok(0); + } + super::write_atomic(path, &new)?; + let _ = writeln!(out, "{catalog}: {} interval rows written", rows.len()); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A small ClinGen BED: the 22q11.2 A-D region between its breakpoint + /// rows, and the Xp22.31 region with its breakpoint, spelled `chrx`. + const SMALL_BED: &str = "track name=\"test\"\n#chrom\tstart\tend\tname\tscore\tstrand\tthickStart\tthickEnd\titemRgb\nchr22\t18159724\t18924717\tBP-A\t0\t.\t18159724\t18924717\t0,0,0\nchr22\t18924718\t21111383\t22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)\t0\t.\t18924718\t21111383\t255,111,0\nchr22\t21111384\t21562827\tBP-D\t0\t.\t21111384\t21562827\t0,0,0\nchrx\t6527561\t6537770\tXp22.31 distal breakpoint\t0\t.\t6527561\t6537770\t0,0,0\nchrx\t6537771\t8156913\tXp22.31 recurrent region (includes STS)\t0\t.\t6537771\t8156913\t255,111,0\n"; + + fn small(bed: &str) -> Result, BuildError> { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(bed.lines(), &mut cuts)?; + Inputs { regions, cuts }.rows() + } + + fn cell(row: &str, i: usize) -> &str { + row.split('\t').nth(i).unwrap() + } + + /// (start, end) of each target of a row. + fn spans(row: &str) -> Vec { + cell(row, 3) + .split(';') + .map(|t| { + let (s, e) = t.split(':').nth(1).unwrap().split_once('-').unwrap(); + (s.parse().unwrap(), e.parse().unwrap()) + }) + .collect() + } + + #[test] + fn the_small_bed_gives_its_two_regions_then_the_hand_rows() { + let rows = small(SMALL_BED).unwrap(); + let ids: Vec<(&str, &str)> = rows.iter().map(|r| (cell(r, 0), cell(r, 2))).collect(); + assert_eq!( + ids[..2], + [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")] + ); + let hand: Vec<&str> = ids[2..].iter().map(|x| x.0).collect(); + let want: Vec<&str> = HAND_ROWS.iter().map(|h| h.locus).collect(); + assert_eq!(hand, want, "the hand rows follow, in their order"); + assert!(rows.iter().all(|r| r.split('\t').count() == 22)); + assert!(rows.iter().all(|r| cell(r, 20) == "cnvpytor:cnvs")); + } + + #[test] + fn an_unknown_contig_stops_the_build() { + let e = small(&SMALL_BED.replace("chrx", "chrQ")).unwrap_err(); + assert!( + e.to_string().contains("'chrQ' is not a GRCh38 chromosome"), + "{e}" + ); + assert_eq!(e.code(), 2, "exit 2, as the Python builder did"); + } + + #[test] + fn no_target_piece_overlaps_a_breakpoint_row() { + let rows = small(SMALL_BED).unwrap(); + for (s, e) in spans(&rows[0]) { + assert!(!(s < 18_924_717 && 18_159_724 < e), "{s}-{e} on BP-A"); + assert!(!(s < 21_562_827 && 21_111_384 < e), "{s}-{e} on BP-D"); + } + // 2.19 Mb in pieces of at most 500 kb, the first labelled. + let t = cell(&rows[0], 3); + assert_eq!(t.split(';').count(), 5); + assert!(t.starts_with("22q11.2_DGS_proximal_A-D_t1:18924718-")); + assert!( + t.split(';') + .next() + .unwrap() + .ends_with(":22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)") + ); + } + + #[test] + fn controls_flank_the_cluster_and_never_sit_on_a_target() { + let rows = small(SMALL_BED).unwrap(); + // 22q11.2: the cluster runs from BP-A to BP-D, so the controls lie + // beyond those, 50 kb out. + assert_eq!( + cell(&rows[0], 4), + "22q11.2_DGS_proximal_A-D_L:18009724-18109724:flank left of the region;22q11.2_DGS_proximal_A-D_R:21612828-21712828:flank right of the region" + ); + for r in &rows { + if cell(r, 2) == "chrY" { + continue; + } + for c in cell(r, 4).split(';') { + let (s, e) = c.split(':').nth(1).unwrap().split_once('-').unwrap(); + let (cs, ce): (i64, i64) = (s.parse().unwrap(), e.parse().unwrap()); + for o in &rows { + if cell(o, 2) == cell(r, 2) { + assert!( + spans(o).iter().all(|&(ts, te)| !(ts < ce && cs < te)), + "{} control on {}", + cell(r, 0), + cell(o, 0) + ); + } + } + } + } + } + + #[test] + fn hand_rows_keep_their_spans_and_columns() { + let rows = small(SMALL_BED).unwrap(); + let get = |l: &str| rows.iter().find(|r| cell(r, 0) == l).unwrap().clone(); + assert_eq!(spans(&get("HBB_619bp_del")), [(5_225_388, 5_226_007)]); + assert_eq!( + spans(&get("HBD_HBB_Sicilian_del")), + [(5_219_877, 5_233_280)] + ); + let az = get("AZFc_DAZ"); + assert_eq!( + (cell(&az, 6), cell(&az, 7), cell(&az, 13), cell(&az, 14)), + ("sum", "pooled", "4", ".") + ); + assert_eq!(spans(&az).len(), 4, "the DAZ copies are never cut"); + assert!(cell(&az, 4).starts_with("AZFc_DAZ_c1:2935200-2982579:ZFY")); + let x = get("Xq28_MECP2"); + assert_eq!( + (cell(&x, 7), cell(&x, 13), cell(&x, 14)), + ("flanks", "8", "1.5") + ); + } + + #[test] + fn a_parascopy_model_is_cut_out_of_a_target() { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(SMALL_BED.lines(), &mut cuts).unwrap(); + cuts[21].parascopy.push((19_000_000, 19_100_000)); + let rows = Inputs { regions, cuts }.rows().unwrap(); + assert!( + spans(&rows[0]) + .iter() + .all(|&(s, e)| e <= 19_000_000 || s >= 19_100_000) + ); + assert!(spans(&rows[0]).iter().any(|&(_, e)| e == 19_000_000)); + } + + #[test] + fn a_parascopy_model_on_a_daz_copy_stops_the_build() { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(SMALL_BED.lines(), &mut cuts).unwrap(); + cuts[CHR_Y].parascopy.push((24_800_000, 24_800_100)); + let e = Inputs { regions, cuts }.rows().unwrap_err(); + assert!( + e.to_string() + .contains("AZFc_DAZ: DAZ3 overlaps a Parascopy model"), + "{e}" + ); + } + + #[test] + fn bad_rows_are_refused() { + let mut c = vec![Cuts::default(); CHROMS.len()]; + assert!( + read_clingen(["chr1\t1\t2"], &mut c) + .unwrap_err() + .to_string() + .contains("a row with 3 columns") + ); + let e = read_clingen(["chr1\t5\t5\tx\t0\t.\t5\t5\t255,111,0"], &mut c).unwrap_err(); + assert!(e.to_string().contains("x: empty span")); + let e = read_clingen(["chr1\t1\t5\tx\t0\t.\t1\t5\t1,2,3"], &mut c).unwrap_err(); + assert!(e.to_string().contains("itemRgb '1,2,3'")); + let e = read_clingen(["chr1\t1\t5\tx\t0\t.\t1\t5\t0,0,0"], &mut c).unwrap_err(); + assert!(e.to_string().contains("no region row")); + } + + #[test] + fn interval_helpers() { + assert_eq!( + merge(vec![(5, 8), (1, 3), (3, 4), (7, 10)]), + [(1, 4), (5, 10)] + ); + assert_eq!( + subtract(0, 20, &[(2, 4), (6, 8), (30, 40)]), + [(0, 2), (4, 6), (8, 20)] + ); + assert_eq!(subtract(3, 7, &[(0, 4), (6, 9)]), [(4, 6)]); + assert!(subtract(3, 7, &[(0, 9)]).is_empty()); + assert_eq!( + pieces(&[(0, 499), (0, 1_000_001)]), + [(0, 333_334), (333_334, 666_668), (666_668, 1_000_001)] + ); + assert_eq!(pieces(&[(0, 500)]), [(0, 500)]); + assert!(clear(4, 6, &[(0, 4), (6, 9)])); + assert!(!clear(3, 6, &[(0, 4), (6, 9)])); + assert_eq!(flank(0, 1_000_000, true, &[]), Some((850_000, 950_000))); + assert_eq!( + flank(0, 1_000_000, true, &[(900_000, 945_000)]), + Some((800_000, 900_000)) + ); + assert_eq!( + flank(0, 100_000, true, &[]), + None, + "past the chromosome start" + ); + } + + #[test] + fn locus_ids_and_labels() { + assert_eq!( + locus_id("22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)"), + "22q11.2_DGS_proximal_A-D" + ); + assert_eq!( + locus_id("Xp22.31 recurrent region (includes STS)"), + "Xp22.31" + ); + assert_eq!(locus_id("1q21.1 (a (b) c)"), "1q21.1_b"); + assert_eq!(label(" a;b\tc "), "a,b,c"); + } + + #[test] + fn a_repeated_locus_gets_a_number() { + let bed = "chr1\t10000000\t10100000\t1p (X) one\t0\t.\t0\t0\t255,111,0\nchr1\t20000000\t20100000\t1p (X) two\t0\t.\t0\t0\t255,111,0\n"; + let rows = small(bed).unwrap(); + assert_eq!((cell(&rows[0], 0), cell(&rows[1], 0)), ("1p_X", "1p_X_2")); + assert!(cell(&rows[1], 3).starts_with("1p_X_2_t1:")); + } + + #[test] + fn splice_keeps_the_head_and_check_counts_both_sides() { + let rows = vec!["a\t1".to_owned(), "b\t2".to_owned()]; + let old = b"# head\nx\n# --- interval rows: written by an older builder ---\nold\n"; + let new = splice(old, &rows); + assert_eq!( + new, + format!("# head\nx\n{MARKER}\na\t1\nb\t2\n").into_bytes() + ); + assert_eq!( + splice(b"x\n", &rows), + format!("x\n{MARKER}\na\t1\nb\t2\n").into_bytes() + ); + let added = [new.as_slice(), b"c\t3\n"].concat(); + assert_eq!(line_diff(&new, &added), (0, 1), "an added line is counted"); + assert_eq!(line_diff(&added, &new), (1, 0)); + let swapped = format!("# head\nx\n{MARKER}\nb\t2\na\t1\n").into_bytes(); + assert_eq!(line_diff(&new, &swapped), (0, 0)); + } +} diff --git a/rust/crates/og-dev/src/build/gtf.rs b/rust/crates/og-dev/src/build/gtf.rs new file mode 100644 index 000000000..89926e516 --- /dev/null +++ b/rust/crates/og-dev/src/build/gtf.rs @@ -0,0 +1,133 @@ +//! The GENCODE GTF reading the callability and SV rule builders share. +//! +//! A GTF line is read as bytes: its columns are found by their tabs, and an +//! attribute is found by a scan of the `key "value";` pairs of the ninth +//! column, only on the lines a builder keeps. GENCODE writes every pair as +//! `key "value"` or `key value` (a bare number), separated by `; `, and no +//! value holds a `;` or a `"`. + +use super::BuildError; + +/// One record of a GTF, by its columns. +pub(crate) struct Record<'a> { + pub seq: &'a [u8], + pub feature: &'a [u8], + start: &'a [u8], + end: &'a [u8], + pub attrs: &'a [u8], +} + +impl<'a> Record<'a> { + /// The record of a data line (without its line ending); `None` for a + /// comment. + pub fn parse(line: &'a [u8]) -> Result>, BuildError> { + if line.first() == Some(&b'#') { + return Ok(None); + } + let mut c = line.splitn(9, |&b| b == b'\t'); + let mut next = || { + c.next().ok_or_else(|| { + BuildError::Input(format!( + "GTF: a line with under 9 columns: {:?}", + String::from_utf8_lossy(line) + )) + }) + }; + let seq = next()?; + let _source = next()?; + let feature = next()?; + let start = next()?; + let end = next()?; + let (_score, _strand, _frame) = (next()?, next()?, next()?); + let attrs = next()?; + Ok(Some(Record { + seq, + feature, + start, + end, + attrs, + })) + } + + /// The 0-based start and the end. + pub fn span(&self) -> Result<(i64, i64), BuildError> { + Ok((number(self.start)? - 1, number(self.end)?)) + } + + /// The value of attribute `key`; the last one when it is given twice, + /// as a dict of the pairs keeps it. A value that is not UTF-8 is an + /// error. + pub fn attr(&self, key: &str) -> Result, BuildError> { + attr(self.attrs, key.as_bytes()) + .map(|v| { + std::str::from_utf8(v) + .map_err(|_| BuildError::Input(format!("GTF: {key} is not UTF-8: {v:?}"))) + }) + .transpose() + } +} + +/// A position column. +fn number(cell: &[u8]) -> Result { + std::str::from_utf8(cell) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + BuildError::Input(format!( + "GTF: not a whole number: {:?}", + String::from_utf8_lossy(cell) + )) + }) +} + +/// The value of the last `key "value"` pair of an attribute column. +pub(crate) fn attr<'a>(attrs: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let mut found = None; + for pair in attrs.split(|&b| b == b';') { + let pair = pair.trim_ascii_start(); + if let Some(rest) = pair.strip_prefix(key) + && let Some(quoted) = rest.strip_prefix(b" \"") + && let Some(end) = quoted.iter().position(|&b| b == b'"') + { + found = Some("ed[..end]); + } + } + found +} + +#[cfg(test)] +mod tests { + use super::*; + + const GENE: &[u8] = b"chr1\tHAVANA\tgene\t11\t20\t.\t+\t.\tgene_id \"ENSG1\"; gene_type \"protein_coding\"; gene_name \"A B\"; level 2; tag \"x\"; tag \"y\"; xgene_name \"no\";"; + + #[test] + fn a_gene_line_reads_its_columns_and_attributes() { + let r = Record::parse(GENE).unwrap().unwrap(); + assert_eq!((r.seq, r.feature), (&b"chr1"[..], &b"gene"[..])); + assert_eq!(r.span().unwrap(), (10, 20)); + assert_eq!(r.attr("gene_name").unwrap(), Some("A B"), "not xgene_name"); + assert_eq!( + r.attr("tag").unwrap(), + Some("y"), + "the last of a repeated key" + ); + assert_eq!( + r.attr("level").unwrap(), + None, + "an unquoted value is no pair" + ); + assert_eq!(r.attr("gene").unwrap(), None, "a key is matched whole"); + assert_eq!(r.attr("hgnc_id").unwrap(), None); + } + + #[test] + fn comments_pass_and_short_lines_stop() { + assert!(Record::parse(b"##description").unwrap().is_none()); + assert!(Record::parse(b"chr1\tx\tgene").is_err()); + let r = Record::parse(b"chr1\tx\tgene\tone\t2\t.\t+\t.\t") + .unwrap() + .unwrap(); + assert!(r.span().is_err()); + } +} diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs new file mode 100644 index 000000000..b2098e3dc --- /dev/null +++ b/rust/crates/og-dev/src/build/mod.rs @@ -0,0 +1,298 @@ +//! `og dev build`: the builders of the asset tables OwnGenome commits. +//! +//! Each verb rebuilds one committed file from pinned public downloads that +//! a shell wrapper in `scripts/ci/` has already fetched and checked against +//! its sha256 (`scripts/ci/lib/fetch-pinned.sh`): `og` itself never +//! downloads. The committed file is the oracle, byte for byte. +//! +//! - `og dev build cn-intervals` ([`cn_intervals`]): the interval rows of +//! `bin/cn_loci.tsv` (`scripts/ci/build-cn-intervals.sh`). +//! - `og dev build callability` ([`callability`]): +//! `assets/callability_genes.tsv` (`scripts/ci/build-callability.sh`). +//! - `og dev build sv-rule-regions` ([`sv_rules`]): +//! `assets/sv_rule_regions.tsv` (`scripts/ci/build-sv-rule-regions.sh`). +//! +//! Exit 0 when the file was written (or `--check` found it right); 1 when +//! `--check` found it different, or when the callability or SV rule +//! downloads hold something their rules refuse (a CMRG gene with no GENCODE +//! gene, a symbol listed twice, not one MHC region), as the Python builders +//! exited; 2 for a usage error or an input that cannot be read, and for +//! every cn-intervals input the rules refuse, as its Python builder exited. + +pub mod callability; +pub mod cn_intervals; +mod gtf; +pub mod sv_rules; + +use og_io::{Lines, Utf8}; +use std::fmt; +use std::io::{self, BufRead, Write}; +use std::path::Path; + +const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n"; + +/// GRCh38's primary chromosomes in their sort order: chr1-22, X, Y. +pub(crate) const CHROMS: [&str; 24] = [ + "chr1", "chr2", "chr3", "chr4", "chr5", "chr6", "chr7", "chr8", "chr9", "chr10", "chr11", + "chr12", "chr13", "chr14", "chr15", "chr16", "chr17", "chr18", "chr19", "chr20", "chr21", + "chr22", "chrX", "chrY", +]; + +/// The index of a primary chromosome in [`CHROMS`], from its name's bytes. +pub(crate) fn chrom_index(name: impl AsRef<[u8]>) -> Option { + let rest = name.as_ref().strip_prefix(b"chr")?; + match rest { + b"X" => Some(22), + b"Y" => Some(23), + [d @ b'1'..=b'9'] => Some(usize::from(d - b'1')), + [a @ b'1'..=b'2', b @ b'0'..=b'9'] => { + let n = usize::from(a - b'0') * 10 + usize::from(b - b'0'); + (n <= 22).then(|| n - 1) + } + _ => None, + } +} + +/// Why a build stopped: exit 1 for downloads the rules refuse, exit 2 for +/// an input that cannot be read. +#[derive(Debug)] +pub enum BuildError { + /// The inputs were read, but they hold what the rules refuse. + Data(String), + /// An input could not be read or parsed. + Input(String), +} + +impl BuildError { + fn code(&self) -> u8 { + match self { + BuildError::Data(_) => 1, + BuildError::Input(_) => 2, + } + } +} + +impl fmt::Display for BuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BuildError::Data(m) | BuildError::Input(m) => f.write_str(m), + } + } +} + +/// An input that cannot be read, named by its path. +pub(crate) fn input_error(path: &Path, e: &io::Error) -> BuildError { + BuildError::Input(crate::py_os_error(e, &path.to_string_lossy())) +} + +/// The lines of a plain or gzip file, without their endings, as Python's +/// text mode reads them. +pub(crate) fn lines(path: &Path, utf8: Utf8) -> Result>, BuildError> { + let r = og_io::open(path).map_err(|e| input_error(path, &e))?; + Ok(Lines::new(r, utf8)) +} + +/// Reads each line of `path` into `f`, stopping at the first error. +pub(crate) fn each_line( + path: &Path, + utf8: Utf8, + mut f: impl FnMut(&str) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + for line in lines(path, utf8)? { + f(&line.map_err(|e| input_error(path, &e))?)?; + } + Ok(()) +} + +/// Reads each line of `path` (plain or gzip) into `f` as bytes, without +/// its `\n` or `\r\n`, stopping at the first error. For the big files, +/// where a text line per line would cost more than the reading. +pub(crate) fn each_raw_line( + path: &Path, + mut f: impl FnMut(&[u8]) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + let mut r = og_io::open(path).map_err(|e| input_error(path, &e))?; + let mut buf = Vec::with_capacity(1 << 12); + loop { + buf.clear(); + if r.read_until(b'\n', &mut buf) + .map_err(|e| input_error(path, &e))? + == 0 + { + return Ok(()); + } + let mut line = buf.as_slice(); + if let Some(l) = line.strip_suffix(b"\n") { + line = l.strip_suffix(b"\r").unwrap_or(l); + } + f(line)?; + } +} + +/// A whole number of a text column, as Python's `int()` reads one. +pub(crate) fn int(cell: &str, what: &str) -> Result { + pycompat::text::strip(cell, None) + .parse() + .map_err(|_| BuildError::Input(format!("{what}: not a whole number: {cell:?}"))) +} + +/// Takes `--flag VALUE` pairs and bare `--switch`es from `args`. +pub(crate) struct Args { + values: Vec<(String, String)>, + switches: Vec, +} + +impl Args { + pub(crate) fn parse( + args: &[String], + flags: &[&str], + switches: &[&str], + ) -> Result { + let mut out = Args { + values: Vec::new(), + switches: Vec::new(), + }; + let mut it = args.iter(); + while let Some(a) = it.next() { + if flags.contains(&a.as_str()) { + let v = it + .next() + .ok_or_else(|| format!("argument {a}: expected one argument"))?; + if out.values.iter().any(|(k, _)| k == a) { + return Err(format!("argument {a}: given twice")); + } + out.values.push((a.clone(), v.clone())); + } else if switches.contains(&a.as_str()) { + out.switches.push(a.clone()); + } else { + return Err(format!("unrecognized arguments: {a}")); + } + } + Ok(out) + } + + pub(crate) fn get(&self, flag: &str) -> Option<&str> { + self.values + .iter() + .find(|(k, _)| k == flag) + .map(|(_, v)| v.as_str()) + } + + pub(crate) fn need(&self, flag: &str) -> Result<&str, String> { + self.get(flag) + .ok_or_else(|| format!("the following arguments are required: {flag}")) + } + + pub(crate) fn has(&self, switch: &str) -> bool { + self.switches.iter().any(|s| s == switch) + } +} + +/// `og dev build VERB ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let result = match args.first().map(String::as_str) { + Some("cn-intervals") => cn_intervals::cli(&args[1..], out, err), + Some("callability") => callability::cli(&args[1..], err), + Some("sv-rule-regions") => sv_rules::cli(&args[1..], err), + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => Err(Failure::Usage(format!("unknown builder '{other}'"))), + None => Err(Failure::Usage( + "the following arguments are required: builder".to_owned(), + )), + }; + match result { + Ok(code) => code, + Err(Failure::Usage(m)) => { + let _ = writeln!(err, "{USAGE}og dev build: error: {m}"); + 2 + } + Err(Failure::Build(e)) => { + let _ = writeln!(err, "ERROR: {e}"); + e.code() + } + } +} + +/// How a builder's command line ended badly. +pub(crate) enum Failure { + Usage(String), + Build(BuildError), +} + +impl From for Failure { + fn from(e: BuildError) -> Self { + Failure::Build(e) + } +} + +impl From for Failure { + fn from(m: String) -> Self { + Failure::Usage(m) + } +} + +/// Writes `bytes` to `path` through a temporary file renamed over it. +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), BuildError> { + let mut f = og_io::AtomicFile::create(path).map_err(|e| input_error(path, &e))?; + f.write_all(bytes).map_err(|e| input_error(path, &e))?; + f.commit().map_err(|e| input_error(path, &e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(v: &[&str]) -> Vec { + v.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn unknown_builder_is_a_usage_error() { + let (mut o, mut e) = (Vec::new(), Vec::new()); + assert_eq!(cli(&s(&["nope"]), &mut o, &mut e), 2); + assert!( + String::from_utf8(e) + .unwrap() + .contains("unknown builder 'nope'") + ); + assert_eq!(cli(&[], &mut o, &mut Vec::new()), 2); + } + + #[test] + fn args_refuse_a_repeated_or_unknown_flag() { + assert!(Args::parse(&s(&["--out", "a", "--out", "b"]), &["--out"], &[]).is_err()); + assert!(Args::parse(&s(&["--what"]), &["--out"], &[]).is_err()); + assert!(Args::parse(&s(&["--out"]), &["--out"], &[]).is_err()); + let a = Args::parse(&s(&["--out", "a", "--check"]), &["--out"], &["--check"]).unwrap(); + assert_eq!(a.get("--out"), Some("a")); + assert!(a.has("--check")); + } + + #[test] + fn chromosome_names() { + for (i, c) in CHROMS.iter().enumerate() { + assert_eq!(chrom_index(c), Some(i)); + } + for c in [ + "chr0", + "chr23", + "chr01", + "chrM", + "chr1_KI270706v1_random", + "1", + "chrx", + "", + ] { + assert_eq!(chrom_index(c), None, "{c}"); + } + } + + #[test] + fn int_reads_as_python_does() { + assert_eq!(int(" 12 ", "x").unwrap(), 12); + assert!(int("1.5", "x").is_err()); + } +} diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs new file mode 100644 index 000000000..7004f02b7 --- /dev/null +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -0,0 +1,649 @@ +//! `og dev build sv-rule-regions`: `assets/sv_rule_regions.tsv`, the GRCh38 +//! regions where `og sv postfilter` overrides AnnotSV's class 4 or 5. +//! +//! All on chr1-22, X and Y: +//! +//! - `tandem_repeat`: every TRExplorer v2.1 repeat of 50 bp or more (the +//! shortest SV) that overlaps a coding exon of a MANE Select transcript; +//! name `GENE:MOTIF` (a motif over 12 bases as its length); +//! - `ig_tr_locus`: the span of GENCODE's IG and TR genes of each of IGH, +//! IGK, IGL, TRA (with TRD inside it), TRB and TRG: per locus the largest +//! group of genes on one chromosome with gaps under 1 Mb, so the orphon +//! copies on other chromosomes stay out; +//! - `mhc`: the MHC region of NCBI's assembly regions report; +//! - `gene_alias`: the GENCODE span of each HGNC pseudogene whose alias or +//! previous symbol is the approved symbol of a gene with a GenCC +//! curation; name `ALIAS|PSEUDOGENE|its HGNC id|the HGNC id of the gene +//! that holds ALIAS as its symbol`. +//! +//! A gene's span is that of the first GENCODE record of its HGNC id, and the +//! `_PAR_Y` copies of the pseudoautosomal genes are skipped, as the +//! callability builder does: the Python this replaces kept the last record, +//! so a PAR pseudogene would have taken its chrY span (none does in the +//! pinned files, so the committed table is the same). +//! +//! The GTF and the catalog are streamed once. The MANE Select coding exons +//! of each chromosome are sorted by start with a running maximum of their +//! ends, so each repeat finds the exons it overlaps by binary search. + +use super::gtf::Record; +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, each_raw_line, int}; +use og_io::{Lines, Utf8}; +use pycompat::csv::{DictReader, Quoting}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::io::{self, Write}; +use std::path::Path; + +/// The tag of a MANE Select transcript's records. +const MANE: &[u8] = b"tag \"MANE_Select\""; +const MIN_TR: i64 = 50; +const LOCUS_GAP: i64 = 1_000_000; +const SRC_TR: &str = "TRExplorer v2.1; GENCODE 50 MANE Select CDS"; +const SRC_IG: &str = "GENCODE 50 IG and TR genes"; +const SRC_MHC: &str = "GRCh38 assembly regions (GCA_000001405.15)"; +const SRC_ALIAS: &str = "HGNC 2026-10-06; GENCODE 50"; + +/// One row of the table. +struct Row { + chrom: usize, + start: i64, + end: i64, + rule: &'static str, + action: &'static str, + name: String, + source: &'static str, +} + +/// What the GTF gives. +#[derive(Default)] +struct Gtf { + /// HGNC id -> (chrom, start, end) of its first record. + span: HashMap, + /// IG or TR locus -> its genes as (chrom name, start, end). + ig: BTreeMap>, + /// Per chromosome, the MANE Select coding exons: (start, end, gene). + cds: Vec>, +} + +/// The IG or TR locus of a gene, from its type and symbol. +fn ig_locus(gene_type: &str, name: &str) -> Option { + let ig_type = gene_type.starts_with("IG_") || gene_type.starts_with("TR_"); + let b = name.as_bytes(); + let ig_name = b.len() >= 3 + && matches!( + (b[0], b[1], b[2]), + (b'I', b'G', b'H' | b'K' | b'L') | (b'T', b'R', b'A' | b'B' | b'D' | b'G') + ); + (ig_type && ig_name).then(|| { + if name.starts_with("TRD") { + "TRA".to_owned() + } else { + name[..3].to_owned() + } + }) +} + +fn read_gtf(path: &Path) -> Result { + let mut g = Gtf { + cds: vec![Vec::new(); CHROMS.len()], + ..Gtf::default() + }; + each_raw_line(path, |line| { + let Some(r) = Record::parse(line)? else { + return Ok(()); + }; + let Some(chrom) = chrom_index(r.seq) else { + return Ok(()); + }; + match r.feature { + b"gene" => { + let (start, end) = r.span()?; + if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) + { + g.span.entry(h.to_owned()).or_insert((chrom, start, end)); + } + let (name, ty) = ( + r.attr("gene_name")?.unwrap_or(""), + r.attr("gene_type")?.unwrap_or(""), + ); + if let Some(locus) = ig_locus(ty, name) { + g.ig.entry(locus) + .or_default() + .push((CHROMS[chrom], start, end)); + } + } + b"CDS" if r.attrs.windows(MANE.len()).any(|w| w == MANE) => { + let name = r.attr("gene_name")?.ok_or_else(|| { + BuildError::Input(format!( + "GTF: a MANE Select CDS with no gene_name: {:?}", + String::from_utf8_lossy(line) + )) + })?; + let (start, end) = r.span()?; + g.cds[chrom].push((start, end, name.to_owned())); + } + _ => {} + } + Ok(()) + })?; + Ok(g) +} + +/// The MANE Select coding exons of one chromosome, sorted, with the +/// running maximum of their ends. +struct Exons { + iv: Vec<(i64, i64, String)>, + max_end: Vec, +} + +impl Exons { + fn new(mut iv: Vec<(i64, i64, String)>) -> Exons { + iv.sort_unstable(); + let mut m = i64::MIN; + let max_end = iv + .iter() + .map(|x| { + m = m.max(x.1); + m + }) + .collect(); + Exons { iv, max_end } + } + + /// The genes of the exons that overlap `[s, e)`, sorted. + fn genes(&self, s: i64, e: i64) -> BTreeSet<&str> { + let mut out = BTreeSet::new(); + let mut j = self.iv.partition_point(|x| x.0 < e); + while j > 0 && self.max_end[j - 1] > s { + j -= 1; + if self.iv[j].1 > s { + out.insert(self.iv[j].2.as_str()); + } + } + out + } +} + +/// tandem_repeat rows: catalog repeats of `MIN_TR` bp or more on a MANE +/// Select coding exon. +fn tandem_repeats( + path: &Path, + cds: Vec>, + rows: &mut Vec, +) -> Result<(), BuildError> { + let index: Vec> = cds + .into_iter() + .map(|iv| (!iv.is_empty()).then(|| Exons::new(iv))) + .collect(); + each_raw_line(path, |line| { + let mut c = line.splitn(5, |&b| b == b'\t'); + let Some(ex) = c + .next() + .and_then(chrom_index) + .and_then(|i| index[i].as_ref().map(|x| (i, x))) + else { + return Ok(()); + }; + let short = || { + BuildError::Input(format!( + "{}: a row with under 4 columns: {:?}", + path.display(), + String::from_utf8_lossy(line) + )) + }; + let (s, e) = (c.next().ok_or_else(short)?, c.next().ok_or_else(short)?); + let (s, e) = (bytes_int(s)?, bytes_int(e)?); + if e - s < MIN_TR { + return Ok(()); + } + let genes = ex.1.genes(s, e); + if !genes.is_empty() { + let motif = std::str::from_utf8(c.next().ok_or_else(short)?).map_err(|_| { + BuildError::Input(format!("{}: a motif that is not UTF-8", path.display())) + })?; + let n = motif.chars().count(); + let motif = if n <= 12 { + motif.to_owned() + } else { + format!("{n}bp motif") + }; + rows.push(Row { + chrom: ex.0, + start: s, + end: e, + rule: "tandem_repeat", + action: "class_3", + name: format!( + "{}:{motif}", + genes.into_iter().collect::>().join("/") + ), + source: SRC_TR, + }); + } + Ok(()) + }) +} + +/// A position of the catalog. +fn bytes_int(cell: &[u8]) -> Result { + int(&String::from_utf8_lossy(cell), "catalog position") +} + +/// ig_tr_locus rows: per locus the largest group of genes on one +/// chromosome with gaps under `LOCUS_GAP` (the first such group on a tie). +fn ig_loci(ig: BTreeMap>, rows: &mut Vec) { + for (locus, mut genes) in ig { + genes.sort_unstable(); + // (chrom, start, end, genes) + let mut groups: Vec<(&str, i64, i64, usize)> = Vec::new(); + for (chrom, s, e) in genes { + match groups.last_mut() { + Some(g) if g.0 == chrom && s - g.2 < LOCUS_GAP => { + g.2 = g.2.max(e); + g.3 += 1; + } + _ => groups.push((chrom, s, e, 1)), + } + } + let mut best = groups[0]; + for g in &groups[1..] { + if g.3 > best.3 { + best = *g; + } + } + rows.push(Row { + chrom: chrom_index(best.0).expect("only chr1-22, X and Y are read"), + start: best.1, + end: best.2, + rule: "ig_tr_locus", + action: "class_3", + name: locus, + source: SRC_IG, + }); + } +} + +/// The mhc row: the assembly report's one MHC region (1-based, inclusive). +fn mhc(path: &Path) -> Result { + let mut found: BTreeSet> = BTreeSet::new(); + each_line(path, Utf8::Strict, |line| { + if line.starts_with("MHC\t") { + found.insert(line.split('\t').take(4).map(str::to_owned).collect()); + } + Ok(()) + })?; + let one = match found.len() { + 1 => found.pop_first().unwrap_or_default(), + n => { + return Err(BuildError::Data(format!( + "want one MHC region in the assembly report, got {n}: {found:?}" + ))); + } + }; + let bad = || { + BuildError::Input(format!( + "{}: an MHC row with under 4 columns: {one:?}", + path.display() + )) + }; + let (chrom, s, e) = ( + one.get(1).ok_or_else(bad)?, + one.get(2).ok_or_else(bad)?, + one.get(3).ok_or_else(bad)?, + ); + let name = format!("chr{chrom}"); + Ok(Row { + chrom: chrom_index(&name).ok_or_else(|| { + BuildError::Input(format!("{}: the MHC is on {name}", path.display())) + })?, + start: int(s, "MHC start")? - 1, + end: int(e, "MHC end")?, + rule: "mhc", + action: "class_3", + name: "MHC".to_owned(), + source: SRC_MHC, + }) +} + +/// The lines of a file with their endings, as a file opened with +/// `newline=""` hands them to `csv`. +struct RawLines(Lines>); + +impl Iterator for RawLines { + type Item = io::Result; + fn next(&mut self) -> Option { + let mut s = String::new(); + match self.0.read_line(&mut s) { + Ok(true) => Some(Ok(s)), + Ok(false) => None, + Err(e) => Some(Err(e)), + } + } +} + +/// gene_alias rows: a pseudogene that carries a disease gene's symbol as an +/// alias or previous symbol. +fn gene_aliases( + path: &Path, + span: &HashMap, + rows: &mut Vec, +) -> Result<(), BuildError> { + let csv_error = |e: pycompat::csv::Error| BuildError::Input(format!("{}: {e}", path.display())); + // symbol -> (HGNC id, has a GenCC curation); the last row of a symbol. + let mut approved: HashMap = HashMap::new(); + // (HGNC id, symbol, the alias and previous symbols) of the pseudogenes + // GENCODE places. + let mut pseudo: Vec<(String, String, BTreeSet)> = Vec::new(); + let lines = RawLines(super::lines(path, Utf8::Strict)?); + for row in DictReader::new(lines, '\t', Quoting::Minimal) { + let row = row.map_err(csv_error)?; + let get = |k: &str| { + row.get(k).flatten().ok_or_else(|| { + BuildError::Input(format!("{}: a row with no {k} cell", path.display())) + }) + }; + let (symbol, hgnc) = (get("symbol")?, get("hgnc_id")?); + approved.insert( + symbol.to_owned(), + (hgnc.to_owned(), !get("gencc")?.is_empty()), + ); + if get("locus_group")? == "pseudogene" && span.contains_key(hgnc) { + let names = format!("{}|{}", get("alias_symbol")?, get("prev_symbol")?); + let syms = names + .split('|') + .map(|x| x.trim_matches('"')) + .filter(|x| !x.is_empty()) + .map(str::to_owned) + .collect(); + pseudo.push((hgnc.to_owned(), symbol.to_owned(), syms)); + } + } + for (hgnc, symbol, syms) in pseudo { + for sym in syms { + if let Some((other, true)) = approved.get(&sym) + && *other != hgnc + { + let (chrom, start, end) = span[&hgnc]; + rows.push(Row { + chrom, + start, + end, + rule: "gene_alias", + action: "remap_gene", + name: format!("{sym}|{symbol}|{hgnc}|{other}"), + source: SRC_ALIAS, + }); + } + } + } + Ok(()) +} + +/// The table's text and its row counts line, from the downloads in `work`. +pub fn build(work: &Path) -> Result<(String, String), BuildError> { + let gtf = read_gtf(&work.join("gencode.gtf.gz"))?; + let mut rows = Vec::new(); + tandem_repeats(&work.join("trx.bed.gz"), gtf.cds, &mut rows)?; + ig_loci(gtf.ig, &mut rows); + rows.push(mhc(&work.join("regions.txt"))?); + gene_aliases(&work.join("hgnc.txt"), >f.span, &mut rows)?; + rows.sort_by(|a, b| { + (a.chrom, a.start, a.end, a.rule, &a.name).cmp(&(b.chrom, b.start, b.end, b.rule, &b.name)) + }); + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for r in &rows { + *counts.entry(r.rule).or_default() += 1; + } + let counts: Vec = counts.iter().map(|(k, n)| format!("{k} {n}")).collect(); + let summary = format!("Rows: {}", counts.join(", ")); + let mut text = String::from( + "# assets/sv_rule_regions.tsv: the GRCh38 regions where og sv postfilter overrides AnnotSV's class 4\n\ + # or 5. Written by scripts/ci/build-sv-rule-regions.sh, which says how each rule's rows are chosen;\n\ + # rebuild it there, do not edit it by hand. start is 0-based and end 1-based, as in a BED file.\n", + ); + text.push_str(&format!( + "# {summary}.\n#chrom\tstart\tend\trule\taction\tname\tsource\n" + )); + for r in &rows { + text.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + CHROMS[r.chrom], r.start, r.end, r.rule, r.action, r.name, r.source + )); + } + Ok((text, summary)) +} + +pub(super) fn cli(args: &[String], err: &mut dyn Write) -> Result { + let a = Args::parse(args, &["--work", "--out"], &[])?; + let (work, out) = (a.need("--work")?, a.need("--out")?); + let (text, summary) = build(Path::new(work))?; + super::write_atomic(Path::new(out), text.as_bytes())?; + let _ = writeln!(err, "{summary}"); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn gz(text: &str) -> Vec { + let mut g = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + g.write_all(text.as_bytes()).unwrap(); + g.finish().unwrap() + } + + fn gene(seq: &str, s: u32, e: u32, id: &str, ty: &str, name: &str, h: &str) -> String { + format!( + "{seq}\tX\tgene\t{s}\t{e}\t.\t+\t.\tgene_id \"{id}\"; gene_type \"{ty}\"; gene_name \"{name}\"; hgnc_id \"{h}\";\n" + ) + } + + fn cds(seq: &str, s: u32, e: u32, name: &str, mane: bool) -> String { + let tag = if mane { " tag \"MANE_Select\";" } else { "" }; + format!("{seq}\tX\tCDS\t{s}\t{e}\t.\t+\t0\tgene_id \"x\"; gene_name \"{name}\";{tag}\n") + } + + /// A work folder with a few rows of each rule; `par_first` puts the + /// _PAR_Y copy of the pseudogene PSP before its chrX record. + fn fixture(dir: &Path, par_first: bool) { + fs::create_dir_all(dir).unwrap(); + let px = gene( + "chrX", + 1001, + 2000, + "GP", + "unprocessed_pseudogene", + "PSP", + "HGNC:30", + ); + let py = gene( + "chrY", + 1001, + 2000, + "GP_PAR_Y", + "unprocessed_pseudogene", + "PSP", + "HGNC:30", + ); + let par = if par_first { + [py.clone(), px.clone()] + } else { + [px, py] + }; + let gtf = [ + "##description: test\n".to_owned(), + gene("chr1", 1001, 3000, "G1", "protein_coding", "AAA", "HGNC:1"), + cds("chr1", 1101, 1300, "AAA", true), + cds("chr1", 1251, 1400, "AAB", true), + cds("chr1", 2001, 2400, "AAA", false), + gene( + "chr14", + 100_001, + 100_500, + "I1", + "IG_V_gene", + "IGHV1", + "HGNC:10", + ), + gene( + "chr14", + 900_001, + 900_500, + "I2", + "IG_C_gene", + "IGHM", + "HGNC:11", + ), + gene( + "chr15", + 1, + 500, + "I3", + "IG_V_pseudogene", + "IGHV1OR15", + "HGNC:12", + ), + gene( + "chr14", + 20_001, + 20_500, + "T1", + "TR_V_gene", + "TRDV1", + "HGNC:13", + ), + gene( + "chr14", + 10_001, + 10_500, + "T2", + "TR_V_gene", + "TRAV1", + "HGNC:14", + ), + gene( + "chr2", + 5001, + 6000, + "G2", + "processed_pseudogene", + "PSA", + "HGNC:20", + ), + par[0].clone(), + par[1].clone(), + ] + .concat(); + fs::write(dir.join("gencode.gtf.gz"), gz(>f)).unwrap(); + fs::write( + dir.join("trx.bed.gz"), + gz("chr1\t1000\t1049\tCAG\nchr1\t1200\t1260\tCAG\nchr1\t1280\t1340\tAAAAAAAAAAAAAT\nchr1\t2100\t2200\tGC\nchrM\t1\t100\tA\n"), + ) + .unwrap(); + fs::write( + dir.join("regions.txt"), + "# Region-Name\tChromosome\tStart\tStop\tRole\nMHC\t6\t28510120\t33480577\tx\r\nMHC\t6\t28510120\t33480577\tx\r\nOTHER\t1\t1\t2\tx\n", + ) + .unwrap(); + fs::write( + dir.join("hgnc.txt"), + "hgnc_id\tsymbol\tlocus_group\talias_symbol\tprev_symbol\tgencc\n\ + HGNC:1\tAAA\tprotein-coding gene\t\t\t\"curated\"\n\ + HGNC:2\tBBB\tprotein-coding gene\t\t\t\n\ + HGNC:3\tCCC\tprotein-coding gene\t\t\tyes\n\ + HGNC:20\tPSA\tpseudogene\t\"AAA|BBB\"\tPSA\t\n\ + HGNC:30\tPSP\tpseudogene\tCCC\t\t\n\ + HGNC:40\tPSZ\tpseudogene\tAAA\t\t\n", + ) + .unwrap(); + } + + #[test] + fn each_rule_gives_its_rows() { + let dir = std::env::temp_dir().join(format!("og-svrules-{}", std::process::id())); + fixture(&dir, false); + let (text, summary) = build(&dir).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + let body: Vec<&str> = text.lines().skip(4).collect(); + assert_eq!( + body, + [ + "#chrom\tstart\tend\trule\taction\tname\tsource", + "chr1\t1200\t1260\ttandem_repeat\tclass_3\tAAA/AAB:CAG\tTRExplorer v2.1; GENCODE 50 MANE Select CDS", + "chr1\t1280\t1340\ttandem_repeat\tclass_3\tAAA/AAB:14bp motif\tTRExplorer v2.1; GENCODE 50 MANE Select CDS", + "chr2\t5000\t6000\tgene_alias\tremap_gene\tAAA|PSA|HGNC:20|HGNC:1\tHGNC 2026-10-06; GENCODE 50", + "chr6\t28510119\t33480577\tmhc\tclass_3\tMHC\tGRCh38 assembly regions (GCA_000001405.15)", + "chr14\t10000\t20500\tig_tr_locus\tclass_3\tTRA\tGENCODE 50 IG and TR genes", + "chr14\t100000\t900500\tig_tr_locus\tclass_3\tIGH\tGENCODE 50 IG and TR genes", + "chrX\t1000\t2000\tgene_alias\tremap_gene\tCCC|PSP|HGNC:30|HGNC:3\tHGNC 2026-10-06; GENCODE 50", + ] + ); + assert_eq!( + summary, + "Rows: gene_alias 2, ig_tr_locus 2, mhc 1, tandem_repeat 2" + ); + assert!(text.contains("\n# Rows: gene_alias 2, ig_tr_locus 2, mhc 1, tandem_repeat 2.\n")); + } + + #[test] + fn a_par_pseudogene_keeps_its_chrx_span_whichever_record_comes_first() { + let dir = std::env::temp_dir().join(format!("og-svrules-par-{}", std::process::id())); + fixture(&dir, true); + let (text, _) = build(&dir).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert!( + text.contains("\nchrX\t1000\t2000\tgene_alias\tremap_gene\tCCC|PSP|HGNC:30|HGNC:3\t"), + "{text}" + ); + assert!(!text.contains("\nchrY\t"), "{text}"); + } + + #[test] + fn two_mhc_regions_stop_the_build() { + let dir = std::env::temp_dir().join(format!("og-svrules-mhc-{}", std::process::id())); + fixture(&dir, false); + fs::write( + dir.join("regions.txt"), + "MHC\t6\t1\t2\tx\nMHC\t6\t1\t3\tx\n", + ) + .unwrap(); + let e = build(&dir).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert!(matches!(e, BuildError::Data(_))); + assert!( + e.to_string() + .starts_with("want one MHC region in the assembly report, got 2"), + "{e}" + ); + } + + #[test] + fn exon_lookup_finds_every_overlap() { + let ex = Exons::new(vec![ + (0, 100, "A".into()), + (10, 20, "B".into()), + (50, 60, "C".into()), + (200, 300, "D".into()), + ]); + assert_eq!( + ex.genes(15, 55).into_iter().collect::>(), + ["A", "B", "C"] + ); + assert_eq!(ex.genes(100, 200).len(), 0, "half-open on both sides"); + assert_eq!( + ex.genes(99, 201).into_iter().collect::>(), + ["A", "D"] + ); + } + + #[test] + fn ig_loci_by_type_and_symbol() { + assert_eq!(ig_locus("IG_V_gene", "IGKV1").as_deref(), Some("IGK")); + assert_eq!(ig_locus("TR_D_gene", "TRDD1").as_deref(), Some("TRA")); + assert_eq!(ig_locus("protein_coding", "IGHM"), None); + assert_eq!(ig_locus("TR_V_gene", "TRX1"), None); + assert_eq!(ig_locus("IG_V_gene", "IG"), None); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 55c93fdf0..88c683e45 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -12,7 +12,10 @@ //! the report carries the report-only error strategy. //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): +//! rebuild a committed asset table from its pinned downloads. +pub mod build; pub mod contract; pub mod parity; pub mod remnants; @@ -22,7 +25,7 @@ mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -57,6 +60,7 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("build") => run(build::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 234ea7cac..a34aa302f 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -74,6 +74,7 @@ const USAGE: &str = "usage: og [args] og exomiser report [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] + og dev build cn-intervals|callability|sv-rule-regions [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash From f47eb8a998025a300368b015e69ca5d59910bd7f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:46 +0100 Subject: [PATCH 02/24] build(ci): switch the asset builders to og dev build, one fetch rule build-callability.sh and build-sv-rule-regions.sh keep their downloads and hand the folder to og dev build; their Python heredocs are gone. build-cn-intervals.py becomes build-cn-intervals.sh with the same --list, --cache and --check. The three share scripts/ci/lib/fetch-pinned.sh: a cached file with the right sha256 is read in place, a 403 or 404 fails at once, anything else is retried for about five minutes. The marker line of bin/cn_loci.tsv names the new script; no row changes. The cn-intervals case, the e2e data fill and the converter checks of tests/test_copy_number.py call the new script and og. Bead: pgp-bs9u.35 --- CHANGELOG.md | 6 + bin/cn_loci.tsv | 2 +- bin/copy_number.py | 2 +- docs/cn-loci.md | 6 +- docs/rust.md | 2 +- docs/testing.md | 2 +- scripts/ci/build-callability.sh | 218 +------------- scripts/ci/build-cn-intervals.py | 423 ---------------------------- scripts/ci/build-cn-intervals.sh | 97 +++++++ scripts/ci/build-sv-rule-regions.sh | 171 ++--------- scripts/ci/e2e-case-paths.tsv | 6 +- scripts/ci/e2e-run.sh | 2 +- scripts/ci/lib/fetch-pinned.sh | 61 ++++ tests/e2e/cn-intervals-1-catalog.sh | 10 +- tests/test_copy_number.py | 77 ++--- tests/test_e2e_data_cache.sh | 2 +- versions.env | 2 +- 17 files changed, 255 insertions(+), 834 deletions(-) delete mode 100755 scripts/ci/build-cn-intervals.py create mode 100755 scripts/ci/build-cn-intervals.sh create mode 100644 scripts/ci/lib/fetch-pinned.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e21d1ac8..7ce7f1507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. + - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". + - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. + - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. - A sample name with `*`, `?` or `[` is matched as written. The Python read it as a file pattern. diff --git a/bin/cn_loci.tsv b/bin/cn_loci.tsv index aa1b5d4c6..6f63e7cf7 100644 --- a/bin/cn_loci.tsv +++ b/bin/cn_loci.tsv @@ -66,7 +66,7 @@ HP gene chr16 HP_UNIT1:72056529-72057466:HP exons 3 and 4, the first copy of the LCE3BC interval chr1 LCE3BC_DEL:152600233-152614098:LCE3C to LCE3B, inside the LCE3C_LCE3B deletion LCE3_FLANK_L:152520000-152560000:flank on the LCE3D side, outside the deletion;LCE3_FLANK_R:152640000-152680000:flank on the LCE3A side, outside the deletion . sum flanks 2 0 .,-Q 1 . . 8 1.5 round:0.3 . 0=LCE3B and LCE3C deleted on both chromosomes (LCE3C_LCE3B-del on both), a common genotype that goes with a higher psoriasis risk|1=LCE3B and LCE3C deleted on one chromosome (LCE3C_LCE3B-del on one)|2=no LCE3B/LCE3C deletion seen|more={n} copies of the LCE3B/LCE3C stretch: more than two, a duplication bedcov cn-depth none 19169253 CCL3L1 gene chr17 CCL3L3:36194868-36196758:CCL3L3, the one copy of CCL3L1/CCL3L3 on the reference (ENST00000619989) CCL3L_FLANK_L:36112000-36148000:single-copy sequence between the CCL3 and CCL3L3 duplications;CCL3L_FLANK_R:36460000-36500000:single-copy sequence past the duplications . sum flanks 2 0 .,-Q 1 . . 8 1.5 round:0.3 . 0=no copy of CCL3L1 or CCL3L3|1=one copy of CCL3L1/CCL3L3 in all|more={n} copies of CCL3L1/CCL3L3 in all: the count differs widely between people, and no reading of it is given bedcov cn-depth none 15637236 CYP21A2 gene chr6 CYP21A2_BODY:32038950-32041644:CYP21A2 from the end of the C4A model to its last base (NM_000500.9);CYP21A1P_BODY:32006220-32008909:the same stretch of CYP21A1P (NR_040090.1) RCCX_FLANK_L:31945000-31975000:single-copy sequence before the RCCX duplication;RCCX_FLANK_R:32070000-32100000:single-copy sequence in TNXB past the duplication PSV_In2G:32039080-32039081:32006352 AC>G c.293-13A/C>G (In2G, intron 2);PSV_I172N:32039425-32039426:32006689 T>A c.518T>A (p.I172N, exon 4);PSV_V237E:32039809-32039810:32007073 T>A c.713T>A (p.V237E, exon 6 cluster);PSV_V281L:32040109-32040110:32007373 G>T c.844G>T (p.V281L, exon 7);PSV_Q318X:32040420-32040421:32007685 C>T c.955C>T (p.Q318X, exon 8);PSV_R356W:32040534-32040535:32007799 C>T c.1069C>T (p.R356W, exon 8) sum flanks 2 0 .,-Q 1 0.02 . 8 1.5 psv:0.4 . none=no sign of the 30 kb chimeric deletion or of a pseudogene conversion at the six sites read; short reads see neither a change elsewhere in CYP21A2 nor two copies on one chromosome and none on the other|deletion=one RCCX module fewer and one CYP21A2 copy fewer at the differing sites: the pattern of the 30 kb CYP21A1P/CYP21A2 chimeric deletion, a common cause of carrier status for 21-hydroxylase deficiency (congenital adrenal hyperplasia); unconfirmed: a clinical test (MLPA or long-range PCR) confirms it|conversion=fewer CYP21A2 copies carry the gene's own base at {sites} than at the other sites: a possible conversion of one CYP21A2 copy to the pseudogene sequence there, a carrier pattern for 21-hydroxylase deficiency (congenital adrenal hyperplasia) when it sits on one chromosome; which chromosome carries it is not known; unconfirmed: a clinical test confirms it|pseudogene_deletion=one RCCX module fewer with CYP21A2 kept at every site read: a CYP21A1P (pseudogene) deletion, common and not a carrier finding bedcov cn-depth paraphase:rccx 34285390,27228465 -# --- interval rows: written by scripts/ci/build-cn-intervals.py; do not edit by hand --- +# --- interval rows: written by scripts/ci/build-cn-intervals.sh; do not edit by hand --- 1q21.1_TAR_syndrome_proximal_BP2-BP3 interval chr1 1q21.1_TAR_syndrome_proximal_BP2-BP3_t1:145686999-145809382:1q21.1 recurrent (TAR syndrome) region (proximal, BP2-BP3) (includes RBM8A);1q21.1_TAR_syndrome_proximal_BP2-BP3_t2:145823828-145923649;1q21.1_TAR_syndrome_proximal_BP2-BP3_t3:145927476-146048495 1q21.1_TAR_syndrome_proximal_BP2-BP3_L:143034587-143134587:flank left of the region;1q21.1_TAR_syndrome_proximal_BP2-BP3_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4 interval chr1 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t1:145687000-145809382:1q21.1 recurrent (TAR syndrome) region (proximal and distal, BP1-BP4) (includes RBM8A and GJA5);1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t2:145823828-145923649;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t3:145927476-146048496;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t4:147105903-147473859;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t5:147473859-147841815;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t6:147842996-147917509 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_L:143034587-143134587:flank left of the region;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . 1q21.1_distal_BP3-BP4 interval chr1 1q21.1_distal_BP3-BP4_t1:147105904-147473860:1q21.1 recurrent region (distal, BP3-BP4) (includes GJA5);1q21.1_distal_BP3-BP4_t2:147473860-147841815;1q21.1_distal_BP3-BP4_t3:147842996-147917509 1q21.1_distal_BP3-BP4_L:143034587-143134587:flank left of the region;1q21.1_distal_BP3-BP4_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . diff --git a/bin/copy_number.py b/bin/copy_number.py index 2e1b95485..9412104b9 100755 --- a/bin/copy_number.py +++ b/bin/copy_number.py @@ -31,7 +31,7 @@ "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] KINDS = ("gene", "interval", "organelle") # GRCh38 names of the primary chromosomes (the ClinGen BED spells chrX 'chrx': -# scripts/ci/build-cn-intervals.py normalises it, and a row with any other +# og dev build cn-intervals normalises it, and a row with any other # spelling is refused here). CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} INPUTS = ("bedcov", "cnvpytor-level") diff --git a/docs/cn-loci.md b/docs/cn-loci.md index eb6f0a44e..a9b918f5f 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -89,7 +89,7 @@ A site is low when its estimate is more than 0.6 copies under the larger of the ### The interval rows -`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. +`scripts/ci/build-cn-intervals.sh` writes these rows with `og dev build cn-intervals`, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. - **The ClinGen recurrent regions**: every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0, pinned by `CLINGEN_RECURRENT_CNV_URL` and its sha256 in `versions.env`). The file's breakpoint and LCR rows are the segmental duplications that bound the regions. They are cut out of the targets, as are GIAB v3.6 segmental duplications, assembly gaps, satellites and the regions of every pinned Parascopy model (CHRNA7 in 15q13.3 stays with Parascopy). Targets are cut to pieces of at most 500 kb, and their depth is pooled. Controls are two 100 kb windows, one on each side of the cluster of ClinGen rows the region belongs to, clear of all of that and of the targets of the hand rows below (so no row's control sits on another row's target); a side with no clear window within 3 Mb (a chromosome end, as next to 22q11.2 and 15q11.2) is left out. The BED spells its chrX rows `chrx`; the script normalises them, and the engine refuses any contig name that is not a GRCh38 chromosome. The classification of a CNV found here follows the ACMG and ClinGen technical standards (Riggs 2020, PMID 31690835); a call from this step is a lead for a clinical test, not a classification. - **Two more ClinGen curations**: the 1p36 terminal region (ISCA-37434, haploinsufficiency score 3) and the Xq28 region that holds MECP2 (ISCA-46304, triplosensitivity score 3: the MECP2 duplication), from ClinGen's region curation list, read on 2026-11-04. @@ -100,7 +100,7 @@ A row on chrX or chrY is read against the sample's own count there: its controls The couple screen (step 39) reads the table with `couple_report.py screen --cn-intervals ID=_cn_intervals.tsv`: each carrier allele is present, absent or not assessable for each partner, and a male partner's AZF deletions are listed the same way (present means the deletion is there). With `--cnvpytor ID=_cnvs.vcf.gz` (step 18), a CNVpytor call that disagrees with the depth call is printed beside it. Step 39 and the Nextflow couple screen pass both when the partner has them. Both reports show the table in step 35's section: the counts by status and each row with a deletion or a duplication. -To change a row, change the script (its hand rows or a pinned download) and run `scripts/ci/build-cn-intervals.py`; `--check` fails when the catalog is not what the downloads give. +To change a row, change its rule or hand row in `rust/crates/og-dev/src/build/cn_intervals.rs`, or a pinned download in `scripts/ci/build-cn-intervals.sh`, rebuild og (`scripts/setup.sh --og`) and run `scripts/ci/build-cn-intervals.sh`; `--check` fails when the catalog is not what the downloads give. ## One owner per locus @@ -118,7 +118,7 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. 1. Check that no pinned caller owns the gene: Parascopy (`assets/parascopy_models_v1.7.txt`), pypgx, Paraphase. 2. Choose single-copy controls near the gene, outside any common deletion or duplication there and outside segmental duplications. -3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to `scripts/ci/build-cn-intervals.py` and rerun the script. +3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to the hand rows of `rust/crates/og-dev/src/build/cn_intervals.rs` and rerun `scripts/ci/build-cn-intervals.sh`. 4. Run it from a step: `og cn bed --locus NAME`, `samtools bedcov` with the row's passes, then `og cn call --locus NAME --bedcov ... --table ...`. 5. Add made-up bedcov tables for the row to `tests/test_copy_number.py`, at every count it can call and at each guard. diff --git a/docs/rust.md b/docs/rust.md index cb6924559..ae50a8042 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -37,7 +37,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/docs/testing.md b/docs/testing.md index 7cd0a5da2..7302f594c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -102,7 +102,7 @@ The base leg runs every base case whenever any case runs, because every shard st The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the six data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.4 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. -Six downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files, the HAPNEST panel and the table builders' downloads below, so an EBI outage costs none of them on a cold cache. It holds the TRExplorer catalog of `build-sv-rule-regions.sh` too (the v2.1 file its `TRX_SHA256` pins), so a miss tries the mirror first and the catalog's own GitHub release after it. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. The sixth, entry `catalogs`, holds the pinned downloads of `scripts/ci/build-callability.sh`, `scripts/ci/build-cn-intervals.py` and `scripts/ci/build-sv-rule-regions.sh`, whose cases `callability-1-catalog`, `cn-intervals-1-catalog` and `followups-ci-3-sv-rule-regions` rebuild `assets/callability_genes.tsv`, the interval rows of `bin/cn_loci.tsv` and `assets/sv_rule_regions.tsv` and compare them with the committed files. Each builder lists its downloads with `--list`; the key is a hash of those lists, so a new pin is a new key. GENCODE's GTF among them comes from EBI's FTP server, which refused every connection for minutes on 2026-11-06 and failed `callability-1-catalog` on main (run 42259432681). The runner fills the entry as it does the others, mirror first and every file checked against its sha256, and each case points its builder's cache at its folder there; a builder that still has to download tries for about five minutes, 30 s apart. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. +Six downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files, the HAPNEST panel and the table builders' downloads below, so an EBI outage costs none of them on a cold cache. It holds the TRExplorer catalog of `build-sv-rule-regions.sh` too (the v2.1 file its `TRX_SHA256` pins), so a miss tries the mirror first and the catalog's own GitHub release after it. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. The sixth, entry `catalogs`, holds the pinned downloads of `scripts/ci/build-callability.sh`, `scripts/ci/build-cn-intervals.sh` and `scripts/ci/build-sv-rule-regions.sh`, whose cases `callability-1-catalog`, `cn-intervals-1-catalog` and `followups-ci-3-sv-rule-regions` rebuild `assets/callability_genes.tsv`, the interval rows of `bin/cn_loci.tsv` and `assets/sv_rule_regions.tsv` and compare them with the committed files. Each builder lists its downloads with `--list`; the key is a hash of those lists, so a new pin is a new key. GENCODE's GTF among them comes from EBI's FTP server, which refused every connection for minutes on 2026-11-06 and failed `callability-1-catalog` on main (run 42259432681). The runner fills the entry as it does the others, mirror first and every file checked against its sha256, and each case points its builder's cache at its folder there; a builder that still has to download tries for about five minutes, 30 s apart, and gives up at once on a 403 or 404 (`fetch_pinned` in `scripts/ci/lib/fetch-pinned.sh`, which the three builders share). The builders download with `curl` and hand the folder to `og dev build`, which reads local files only. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. diff --git a/scripts/ci/build-callability.sh b/scripts/ci/build-callability.sh index e47058489..ab594545b 100755 --- a/scripts/ci/build-callability.sh +++ b/scripts/ci/build-callability.sh @@ -8,9 +8,8 @@ # scripts/ci/build-callability.sh --list the downloads, one "NAME URL SHA256" per line # # Downloads seven pinned files (about 85 MB, each checked against its -# sha256) into a temporary folder, or reads them from CALLABILITY_CACHE when -# that folder holds them with the right sha256, writes OUT and deletes the -# downloads: +# sha256) into CALLABILITY_CACHE, where a file with the right sha256 is read +# as it is, or else into a temporary folder that is deleted, and writes OUT: # # GIAB CMRG v1.00's GRCh38 gene coordinates (Wagner 2022, PMID 35132260): # the 273 challenging medically relevant genes @@ -53,7 +52,9 @@ # NEB, PIK3CA, PLG, VWF). Step 35's own report section reads all of them. # tests/test_paralog_loci.py holds that list and goes red when it changes. # -# Needs curl, python3 and sha256sum or shasum. The e2e case +# The rules are in og dev build callability +# (rust/crates/og-dev/src/build/callability.rs). Needs curl, og +# (scripts/setup.sh --og) and sha256sum or shasum. The e2e case # tests/e2e/callability-1-catalog.sh runs it and fails on any difference with # the committed file. set -euo pipefail @@ -81,208 +82,17 @@ if [ "${1:-}" = --list ]; then exit 0 fi -sha256() { - if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi -} - -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT - -# get NAME URL SHA256: WORK/NAME from CALLABILITY_CACHE when it matches, else -# downloaded; stop unless it matches. -get() { - local cached="${CALLABILITY_CACHE:-}/$1" got - if [ -n "${CALLABILITY_CACHE:-}" ] && [ -f "$cached" ] && [ "$(sha256 "$cached")" = "$3" ]; then - cp "$cached" "${WORK}/$1" - return - fi - echo "Downloading $2" >&2 - # Tries spread over about five minutes: EBI's FTP server refused every - # connection for minutes at a time (curl exit 7, 2026-11-06). - curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "${WORK}/$1" "$2" - got=$(sha256 "${WORK}/$1") - if [ "$got" != "$3" ]; then - echo "ERROR: $1 has sha256 ${got}, want $3" >&2 - exit 1 - fi - if [ -n "${CALLABILITY_CACHE:-}" ]; then - mkdir -p "$CALLABILITY_CACHE" && cp "${WORK}/$1" "$cached" - fi -} +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" +DIR=${CALLABILITY_CACHE:-} +if [ -z "$DIR" ]; then + DIR=$(mktemp -d) + trap 'rm -rf "$DIR"' EXIT +fi for f in "${FILES[@]}"; do IFS='|' read -r name url sum <<<"$f" - get "$name" "$url" "$sum" + fetch_pinned "$DIR" "$name" "$url" "$sum" || exit 1 done -python3 - "$WORK" "$REPO" "${OUT}.part" <<'PY' -import csv -import gzip -import re -import sys -from collections import defaultdict - -work, repo, out = sys.argv[1:4] -CHROMS = [f"chr{i}" for i in range(1, 23)] + ["chrX", "chrY"] -ORDER = {c: i for i, c in enumerate(CHROMS)} -PRIMARY = set(CHROMS) | {"chrM"} -ATT = re.compile(r'(\S+) "([^"]*)"') -SEGDUP_SHARE = 0.5 -REASONS = ("CMRG", "segdup", "false_duplication", "collapsed", "ALT-only") - - -def bed(name): - d = defaultdict(list) - with (gzip.open if name.endswith(".gz") else open)(f"{work}/{name}", "rt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not line.startswith(("#", "track", "browser")): - d[c[0]].append((int(c[1]), int(c[2]))) - for k in d: - d[k].sort() - return d - - -def covered(chrom, s, e, d): - """Bases of [s, e) inside the intervals of d (merged as it goes).""" - n, cur = 0, s - for a, b in d.get(chrom, []): - if b <= cur: - continue - if a >= e: - break - a, b = max(a, cur), min(b, e) - if b > a: - n += b - a - cur = b - return n - - -# ALT scaffolds: GenBank accession -> UCSC name (GENCODE names them by accession) -alt = {} -with open(f"{work}/report.txt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if not line.startswith("#") and len(c) >= 10 and c[1] == "alt-scaffold": - alt[c[4]] = c[9] - -genes = {} # hgnc id -> [symbol, chrom, start0, end] (any gene type) -coding = set() # hgnc ids of the protein-coding ones -on_primary = set() # every gene symbol on chr1-22, X, Y or M -alt_genes = {} # hgnc id -> [symbol, UCSC ALT name] (first ALT copy) -with gzip.open(f"{work}/gencode.gtf.gz", "rt") as f: - for line in f: - if line.startswith("#"): - continue - c = line.split("\t", 9) - if c[2] != "gene": - continue - a = dict(ATT.findall(c[8])) - if c[0] in PRIMARY: - on_primary.add(a.get("gene_name")) - h = a.get("hgnc_id") - if not h or a["gene_id"].endswith("_PAR_Y"): - continue - if c[0] in ORDER and h not in genes: - genes[h] = [a["gene_name"], c[0], int(c[3]) - 1, int(c[4])] - if a.get("gene_type") == "protein_coding": - coding.add(h) - elif c[0] in alt and h not in alt_genes and a.get("gene_type") == "protein_coding": - alt_genes[h] = [a["gene_name"], alt[c[0]]] - -reasons = defaultdict(set) -by_symbol = {g[0]: h for h, g in genes.items()} -with open(f"{work}/cmrg.bed") as f: - for line in f: - chrom, s, e, sym = line.rstrip("\n").split("\t")[:4] - s, e = int(s), int(e) - h = by_symbol.get(sym) - if h is None: - # renamed since CMRG v1.00: the protein-coding gene that covers most of its span - best = max(((min(e, genes[k][3]) - max(s, genes[k][2]), k) for k in coding if genes[k][1] == chrom), - default=(0, None)) - if best[0] <= 0: - sys.exit(f"ERROR: CMRG gene {sym} ({chrom}:{s}-{e}) matches no GENCODE 50 gene") - h = best[1] - reasons[h].add("CMRG") - -segdups, collapsed = bed("segdups.bed.gz"), bed("collapsed.bed.gz") -falsedup = bed("falsedup_correct.bed.gz") -for k, v in bed("falsedup_incorrect.bed.gz").items(): - falsedup[k] = sorted(falsedup[k] + v) -for h in coding: - sym, chrom, s, e = genes[h] - if covered(chrom, s, e, segdups) >= SEGDUP_SHARE * (e - s): - reasons[h].add("segdup") - if covered(chrom, s, e, falsedup): - reasons[h].add("false_duplication") - if covered(chrom, s, e, collapsed): - reasons[h].add("collapsed") -for h, (sym, name) in alt_genes.items(): - if h not in genes and sym not in on_primary: - reasons[h].add("ALT-only") - - -def table(path): - with open(path) as f: - return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", - quoting=csv.QUOTE_NONE)) - - -parascopy = {g for r in table(f"{repo}/assets/paralog_loci.tsv") for g in r["genes"].split(",")} -# A gene step 32 skips on the default no-ALT reference (callable_without_alt no) has no -# caller there, and step 32 refuses a BAM with ALT contigs, so it is not pypgx's. -pypgx = {r["gene"] for r in table(f"{repo}/assets/pgx_genes.tsv") - if r["mode"] == "bam" and r["callable_without_alt"] != "no"} -# A cn_loci gene row reports the genes it names (its locus, as RHD, or LPA for LPA_KIV2, -# or a target, as HBA1 and HBA2), not every gene whose span holds one of its targets. -cn_names = {} -for r in table(f"{repo}/bin/cn_loci.tsv"): - if r["kind"] == "gene": - names = {r["locus"], r["locus"].split("_")[0]} - for part in r["targets"].split(";"): - names |= {part.split(":")[0], part.split(":")[0].split("_")[0]} - for n in names: - cn_names.setdefault(n, r["locus"]) - - -def owner(sym): - if sym in parascopy: - return "parascopy" - if sym in pypgx: - return "pypgx" - if sym in cn_names: - return f"cn_loci:{cn_names[sym]}" - return "." - - -rows = [] -for h, why in reasons.items(): - if h in genes: - sym, chrom, s, e = genes[h] - rows.append((ORDER[chrom], s, sym, h, chrom, s, e, why, owner(sym))) - else: - sym, name = alt_genes[h] - rows.append((len(ORDER), 0, sym, h, name, ".", ".", why, owner(sym))) -rows.sort() -syms = [r[2] for r in rows] -dup = sorted({x for x in syms if syms.count(x) > 1}) -if dup: - sys.exit(f"ERROR: gene symbols listed twice: {', '.join(dup)}") -counts = {k: sum(1 for r in rows if k in r[7]) for k in REASONS} -owned = sum(1 for r in rows if r[8] != ".") -with open(out, "w") as f: - f.write("# assets/callability_genes.tsv: the genes where short reads are often not placed with confidence, so a\n" - "# 'no finding' there needs a reading of its own. Step 16b measures each gene on chr1-22, X and Y; the\n" - "# report lists the ones it could not read. Written by scripts/ci/build-callability.sh, which says how the\n" - "# rows and the owners are chosen; rebuild it there, do not edit it by hand. start is 0-based and end\n" - "# 1-based, as in a BED file; an ALT-only gene has its ALT scaffold as chrom and '.' as start and end.\n" - f"# Genes: {len(rows)} (" + ", ".join(f"{k} {counts[k]}" for k in REASONS) + f"); with an owner: {owned}.\n") - f.write("gene\thgnc_id\tchrom\tstart\tend\treasons\towner\n") - for r in rows: - f.write("\t".join(map(str, (r[2], r[3], r[4], r[5], r[6], ",".join(k for k in REASONS if k in r[7]), r[8]))) - + "\n") -print(f"Genes: {len(rows)} (" + ", ".join(f"{k} {counts[k]}" for k in REASONS) + f"); with an owner: {owned}", - file=sys.stderr) -PY -mv "${OUT}.part" "$OUT" +"${REPO}/bin/og" dev build callability --work "$DIR" --repo "$REPO" --out "$OUT" echo "Wrote ${OUT}" >&2 diff --git a/scripts/ci/build-cn-intervals.py b/scripts/ci/build-cn-intervals.py deleted file mode 100755 index 5982debdf..000000000 --- a/scripts/ci/build-cn-intervals.py +++ /dev/null @@ -1,423 +0,0 @@ -#!/usr/bin/env python3 -"""build-cn-intervals.py: write the interval rows of bin/cn_loci.tsv. - -The interval rows are the copy-number checks step 35 runs with one -`samtools bedcov` pass (og cn intervals): - - - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, - version 2.1, GRCh38 (CLINGEN_RECURRENT_CNV_URL and _SHA256 in - versions.env, CC0): 22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12, 17p11.2, - 7q11.23, 3q29, Xp22.31 and the rest. The file's breakpoint and LCR rows - (itemRgb 0,0,0) are the segmental duplications that bound the regions: - they are cut out of the targets and kept out of the controls. Its chrX - rows are spelled 'chrx' and are normalised to chrX; any other name that is - not a GRCh38 chromosome stops the build; - - two ClinGen curations the BED lacks (HAND_ROWS): the 1p36 terminal - region (ISCA-37434, haploinsufficiency 3) and the Xq28 region that holds - MECP2 (ISCA-46304, triplosensitivity 3), from - ClinGen_region_curation_list_GRCh38.tsv as read on 2026-11-04; - - three carrier deletion alleles at their ClinVar GRCh38 spans (HAND_ROWS): - del(GJB6-D13S1830) (VCV 5546), CFTRdele2,3 (VCV 66105) and a 621 bp HBB - deletion that removes exon 3 (VCV 3769040); and two beta-globin - deletions at their HbVar GRCh38 spans, checked on 2026-11-04 by placing - 50 bp on each side of the NG_000007.3 breakpoints on NC_000011.10: the - 619 bp HBB deletion common in India and Pakistan (HbVar 979, - NG_000007.3:g.71609_72227del619, Orkin 1979) and the 13.4 kb Sicilian - (delta-beta)0 deletion from HBD to 3' of HBB (HbVar 1035, - NG_000007.3:g.64336_77738del13403, Henthorn 1990). The alpha-globin - --SEA deletion is read by the HBA row (step 35's HBA table), not here; - - the Y chromosome AZF regions (HAND_ROWS), read only on a sample with a Y: - AZFa as USP9Y and DDX3Y, AZFb as KDM5D and EIF1AY (single-copy genes each - deletion removes), and AZFc as the four DAZ copies added up (a complete - AZFc deletion leaves none, the partial gr/gr and b2/b3 deletions two; - which copies are lost, and the breakpoints, are not claimed). Gene spans - from Ensembl's REST lookup, checked on 2026-11-04. Controls: ZFY and - NLGN4Y, single-copy genes outside every AZF region. - -Targets are cut to pieces of at most 500 kb and lose every stretch inside a -breakpoint or LCR row of the BED, a GIAB v3.6 segmental duplication, an -assembly gap (GIAB's gaps with 15 kb on each side), a satellite or a region of -a model the pinned Parascopy data holds (assets/parascopy_models_v1.7.txt: -Parascopy owns those, CHRNA7 in 15q13.3 among them); the AZF targets keep -their segmental duplications, since the DAZ copies are one. Controls are two -100 kb windows, one on each side of the cluster of ClinGen rows the region -belongs to, the nearest that stay clear of all of that, within 3 Mb; a side -with none (a chromosome end) is left out. Every row: depth pooled over the -targets against depth pooled over the controls, scale 2, a whole number -within 0.3 (round:0.3), at least 8x of control depth (4x on chrY), the two -flanks within 1.5x of each other; cross_check cnvpytor:cnvs (step 18). - -Usage: - scripts/ci/build-cn-intervals.py [--catalog bin/cn_loci.tsv] [--cache DIR] [--check] - scripts/ci/build-cn-intervals.py --list the downloads, one "NAME URL SHA256" per line - -Rewrites the catalog's rows from the marker line on (MARKER), or with --check -exits 1 when the catalog is not what the downloads give. The downloads (about -150 KB) are checked against their sha256 and kept in --cache when given. The -e2e case tests/e2e/cn-intervals-1-catalog.sh runs --check. -""" -import argparse -import gzip -import hashlib -import os -import re -import sys -import tempfile -import time -import urllib.request - -REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -MARKER = "# --- interval rows: written by scripts/ci/build-cn-intervals.py; do not edit by hand ---" -GIAB = "https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/genome-stratifications/v3.6/GRCh38@all" -MASKS = [ - ("segdups.bed.gz", f"{GIAB}/SegmentalDuplications/GRCh38_segdups.bed.gz", - "09ac029a7d0d681a9f0ed6581a6bc10fc708a834bc3f91c46a9bef504f6055c5"), - ("gaps.bed.gz", f"{GIAB}/OtherDifficult/GRCh38_gaps_slop15kb.bed.gz", - "b5fdbfc7e5b9cf4fa356f23841fd7b8a74fa239fd8cbc5b5c3463c9e7bdc0902"), - ("satellites.bed.gz", f"{GIAB}/LowComplexity/GRCh38_satellites_slop5.bed.gz", - "10207aa2dac0e1189ea553a374abbdcb4de25a81378d88dc4ba03aea8718b5a2"), -] -REGION_RGB, MASK_RGB = "255,111,0", "0,0,0" -CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y"]} -# GRCh38 chromosome lengths (the no-ALT analysis set's .fai). -LENGTHS = {"chr1": 248956422, "chr2": 242193529, "chr3": 198295559, "chr4": 190214555, "chr5": 181538259, - "chr6": 170805979, "chr7": 159345973, "chr8": 145138636, "chr9": 138394717, "chr10": 133797422, - "chr11": 135086622, "chr12": 133275309, "chr13": 114364328, "chr14": 107043718, "chr15": 101991189, - "chr16": 90338345, "chr17": 83257441, "chr18": 80373285, "chr19": 58617616, "chr20": 64444167, - "chr21": 46709983, "chr22": 50818468, "chrX": 156040895, "chrY": 57227415} -PIECE, FLANK, GAP, SEARCH, STEP, MIN_PIECE = 500_000, 100_000, 50_000, 3_000_000, 10_000, 500 - -# Rows the BED lacks: locus, chrom, [(name, 0-based start, end)] targets, [controls] or None (flanks), -# target_depth, whether segmental duplications are cut out of the targets, pmids. The label of the -# first target is the row's name in step 35's table. -Y_CONTROLS = [("ZFY", 2935200, 2982579, "ZFY, single-copy control on Yp"), - ("NLGN4Y", 14522572, 14845654, "NLGN4Y, single-copy control on Yq")] -HAND_ROWS = [ - ("1p36_terminal", "chr1", [("1p36 terminal region (includes GABRD), ClinGen ISCA-37434", 898702, 6229913)], - None, "pooled", True, "."), - ("Xq28_MECP2", "chrX", [("Xq28 region (includes MECP2), ClinGen ISCA-46304: the MECP2 duplication", - 154008528, 154110279)], None, "pooled", True, "."), - ("GJB6_D13S1830", "chr13", [("del(GJB6-D13S1830), ClinVar 5546", 20223037, 20531806)], None, "pooled", True, - "11807148"), - ("CFTR_dele2_3", "chr7", [("CFTRdele2,3 (21 kb), ClinVar 66105", 117498312, 117519392)], None, "pooled", True, - "."), - ("HBB_exon3_del", "chr11", [("HBB deletion of exon 3 (621 bp), ClinVar 3769040", 5225254, 5225875)], None, - "pooled", True, "."), - ("HBB_619bp_del", "chr11", [("HBB 619 bp deletion (South Asian), HbVar 979", 5225388, 5226007)], None, - "pooled", True, "287080"), - ("HBD_HBB_Sicilian_del", "chr11", [("Sicilian (delta-beta)0 deletion (13.4 kb, HBD to HBB 3'), HbVar 1035", - 5219877, 5233280)], None, "pooled", True, "2307466"), - ("AZFa", "chrY", [("AZFa (USP9Y and DDX3Y)", 12537649, 12860852), ("DDX3Y", 12904107, 12920478)], - Y_CONTROLS, "pooled", False, "37674303"), - ("AZFb", "chrY", [("AZFb (KDM5D and EIF1AY)", 19702297, 19745148), ("EIF1AY", 20575693, 20593154)], - Y_CONTROLS, "pooled", False, "37674303"), - ("AZFc_DAZ", "chrY", [("AZFc (DAZ1 to DAZ4 added up; gr/gr and b2/b3 leave two)", 23129354, 23199010), - ("DAZ2", 23219446, 23291356), ("DAZ3", 24763068, 24813492), ("DAZ4", 24833842, 24907040)], - Y_CONTROLS, "sum", False, "37674303"), -] -COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", - "scale", "offset", "bedcov", "mapq0_min", "mapq0_max", "min_control_depth", "max_control_ratio", - "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] - - -class BuildError(ValueError): - pass - - -def sha256(path): - h = hashlib.sha256() - with open(path, "rb") as f: - for b in iter(lambda: f.read(1 << 20), b""): - h.update(b) - return h.hexdigest() - - -def fetch(name, url, want, folder): - path = os.path.join(folder, name) - if not (os.path.exists(path) and sha256(path) == want): - for i in range(3): - try: - with urllib.request.urlopen(url, timeout=120) as r, open(path + ".part", "wb") as f: - f.write(r.read()) - os.replace(path + ".part", path) - break - except OSError as e: - # A missing or forbidden file stays so: no point waiting. - if i == 2 or getattr(e, "code", None) in (403, 404): - raise BuildError(f"cannot download {url}: {e}") - time.sleep(30) - got = sha256(path) - if got != want: - raise BuildError(f"{url}: sha256 {got}, pinned {want}") - return path - - -def versions_env(key): - with open(os.path.join(REPO, "versions.env")) as f: - for line in f: - m = re.match(rf'^{key}="([^"]*)"', line) - if m: - return m.group(1) - raise BuildError(f"no {key} in versions.env") - - -def norm_chrom(c): - """chrx -> chrX (the ClinGen BED's spelling); anything not a GRCh38 chromosome stops the build.""" - c2 = re.sub(r"^chr([xy])$", lambda m: "chr" + m.group(1).upper(), c) - if c2 not in CHROMS: - raise BuildError(f"chromosome {c!r} is not a GRCh38 chromosome name") - return c2 - - -def read_clingen(lines): - """(regions [(chrom, start, end, name)], masks [(chrom, start, end)]) of the BED's lines.""" - regions, masks = [], [] - for line in lines: - if not line.strip() or line.startswith(("#", "track", "browser")): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 9: - raise BuildError(f"ClinGen BED: a row with {len(c)} columns: {line.strip()!r}") - chrom, s, e, name, rgb = norm_chrom(c[0]), int(c[1]), int(c[2]), c[3].strip(), c[8].strip() - if e <= s: - raise BuildError(f"ClinGen BED: {name}: empty span") - if rgb == REGION_RGB: - regions.append((chrom, s, e, name)) - elif rgb == MASK_RGB: - masks.append((chrom, s, e)) - else: - raise BuildError(f"ClinGen BED: {name}: itemRgb {rgb!r} is neither a region nor a breakpoint row") - if not regions: - raise BuildError("ClinGen BED: no region row") - return regions, masks - - -def read_bed(path): - out = [] - with gzip.open(path, "rt") as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.split("\t") - out.append((c[0], int(c[1]), int(c[2]))) - return out - - -def read_parascopy(path): - out = [] - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - _, reg, _ = line.split() - c, span = reg.split(":") - s, e = span.split("-") - out.append((c, int(s) - 1, int(e))) - return out - - -def merge(spans): - out = [] - for s, e in sorted(spans): - if out and s <= out[-1][1]: - out[-1][1] = max(out[-1][1], e) - else: - out.append([s, e]) - return [tuple(x) for x in out] - - -def subtract(s, e, cuts): - """[(s, e)] of s-e outside the merged spans cuts.""" - out, pos = [], s - for cs, ce in cuts: - if ce <= pos or cs >= e: - continue - if cs > pos: - out.append((pos, cs)) - pos = max(pos, ce) - if pos < e: - out.append((pos, e)) - return out - - -def pieces(spans): - out = [] - for s, e in spans: - if e - s < MIN_PIECE: - continue - n = -(-(e - s) // PIECE) - step = -(-(e - s) // n) - for i in range(n): - out.append((s + i * step, min(e, s + (i + 1) * step))) - return out - - -def clear(s, e, cuts): - return all(ce <= s or cs >= e for cs, ce in cuts) - - -def flank(chrom, edge, side, cuts): - """The nearest FLANK window GAP beyond edge on side (-1 left, +1 right), clear of cuts, within SEARCH.""" - for k in range(0, SEARCH // STEP + 1): - if side < 0: - e = edge - GAP - k * STEP - s = e - FLANK - else: - s = edge + GAP + k * STEP - e = s + FLANK - if s < 0 or e > LENGTHS[chrom]: - return None - if clear(s, e, cuts): - return (s, e) - return None - - -def locus_id(name): - band = name.split()[0] - groups = [g for g in re.findall(r"\(([^()]*)\)", name) if not g.strip().startswith("includes")] - words = re.sub(r"[^A-Za-z0-9.-]+", "_", " ".join(groups)).strip("_") - return f"{band}_{words}" if words else band - - -def label(text): - return re.sub(r"[;\t\n]", ",", text).strip() - - -def build(clingen_lines, segdups, gaps, sats, parascopy): - regions, masks = read_clingen(clingen_lines) - by_chrom = {} - for name, items in (("mask", masks), ("segdup", segdups), ("gap", gaps), ("sat", sats), ("para", parascopy)): - for c, s, e in items: - by_chrom.setdefault(c, {}).setdefault(name, []).append((s, e)) - # Clusters: ClinGen rows (regions and breakpoints) that touch or overlap, per chromosome. - clusters = {} - for c in CHROMS: - spans = [(s, e + 1) for cc, s, e, _ in regions if cc == c] + [(s, e + 1) for cc, s, e in masks if cc == c] - clusters[c] = merge(spans) - - def cuts(c, with_segdups=True): - d = by_chrom.get(c, {}) - keys = ("mask", "segdup", "gap", "sat", "para") if with_segdups else ("mask", "gap", "para") - return merge([x for k in keys for x in d.get(k, [])]) - - def flank_cuts(c): - # A control never sits on a target: the ClinGen regions and the HAND_ROWS targets are cut out too. - d = by_chrom.get(c, {}) - return merge([x for k in ("mask", "segdup", "gap", "sat", "para") for x in d.get(k, [])] - + [(s, e) for cc, s, e, _ in regions if cc == c] - + [(s, e) for _, cc, tl, *_ in HAND_ROWS if cc == c for _, s, e in tl]) - - rows, seen = [], {} - - def add(locus, chrom, targets, controls, target_depth, pmids): - if not targets: - raise BuildError(f"{locus}: no target stretch is left outside the masks") - if locus in seen: - seen[locus] += 1 - locus = f"{locus}_{seen[locus]}" - else: - seen[locus] = 1 - y = chrom == "chrY" - tg = ";".join(f"{locus}_t{i + 1}:{s}-{e}" + (f":{lab}" if lab else "") for i, (s, e, lab) in enumerate(targets)) - ct = ";".join(f"{locus}_{n}:{s}-{e}:{lab}" for n, s, e, lab in controls) - rows.append({"locus": locus, "kind": "interval", "chrom": chrom, "targets": tg, "controls": ct, - "details": ".", "target_depth": target_depth, "control_depth": "pooled" if y else "flanks", - "scale": "2", "offset": "0", "bedcov": ".", "mapq0_min": ".", "mapq0_max": ".", - "min_control_depth": "4" if y else "8", "max_control_ratio": "." if y else "1.5", - "call": "round:0.3", "interval": ".", "readings": ".", "input": "bedcov", "owner": "cn-depth", - "cross_check": "cnvpytor:cnvs", "pmids": pmids}) - - def flanks(chrom, s, e): - cs, ce = next(((a, b) for a, b in clusters[chrom] if a <= s and e <= b), (s, e)) - fc = flank_cuts(chrom) - out = [] - for side, edge, n in ((-1, cs, "L"), (1, ce, "R")): - w = flank(chrom, edge, side, fc) - if w: - out.append((n, w[0], w[1], "flank " + ("left" if side < 0 else "right") + " of the region")) - if not out: - raise BuildError(f"{chrom}:{s}-{e}: no flank clear of the masks within {SEARCH // 1_000_000} Mb") - return out - - for chrom, s, e, name in sorted(regions, key=lambda r: (sorted(CHROMS, key=_order).index(r[0]), r[1], r[2])): - segs = pieces(subtract(s, e, cuts(chrom))) - targets = [(a, b, label(name) if i == 0 else "") for i, (a, b) in enumerate(segs)] - add(locus_id(name), chrom, targets, flanks(chrom, s, e), "pooled", ".") - for locus, chrom, tlist, controls, td, with_sd, pmids in HAND_ROWS: - c = cuts(chrom, with_sd) - targets = [] - for name, s, e in tlist: - segs = pieces(subtract(s, e, c)) if td == "pooled" else [(s, e)] - if td == "sum" and not clear(s, e, merge(by_chrom.get(chrom, {}).get("para", []))): - raise BuildError(f"{locus}: {name} overlaps a Parascopy model") - targets += [(a, b, label(name) if i == 0 else "") for i, (a, b) in enumerate(segs)] - if controls is None: - ctl = flanks(chrom, tlist[0][1], tlist[-1][2]) - else: - ctl = [(f"c{i + 1}", s, e, lab) for i, (_, s, e, lab) in enumerate(controls)] - add(locus, chrom, targets, ctl, td, pmids) - return rows - - -def _order(c): - k = c[3:] - return (0, int(k)) if k.isdigit() else (1, k) - - -def render(rows): - return [MARKER + "\n"] + ["\t".join(r[c] for c in COLUMNS) + "\n" for r in rows] - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--catalog", default=os.path.join(REPO, "bin", "cn_loci.tsv")) - ap.add_argument("--cache") - ap.add_argument("--check", action="store_true") - ap.add_argument("--list", action="store_true") - a = ap.parse_args(argv) - if a.list and a.check: - ap.error("--list and --check do not go together") - if a.list: - try: - files = [("clingen_recurrent_cnv.bed", versions_env("CLINGEN_RECURRENT_CNV_URL"), - versions_env("CLINGEN_RECURRENT_CNV_SHA256"))] + MASKS - except BuildError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - for n, u, h in files: - print(n, u, h) - return 0 - try: - with tempfile.TemporaryDirectory() as tmp: - folder = a.cache or tmp - os.makedirs(folder, exist_ok=True) - bed = fetch("clingen_recurrent_cnv.bed", versions_env("CLINGEN_RECURRENT_CNV_URL"), - versions_env("CLINGEN_RECURRENT_CNV_SHA256"), folder) - sd, gp, st = (read_bed(fetch(n, u, h, folder)) for n, u, h in MASKS) - with open(bed, errors="replace") as f: - rows = build(f.readlines(), sd, gp, st, - read_parascopy(os.path.join(REPO, "assets", "parascopy_models_v1.7.txt"))) - except (OSError, BuildError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - with open(a.catalog) as f: - lines = f.readlines() - keep = lines[:lines.index(MARKER + "\n")] if MARKER + "\n" in lines else lines - new = keep + render(rows) - if a.check: - if new != lines: - old = set(lines) - set(new) - print(f"{a.catalog} is not what the pinned downloads give ({len(old)} line(s) differ); " - "rerun scripts/ci/build-cn-intervals.py", file=sys.stderr) - return 1 - print(f"{a.catalog}: the {len(rows)} interval rows are what the pinned downloads give") - return 0 - with open(a.catalog + ".tmp", "w") as f: - f.writelines(new) - os.replace(a.catalog + ".tmp", a.catalog) - print(f"{a.catalog}: {len(rows)} interval rows written") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/ci/build-cn-intervals.sh b/scripts/ci/build-cn-intervals.sh new file mode 100755 index 000000000..ec40d052c --- /dev/null +++ b/scripts/ci/build-cn-intervals.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# build-cn-intervals.sh: rewrite the interval rows of bin/cn_loci.tsv, the +# copy-number checks step 35 runs with one samtools bedcov pass +# (og cn intervals). +# +# Usage: scripts/ci/build-cn-intervals.sh [--catalog FILE] [--cache DIR] [--check] +# scripts/ci/build-cn-intervals.sh --list the downloads, one "NAME URL SHA256" per line +# +# Rewrites FILE (default bin/cn_loci.tsv) from its marker line on, or with +# --check exits 1 when it is not what the downloads give. The four downloads +# (about 150 KB, each checked against its sha256) are kept in --cache when +# given, else in a temporary folder that is deleted: +# +# ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 +# (CLINGEN_RECURRENT_CNV_URL and _SHA256 in versions.env, CC0) +# GIAB genome stratifications v3.6, GRCh38: segmental duplications, +# assembly gaps with 15 kb on each side, satellites +# +# with assets/parascopy_models_v1.7.txt. Every region row of the BED is a +# row, then the rows the BED lacks: 1p36 and Xq28 (MECP2) from ClinGen's +# curations, five carrier deletions (GJB6-D13S1830, CFTRdele2,3, three +# beta-globin deletions) and the Y chromosome's AZF regions. Targets are cut +# to pieces of at most 500 kb and lose the BED's breakpoint and LCR rows, +# segmental duplications (not on the AZF rows), gaps, satellites and +# Parascopy models; controls are two 100 kb flanks clear of all of that. The +# rules and the hand rows are in og dev build cn-intervals +# (rust/crates/og-dev/src/build/cn_intervals.rs); docs/cn-loci.md says what +# each row reads. +# +# Needs curl, og (scripts/setup.sh --og) and sha256sum or shasum. The e2e +# case tests/e2e/cn-intervals-1-catalog.sh runs --check. +set -euo pipefail + +REPO=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" + +CATALOG=${REPO}/bin/cn_loci.tsv +CACHE="" +CHECK="" +LIST="" +while [ $# -gt 0 ]; do + case $1 in + --catalog) CATALOG=${2:?--catalog needs a file}; shift 2 ;; + --cache) CACHE=${2:?--cache needs a folder}; shift 2 ;; + --check) CHECK=1; shift ;; + --list) LIST=1; shift ;; + *) echo "Usage: $0 [--catalog FILE] [--cache DIR] [--check] | --list" >&2; exit 2 ;; + esac +done +if [ -n "$LIST" ] && [ -n "$CHECK" ]; then + echo "ERROR: --list and --check do not go together" >&2 + exit 2 +fi + +# versions_env KEY: the value of KEY in versions.env. +versions_env() { + local v + v=$(sed -n "s/^$1=\"\([^\"]*\)\".*/\1/p" "${REPO}/versions.env" | head -n 1) + [ -n "$v" ] || { echo "ERROR: no $1 in versions.env" >&2; return 1; } + echo "$v" +} + +GIAB=https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/genome-stratifications/v3.6/GRCh38@all +clingen_url=$(versions_env CLINGEN_RECURRENT_CNV_URL) || exit 2 +clingen_sha=$(versions_env CLINGEN_RECURRENT_CNV_SHA256) || exit 2 +FILES=( + "clingen_recurrent_cnv.bed|${clingen_url}|${clingen_sha}" + "segdups.bed.gz|${GIAB}/SegmentalDuplications/GRCh38_segdups.bed.gz|09ac029a7d0d681a9f0ed6581a6bc10fc708a834bc3f91c46a9bef504f6055c5" + "gaps.bed.gz|${GIAB}/OtherDifficult/GRCh38_gaps_slop15kb.bed.gz|b5fdbfc7e5b9cf4fa356f23841fd7b8a74fa239fd8cbc5b5c3463c9e7bdc0902" + "satellites.bed.gz|${GIAB}/LowComplexity/GRCh38_satellites_slop5.bed.gz|10207aa2dac0e1189ea553a374abbdcb4de25a81378d88dc4ba03aea8718b5a2" +) + +if [ -n "$LIST" ]; then + for f in "${FILES[@]}"; do + IFS='|' read -r name url sum <<< "$f" + echo "${name} ${url} ${sum}" + done + exit 0 +fi + +if [ -z "$CACHE" ]; then + CACHE=$(mktemp -d) + trap 'rm -rf "$CACHE"' EXIT +fi +for f in "${FILES[@]}"; do + IFS='|' read -r name url sum <<< "$f" + fetch_pinned "$CACHE" "$name" "$url" "$sum" || exit 2 +done + +# Not exec: the EXIT trap deletes a temporary download folder. +rc=0 +"${REPO}/bin/og" dev build cn-intervals \ + --clingen "${CACHE}/clingen_recurrent_cnv.bed" --segdups "${CACHE}/segdups.bed.gz" \ + --gaps "${CACHE}/gaps.bed.gz" --satellites "${CACHE}/satellites.bed.gz" \ + --parascopy "${REPO}/assets/parascopy_models_v1.7.txt" --catalog "$CATALOG" ${CHECK:+--check} || rc=$? +exit "$rc" diff --git a/scripts/ci/build-sv-rule-regions.sh b/scripts/ci/build-sv-rule-regions.sh index cb21651da..031b80fab 100755 --- a/scripts/ci/build-sv-rule-regions.sh +++ b/scripts/ci/build-sv-rule-regions.sh @@ -10,9 +10,8 @@ # "NAME URL SHA256" per line # # Downloads four pinned files (about 145 MB, each checked against its sha256) -# into a temporary folder, or reads them from SV_RULE_CACHE when that folder -# holds them with the right sha256 (and keeps what it downloads there), writes -# OUT and deletes the downloads: +# into SV_RULE_CACHE, where a file with the right sha256 is read as it is, or +# else into a temporary folder that is deleted, and writes OUT: # # TRExplorer v2.1's GRCh38 tandem repeat catalog (broadinstitute/trexplorer-catalog) # GENCODE 50's basic annotation (the release scripts/lib/common.sh pins) @@ -35,8 +34,12 @@ # GenCC disease curation; name ALIAS|PSEUDOGENE|its HGNC id| # the HGNC id of the gene that holds ALIAS as its symbol # -# Needs curl, python3 and sha256sum or shasum. It reads about 1 GB of text, -# so run it on a box with room, not in a unit test. +# A gene's span is that of the first GENCODE record of its HGNC id, and the +# _PAR_Y copies of the pseudoautosomal genes are skipped, as +# build-callability.sh does. The rules are in og dev build sv-rule-regions +# (rust/crates/og-dev/src/build/sv_rules.rs). Needs curl, og +# (scripts/setup.sh --og) and sha256sum or shasum. It reads about 3 GB of +# text in a few seconds and keeps little of it. set -euo pipefail REPO=$(cd "$(dirname "$0")/../.." && pwd) @@ -66,161 +69,19 @@ if [ "${1:-}" = --list ]; then exit 0 fi -sha256() { - if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi -} - +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT +DIR=${SV_RULE_CACHE:-$WORK} [ -z "$CHECK" ] || OUT="${WORK}/rebuilt.tsv" -# get NAME URL SHA256: WORK/NAME from SV_RULE_CACHE when it matches, else -# downloaded; stop unless it matches. -get() { - local cached="${SV_RULE_CACHE:-}/$1" got - if [ -n "${SV_RULE_CACHE:-}" ] && [ -f "$cached" ] && [ "$(sha256 "$cached")" = "$3" ]; then - cp "$cached" "${WORK}/$1" - return - fi - echo "Downloading $2" >&2 - # Tries spread over about five minutes: EBI's FTP server refused every - # connection for minutes at a time (curl exit 7, 2026-11-06). - curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "${WORK}/$1" "$2" - got=$(sha256 "${WORK}/$1") - if [ "$got" != "$3" ]; then - echo "ERROR: $1 has sha256 ${got}, want $3" >&2 - exit 1 - fi - if [ -n "${SV_RULE_CACHE:-}" ]; then - mkdir -p "$SV_RULE_CACHE" && cp "${WORK}/$1" "$cached" - fi -} -get trx.bed.gz "$TRX_URL" "$TRX_SHA256" -get gencode.gtf.gz "$GENCODE_URL" "$GENCODE_SHA256" -get hgnc.txt "$HGNC_URL" "$HGNC_SHA256" -get regions.txt "$REGIONS_URL" "$REGIONS_SHA256" - -python3 - "$WORK" "${OUT}.part" <<'PY' -import bisect -import csv -import gzip -import re -import sys -from collections import defaultdict - -work, out = sys.argv[1:3] -CHROMS = [f"chr{i}" for i in range(1, 23)] + ["chrX", "chrY"] -ORDER = {c: i for i, c in enumerate(CHROMS)} -MIN_TR = 50 -LOCUS_GAP = 1_000_000 -ATT = re.compile(r'(\S+) "([^"]*)"') -SRC_TR = "TRExplorer v2.1; GENCODE 50 MANE Select CDS" -SRC_IG = "GENCODE 50 IG and TR genes" -SRC_MHC = "GRCh38 assembly regions (GCA_000001405.15)" -SRC_ALIAS = "HGNC 2026-10-06; GENCODE 50" - -rows = [] -gene_span = {} # HGNC id -> (chrom, start0, end) -ig = defaultdict(list) # locus -> [(chrom, start0, end)] -cds = defaultdict(list) # chrom -> [(start0, end, gene)] -with gzip.open(f"{work}/gencode.gtf.gz", "rt") as f: - for line in f: - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if c[0] not in ORDER: - continue - start, end = int(c[3]) - 1, int(c[4]) - if c[2] == "gene": - a = dict(ATT.findall(c[8])) - if a.get("hgnc_id"): - gene_span[a["hgnc_id"]] = (c[0], start, end) - name, gtype = a.get("gene_name", ""), a.get("gene_type", "") - if re.match(r"(IG|TR)_", gtype) and re.match(r"(IG[HKL]|TR[ABDG])", name): - locus = "TRA" if name.startswith("TRD") else name[:3] - ig[locus].append((c[0], start, end)) - elif c[2] == "CDS" and 'tag "MANE_Select"' in c[8]: - cds[c[0]].append((start, end, dict(ATT.findall(c[8]))["gene_name"])) - -# tandem_repeat: catalog repeats of MIN_TR bp or more on a MANE Select coding exon -index = {} -for chrom, iv in cds.items(): - iv.sort() - starts = [s for s, _, _ in iv] - maxend, m = [], 0 - for _, e, _ in iv: - m = max(m, e) - maxend.append(m) - index[chrom] = (iv, starts, maxend) -with gzip.open(f"{work}/trx.bed.gz", "rt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if c[0] not in index: - continue - s, e = int(c[1]), int(c[2]) - if e - s < MIN_TR: - continue - iv, starts, maxend = index[c[0]] - j = bisect.bisect_left(starts, e) - 1 - genes = set() - while j >= 0 and maxend[j] > s: - if iv[j][1] > s: - genes.add(iv[j][2]) - j -= 1 - if genes: - motif = c[3] if len(c[3]) <= 12 else f"{len(c[3])}bp motif" - rows.append((c[0], s, e, "tandem_repeat", "class_3", f"{'/'.join(sorted(genes))}:{motif}", SRC_TR)) - -# ig_tr_locus: per locus the largest group of genes on one chromosome -for locus, genes in ig.items(): - groups = [] - for chrom, s, e in sorted(genes): - g = groups[-1] if groups else None - if g and g[0] == chrom and s - g[2] < LOCUS_GAP: - g[2] = max(g[2], e) - g[3] += 1 - else: - groups.append([chrom, s, e, 1]) - chrom, s, e, _ = max(groups, key=lambda g: g[3]) - rows.append((chrom, s, e, "ig_tr_locus", "class_3", locus, SRC_IG)) - -# mhc: the assembly report's region (1-based, inclusive) -with open(f"{work}/regions.txt") as f: - mhc = sorted({tuple(l.split("\t")[:4]) for l in f if l.startswith("MHC\t")}) -if len(mhc) != 1: - sys.exit(f"ERROR: want one MHC region in the assembly report, got {mhc}") -_, chrom, s, e = mhc[0] -rows.append((f"chr{chrom}", int(s) - 1, int(e), "mhc", "class_3", "MHC", SRC_MHC)) - -# gene_alias: a pseudogene that carries a disease gene's symbol as alias or previous symbol -with open(f"{work}/hgnc.txt", newline="") as f: - hgnc = list(csv.DictReader(f, delimiter="\t")) -approved = {r["symbol"]: r for r in hgnc} -for r in hgnc: - if r["locus_group"] != "pseudogene" or r["hgnc_id"] not in gene_span: - continue - for sym in sorted({x.strip('"') for x in (r["alias_symbol"] + "|" + r["prev_symbol"]).split("|")} - {""}): - d = approved.get(sym) - if d and d["hgnc_id"] != r["hgnc_id"] and d["gencc"]: - chrom, s, e = gene_span[r["hgnc_id"]] - rows.append((chrom, s, e, "gene_alias", "remap_gene", f"{sym}|{r['symbol']}|{r['hgnc_id']}|{d['hgnc_id']}", - SRC_ALIAS)) +fetch_pinned "$DIR" trx.bed.gz "$TRX_URL" "$TRX_SHA256" || exit 1 +fetch_pinned "$DIR" gencode.gtf.gz "$GENCODE_URL" "$GENCODE_SHA256" || exit 1 +fetch_pinned "$DIR" hgnc.txt "$HGNC_URL" "$HGNC_SHA256" || exit 1 +fetch_pinned "$DIR" regions.txt "$REGIONS_URL" "$REGIONS_SHA256" || exit 1 -rows.sort(key=lambda x: (ORDER[x[0]], x[1], x[2], x[3], x[5])) -counts = defaultdict(int) -for x in rows: - counts[x[3]] += 1 -with open(out, "w") as f: - f.write("# assets/sv_rule_regions.tsv: the GRCh38 regions where og sv postfilter overrides AnnotSV's class 4\n" - "# or 5. Written by scripts/ci/build-sv-rule-regions.sh, which says how each rule's rows are chosen;\n" - "# rebuild it there, do not edit it by hand. start is 0-based and end 1-based, as in a BED file.\n" - "# Rows: " + ", ".join(f"{k} {counts[k]}" for k in sorted(counts)) + ".\n") - f.write("#chrom\tstart\tend\trule\taction\tname\tsource\n") - for x in rows: - f.write("\t".join(map(str, x)) + "\n") -print("Rows: " + ", ".join(f"{k} {counts[k]}" for k in sorted(counts)), file=sys.stderr) -PY -mv "${OUT}.part" "$OUT" +"${REPO}/bin/og" dev build sv-rule-regions --work "$DIR" --out "$OUT" if [ -n "$CHECK" ]; then if cmp -s "$OUT" "$CHECK"; then echo "${CHECK} is what the pinned downloads give, byte for byte" >&2 diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dcf..fe64ffe70 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -62,7 +62,7 @@ bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh bloodtype-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh -callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh +callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz @@ -75,7 +75,7 @@ clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sit clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh +cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py @@ -91,7 +91,7 @@ expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.js families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh +followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index 8d4bfe1c4..4a5a45800 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -196,7 +196,7 @@ PGS000066 b34f31b8b3a7dbbbb58882a70e14649d"} # EBI's FTP server, which refused every connection on 2026-11-06 (run # 42259432681). CATALOGS="callability bash ${REPO}/scripts/ci/build-callability.sh -cn-intervals python3 ${REPO}/scripts/ci/build-cn-intervals.py +cn-intervals bash ${REPO}/scripts/ci/build-cn-intervals.sh sv-rule-regions bash ${REPO}/scripts/ci/build-sv-rule-regions.sh" # catalog_files: "FOLDER NAME URL SHA256" of every download of CATALOGS. catalog_files() { diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh new file mode 100644 index 000000000..ccdb82f01 --- /dev/null +++ b/scripts/ci/lib/fetch-pinned.sh @@ -0,0 +1,61 @@ +# shellcheck shell=bash +# fetch-pinned.sh: the one download rule of the asset builders in +# scripts/ci/ (build-callability.sh, build-cn-intervals.sh, +# build-sv-rule-regions.sh). Source it; it defines two functions. +# +# fetch_pinned DIR NAME URL SHA256 +# Makes DIR/NAME the file at URL with that sha256. A file already there +# with the right sha256 is kept, so DIR can be a cache that outlives the +# run. Otherwise it prints "Downloading URL" on stderr (the e2e cases +# count these lines), downloads to DIR/NAME.part and renames it when the +# sha256 matches. A 403 or 404 fails at once: a missing or forbidden file +# stays so. Any other failure is tried again 6 times, 30 s apart, for +# about five minutes: EBI's FTP server refused every connection for +# minutes at a time on 2026-11-06 (curl exit 7). Returns 1 with an ERROR +# line when the file cannot be had. +# +# pinned_sha256 FILE +# The sha256 of FILE, from sha256sum or shasum. +# +# og never downloads: the builders fetch here, then hand the folder to +# og dev build. + +pinned_sha256() { + if command -v sha256sum > /dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +fetch_pinned() { + local dir=$1 name=$2 url=$3 want=$4 file code got + file="${dir}/${name}" + if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then + return 0 + fi + mkdir -p "$dir" || return 1 + echo "Downloading ${url}" >&2 + if ! code=$(curl -fsSL -w '%{http_code}' -o "${file}.part" "$url"); then + case $code in + 403 | 404) + rm -f "${file}.part" + echo "ERROR: cannot download ${url}: HTTP ${code}" >&2 + return 1 + ;; + esac + if ! curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 \ + -o "${file}.part" "$url"; then + rm -f "${file}.part" + echo "ERROR: cannot download ${url}" >&2 + return 1 + fi + fi + got=$(pinned_sha256 "${file}.part") + if [ "$got" != "$want" ]; then + rm -f "${file}.part" + echo "ERROR: ${name} has sha256 ${got}, want ${want}" >&2 + return 1 + fi + mv "${file}.part" "$file" +} diff --git a/tests/e2e/cn-intervals-1-catalog.sh b/tests/e2e/cn-intervals-1-catalog.sh index 5b0db18cb..dd2a6262f 100755 --- a/tests/e2e/cn-intervals-1-catalog.sh +++ b/tests/e2e/cn-intervals-1-catalog.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The interval rows of bin/cn_loci.tsv are what scripts/ci/build-cn-intervals.py +# The interval rows of bin/cn_loci.tsv are what scripts/ci/build-cn-intervals.sh # writes from its pinned downloads (ClinGen's recurrent CNV BED V2.1 and three # GIAB v3.6 stratifications, each checked against its sha256): a hand edit of # an interval row fails here. Control: a copy of the catalog with one target @@ -11,9 +11,9 @@ CACHE="${CASE_TMP}/downloads" [ -z "${E2E_DATA_CACHE:-}" ] || CACHE="${E2E_DATA_CACHE}/catalogs/cn-intervals" mkdir -p "$CACHE" -echo "+ scripts/ci/build-cn-intervals.py --check" -python3 "${REPO}/scripts/ci/build-cn-intervals.py" --cache "$CACHE" --check 2>&1 | tee "$STEP_LOG" -check_eq "build-cn-intervals.py --check exits 0 on the committed catalog" "${PIPESTATUS[0]}" 0 +echo "+ scripts/ci/build-cn-intervals.sh --check" +bash "${REPO}/scripts/ci/build-cn-intervals.sh" --cache "$CACHE" --check 2>&1 | tee "$STEP_LOG" +check_eq "build-cn-intervals.sh --check exits 0 on the committed catalog" "${PIPESTATUS[0]}" 0 check "the log says all 90 interval rows were checked" grep -q "the 90 interval rows are what the pinned downloads give" "$STEP_LOG" # Control: one interval target moved by 1 kb @@ -25,7 +25,7 @@ awk -F'\t' -v OFS='\t' '$1 == "22q11.2_DGS_proximal_A-B" && !done { "${REPO}/bin/cn_loci.tsv" > "$EDITED" # shellcheck disable=SC2016 # the positional arguments of bash -c check "control: the edited copy differs from the catalog" bash -c '! cmp -s "$1" "$2"' _ "$EDITED" "${REPO}/bin/cn_loci.tsv" -python3 "${REPO}/scripts/ci/build-cn-intervals.py" --cache "$CACHE" --catalog "$EDITED" --check > "${CASE_TMP}/edited.log" 2>&1 +bash "${REPO}/scripts/ci/build-cn-intervals.sh" --cache "$CACHE" --catalog "$EDITED" --check > "${CASE_TMP}/edited.log" 2>&1 check_eq "control: --check fails (exit 1) on a catalog with one interval target moved" "$?" 1 cat "${CASE_TMP}/edited.log" diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index b00a00302..337dff70a 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -36,7 +36,7 @@ each, and a row shifted by 1 kb no longer matches its fixture; the PARASCOPY module runs the same list; the CCL3L1 reading names no disease (a planted 'HIV' is caught); - 9. the interval rows (scripts/ci/build-cn-intervals.py): a small ClinGen + 9. the interval rows (og dev build cn-intervals, with og): a small ClinGen BED with a 'chrx' row converts to rows on chrX that load, the same rows left as 'chrx' are refused, and an unknown contig stops the build; the committed rows are the ClinGen regions plus the named hand rows, with @@ -684,7 +684,6 @@ def dosage(cat, read): # Section 9: the interval rows. -BUILD = os.path.join(REPO, "scripts", "ci", "build-cn-intervals.py") SMALL_BED = """track name="test" #chrom\tstart\tend\tname\tscore\tstrand\tthickStart\tthickEnd\titemRgb chr22\t18159724\t18924717\tBP-A\t0\t.\t18159724\t18924717\t0,0,0 @@ -724,39 +723,49 @@ def depths(r, target=30.0, control=30.0): return {n: target if n in tg else control for *_, n in cn.regions(r)} -def intervals(cat, read): - import importlib.util - spec = importlib.util.spec_from_file_location("build_cn_intervals", BUILD) - bld = importlib.util.module_from_spec(spec) - spec.loader.exec_module(bld) - rows = bld.build(SMALL_BED.splitlines(True), [], [], [], []) - check("converter: the two region rows of the small BED, its chrx row on chrX", - [(r["locus"], r["chrom"]) for r in rows[:2]] == [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")], - [(r["locus"], r["chrom"]) for r in rows]) +def build_rows(bed): + """og dev build cn-intervals --stdout on a ClinGen BED and no masks: (exit code, rows as dicts, stderr).""" with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - write_catalog(p, rows) - try: - got = len(cn.load_catalog(p)) - except cn.CatalogError as e: - got = str(e) - check("converter: its rows load", got == len(rows), got) - write_catalog(p, [dict(r, chrom="chrx") if r["chrom"] == "chrX" else r for r in rows]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check("converter: the same rows left as 'chrx' are refused", "chrom 'chrx'" in got, got) - try: - bld.build(SMALL_BED.replace("chrx", "chrQ").splitlines(True), [], [], [], []) - got = "built" - except bld.BuildError as e: - got = str(e) - check("converter: an unknown contig stops the build", "'chrQ' is not a GRCh38 chromosome" in got, got) - check("converter: no target piece overlaps a breakpoint row", - all(not (s < 18924717 and 18159724 < e) and not (s < 21562827 and 21111384 < e) for r in rows[:1] - for _, s, e, _ in cn.parse_row(r)["_targets"])) + p = os.path.join(t, "clingen.bed") + with open(p, "w") as f: + f.write(bed) + out = subprocess.run([OG, "dev", "build", "cn-intervals", "--clingen", p, "--segdups", os.devnull, + "--gaps", os.devnull, "--satellites", os.devnull, "--parascopy", os.devnull, + "--stdout"], capture_output=True, text=True) + rows = [dict(zip(cn.COLUMNS, line.split("\t"))) for line in out.stdout.splitlines() if not line.startswith("#")] + return out.returncode, rows, out.stderr + + +def intervals(cat, read): + if not OG: + print("skip: OG is not set: the converter checks of og dev build cn-intervals did not run") + else: + code, rows, err = build_rows(SMALL_BED) + check("converter: the two region rows of the small BED, its chrx row on chrX", + code == 0 and [(r["locus"], r["chrom"]) for r in rows[:2]] + == [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")], + (code, err, [(r["locus"], r["chrom"]) for r in rows])) + with tempfile.TemporaryDirectory() as t: + p = os.path.join(t, "c.tsv") + write_catalog(p, rows) + try: + got = len(cn.load_catalog(p)) + except cn.CatalogError as e: + got = str(e) + check("converter: its rows load", got == len(rows), got) + write_catalog(p, [dict(r, chrom="chrx") if r["chrom"] == "chrX" else r for r in rows]) + try: + cn.load_catalog(p) + got = "loaded" + except cn.CatalogError as e: + got = str(e) + check("converter: the same rows left as 'chrx' are refused", "chrom 'chrx'" in got, got) + code, _, err = build_rows(SMALL_BED.replace("chrx", "chrQ")) + check("converter: an unknown contig stops the build", + code == 2 and "'chrQ' is not a GRCh38 chromosome" in err, (code, err)) + check("converter: no target piece overlaps a breakpoint row", + all(not (s < 18924717 and 18159724 < e) and not (s < 21562827 and 21111384 < e) for r in rows[:1] + for _, s, e, _ in cn.parse_row(r)["_targets"])) iv = [r for r in cat.values() if r["kind"] == "interval" and r["_passes"] == ["."]] names = [r["locus"] for r in iv] diff --git a/tests/test_e2e_data_cache.sh b/tests/test_e2e_data_cache.sh index bfaf97a2d..e403f51ca 100755 --- a/tests/test_e2e_data_cache.sh +++ b/tests/test_e2e_data_cache.sh @@ -74,7 +74,7 @@ else echo "FAIL: the pgs key"; failed=1; fi if "$RUN" --data-key catalogs | grep -qE '^e2e-data-catalogs-[0-9a-f]{16}$'; then echo "PASS: the catalogs key is a hash" else echo "FAIL: the catalogs key"; failed=1; fi # Every row must be "NAME URL SHA256": a malformed row would reach the fill. -list=$( { bash "${REPO}/scripts/ci/build-callability.sh" --list; python3 "${REPO}/scripts/ci/build-cn-intervals.py" --list; bash "${REPO}/scripts/ci/build-sv-rule-regions.sh" --list; } ) +list=$( { bash "${REPO}/scripts/ci/build-callability.sh" --list; bash "${REPO}/scripts/ci/build-cn-intervals.sh" --list; bash "${REPO}/scripts/ci/build-sv-rule-regions.sh" --list; } ) n=$(awk 'NF == 3 && $2 ~ /^https:/ && $3 ~ /^[0-9a-f]{64}$/' <<< "$list" | wc -l | tr -d ' ') all=$(wc -l <<< "$list" | tr -d ' ') if [ "$n" = 15 ] && [ "$all" = 15 ]; then echo "PASS: the three builders list 15 pinned downloads (7, 4 and 4), every row well formed" diff --git a/versions.env b/versions.env index b67c1e177..304024130 100644 --- a/versions.env +++ b/versions.env @@ -407,7 +407,7 @@ EXOMISER_PHENOTYPE_SHA256="aa923530ca43cf5051ed2ba7b221bb73deb2b0dd66695b2ba6046 # The recurrent CNV regions of the interval rows of bin/cn_loci.tsv (step 35): # ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0), # checked against the sha256 of the file as downloaded on 2026-11-04. -# scripts/ci/build-cn-intervals.py turns it into the rows; the e2e case +# scripts/ci/build-cn-intervals.sh turns it into the rows; the e2e case # tests/e2e/cn-intervals-1-catalog.sh rebuilds them and fails on any # difference. Change the two together, then rerun the script. CLINGEN_RECURRENT_CNV_URL="https://ftp.clinicalgenome.org/ClinGen_recurrent_CNV_V2.1-hg38.bed" From 1f2b7fcadaf904c7d42df9284c78f1e9b679dfe2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:29:25 +0100 Subject: [PATCH 03/24] test(dev): plant three wrong builder rules to show the tests go red Red-first control, reverted in the next commit: callability keeps the _PAR_Y copies, the SV rule builder keeps the last record per HGNC id, and each cn-intervals piece ends one base short. Bead: pgp-bs9u.35 --- rust/crates/og-dev/src/build/callability.rs | 3 ++- rust/crates/og-dev/src/build/cn_intervals.rs | 3 ++- rust/crates/og-dev/src/build/sv_rules.rs | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs index d7ce4dece..556c65865 100644 --- a/rust/crates/og-dev/src/build/callability.rs +++ b/rust/crates/og-dev/src/build/callability.rs @@ -111,7 +111,8 @@ fn read_gtf(path: &Path, alt: &HashMap) -> Result Vec { // Ceiling divisions of positive numbers. let n = (len + PIECE - 1) / PIECE; let step = (len + n - 1) / n; - out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); + // PLANTED, reverted in the next commit: each piece ends one base short. + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step) - 1))); } out } diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs index 7004f02b7..1007fc81b 100644 --- a/rust/crates/og-dev/src/build/sv_rules.rs +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -99,9 +99,10 @@ fn read_gtf(path: &Path) -> Result { b"gene" => { let (start, end) = r.span()?; if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) - && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_X")) { - g.span.entry(h.to_owned()).or_insert((chrom, start, end)); + // PLANTED, reverted in the next commit: the last record wins. + g.span.insert(h.to_owned(), (chrom, start, end)); } let (name, ty) = ( r.attr("gene_name")?.unwrap_or(""), From fb4c96c80c57ca9a15888f15b5c92f7e720ca751 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:32:54 +0100 Subject: [PATCH 04/24] feat(summary): read the clinical sections in Rust (og summary collect --only b) ClinVar, the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical filter with its ACMG SF tier, slivar and the cyp2d6 block, each input read once, beside the Python collector. No caller changes. Bead: pgp-bs9u.42 --- rust/Cargo.lock | 2 + rust/crates/og-summary/Cargo.toml | 4 +- rust/crates/og-summary/src/collect.rs | 21 +- rust/crates/og-summary/src/collect/acmg.rs | 871 ++++++++++++++++ rust/crates/og-summary/src/collect/assets.rs | 297 ++++++ rust/crates/og-summary/src/collect/carrier.rs | 146 +++ .../crates/og-summary/src/collect/clinical.rs | 208 ++++ rust/crates/og-summary/src/collect/clinvar.rs | 799 ++++++++++++++ rust/crates/og-summary/src/collect/cpsr.rs | 264 +++++ rust/crates/og-summary/src/collect/group_b.rs | 284 +++++ rust/crates/og-summary/src/collect/model.rs | 5 + rust/crates/og-summary/src/collect/pgx.rs | 618 +++++++++++ rust/crates/og-summary/src/collect/scan.rs | 19 + .../crates/og-summary/src/collect/sections.rs | 18 + rust/crates/og-summary/src/collect/text.rs | 195 ++++ .../crates/og-summary/src/collect/two_hits.rs | 980 ++++++++++++++++++ 16 files changed, 4727 insertions(+), 4 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/acmg.rs create mode 100644 rust/crates/og-summary/src/collect/assets.rs create mode 100644 rust/crates/og-summary/src/collect/carrier.rs create mode 100644 rust/crates/og-summary/src/collect/clinical.rs create mode 100644 rust/crates/og-summary/src/collect/clinvar.rs create mode 100644 rust/crates/og-summary/src/collect/cpsr.rs create mode 100644 rust/crates/og-summary/src/collect/group_b.rs create mode 100644 rust/crates/og-summary/src/collect/pgx.rs create mode 100644 rust/crates/og-summary/src/collect/text.rs create mode 100644 rust/crates/og-summary/src/collect/two_hits.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d91a18ca..2edbd24e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -839,6 +839,8 @@ dependencies = [ "flate2", "indexmap", "memchr", + "og-bam", + "og-clinvar", "og-io", "og-steps", "pycompat", diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 1a7fa879c..d17584417 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-summary" -description = "The summary cluster: the step tools that turn a third-party tool's output into one of OwnGenome's tables (step 33's sample QC verdict, step 25's PRS table, step 20's chrM tables)" +description = "The summary cluster: the step tools that turn a third-party tool's output into one of OwnGenome's tables (step 33's sample QC verdict, step 25's PRS table, step 20's chrM tables), and og summary collect, summary.json section by section" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,8 @@ publish.workspace = true [dependencies] pycompat.workspace = true og-io.workspace = true +og-bam.workspace = true +og-clinvar.workspace = true og-steps.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 0ae779232..cc5e827f3 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -16,10 +16,20 @@ //! sections are read. A reader's error stays in its section, which is //! then `unreadable` with a note that names the file. +pub mod acmg; +pub mod assets; +pub mod carrier; +pub mod clinical; +pub mod clinvar; +pub mod cpsr; +pub mod group_b; pub mod model; +pub mod pgx; pub mod scan; pub mod sections; +pub mod text; pub mod time; +pub mod two_hits; use crate::args::{self, Kind, Opt}; use model::{GROUP_A, Manifest, RunStatus, Section, Spec, Summary}; @@ -29,7 +39,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only a +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,b} --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] "; @@ -39,7 +49,7 @@ const OPTS: &[Opt] = &[ Opt("--declared-sex", Kind::Value, false), Opt("--out", Kind::Value, true), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a"]), true), + Opt("--only", Kind::Choice(&["a", "b"]), true), ]; /// What one collect reads. @@ -75,7 +85,11 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { // A part of the sections must never replace the sample's complete // summary.json, so there is no default path while `--only` exists. let target = a.text("--out").to_owned(); - let summary = match collect(&input) { + let read = match a.text("--only") { + "b" => group_b::collect, + _ => collect, + }; + let summary = match read(&input) { Ok(s) => s, Err(e) => return crate::finish(Err(e), err), }; @@ -195,6 +209,7 @@ pub fn collect(input: &Input) -> Result { manifest, declared_sex, sections, + cyp2d6: None, }) } diff --git a/rust/crates/og-summary/src/collect/acmg.rs b/rust/crates/og-summary/src/collect/acmg.rs new file mode 100644 index 000000000..70439bbd8 --- /dev/null +++ b/rust/crates/og-summary/src/collect/acmg.rs @@ -0,0 +1,871 @@ +//! The ACMG secondary-findings tier of the clinical section: the variants +//! that are secondary findings by the rule of their gene +//! (`assets/acmg_sf.tsv`, ACMG SF v3.3, Lee et al., Genet Med 2025). +//! +//! Three sources: step 06's ClinVar P/LP hits, the clinical filter's rare +//! HIGH-impact records (step 23, ranked MANE Select, then canonical, then +//! any transcript, as `og vcf csq-rank` ranks them) and step 05's class 4 +//! and 5 SVs in the genes whose rule allows them. A variant of an SF gene +//! that its rule does not report (one MUTYH variant, a heterozygous HFE +//! p.C282Y) is listed under `not_reported` with the reason. A called +//! deletion counts as the second hit of a gene whose rule needs two +//! ([`two_hits`](super::two_hits)), unless read-backed phase puts both on +//! one copy. + +use super::assets::{ACMG_SF_VERSION, Key, Rule, acmg_sf_rules, carrier_genes}; +use super::clinvar::{ + COMMON_AF, Floor, Hit, HitRecords, allele_copies, floor_values, obj, opt_int, opt_s, +}; +use super::text::{self, digits, format_map, gt_field, info_map}; +use super::two_hits::{DeletionInputs, Spans, Variant, gene_deletions, two_hits}; +use indexmap::IndexMap; +use pycompat::json::Value; +use pycompat::text::isdigit; +use std::collections::{HashMap, HashSet}; + +/// The genotype check's floors: step 06's GQ and DP, the alt read share a +/// heterozygote must lie within and a homozygous alt call must reach. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ClinicalFloor { + pub call: Floor, + pub het_ab: (f64, f64), + pub hom_ab: f64, +} + +impl Default for ClinicalFloor { + fn default() -> Self { + ClinicalFloor { + call: Floor::default(), + het_ab: (0.2, 0.8), + hom_ab: 0.8, + } + } +} + +/// The length of the `[0-9.]+` run at the start of `s`. +fn run(s: &str) -> usize { + s.bytes() + .take_while(|c| c.is_ascii_digit() || *c == b'.') + .count() +} + +/// `re.search(KEY([0-9.]+)-([0-9.]+))` (with `pair`) or `KEY([0-9.]+)`: +/// the first place the whole pattern matches. +fn search<'a>(line: &'a str, key: &str, pair: bool) -> Option<(&'a str, Option<&'a str>)> { + line.match_indices(key).find_map(|(i, _)| { + let rest = &line[i + key.len()..]; + let n = run(rest); + if n == 0 { + return None; + } + if !pair { + return Some((&rest[..n], None)); + } + let hi = rest[n..].strip_prefix('-')?; + let m = run(hi); + (m > 0).then(|| (&rest[..n], Some(&hi[..m]))) + }) +} + +fn float(s: &str) -> Result { + crate::float(s).map_err(|e| format!("ValueError: {e}")) +} + +impl ClinicalFloor { + /// Reads the floors `og vcf csq-rank` wrote into the header. + pub fn read(lines: &[String]) -> Result { + let mut f = ClinicalFloor::default(); + for l in lines.iter().take_while(|l| l.starts_with("##")) { + if !l.starts_with("##PGP_CALL_FLOOR=") { + continue; + } + floor_values(l, true, &mut f.call); + if let Some((lo, Some(hi))) = search(l, "HET_AB=", true) { + f.het_ab = (float(lo)?, float(hi)?); + } + if let Some((v, _)) = search(l, "HOM_AB=", false) { + f.hom_ab = float(v)?; + } + } + Ok(f) + } +} + +/// One ranked CSQ entry. +#[derive(Debug, Clone, PartialEq)] +pub struct CsqEntry { + pub impact: String, + pub gene: String, + pub consequence: String, +} + +/// `csq_rank`: the transcript class (MANE, CANONICAL or ANY), its entries, +/// and the frequency word (rare, common, unknown, or none when the CSQ has +/// no frequency field) from MAX_AF, else gnomADe_AF and gnomADg_AF, over +/// every entry. +pub fn csq_rank(fmt: &[String], csq: &str) -> (&'static str, Vec, &'static str) { + let ix: HashMap<&str, usize> = fmt + .iter() + .enumerate() + .map(|(i, k)| (k.as_str(), i)) + .collect(); + let entries: Vec> = csq + .split(',') + .filter(|t| !t.is_empty()) + .map(|t| t.split('|').collect::>()) + .filter(|v| v.len() == fmt.len()) + .collect(); + let get = |v: &[&str], k: &str| ix.get(k).map_or("", |&i| v[i]).to_owned(); + let mane: Vec<&Vec<&str>> = entries + .iter() + .filter(|v| !get(v, "MANE_SELECT").is_empty()) + .collect(); + let canon: Vec<&Vec<&str>> = entries + .iter() + .filter(|v| get(v, "CANONICAL") == "YES") + .collect(); + let (cls, chosen) = if !mane.is_empty() { + ("MANE", mane) + } else if !canon.is_empty() { + ("CANONICAL", canon) + } else { + ("ANY", entries.iter().collect()) + }; + let fields: Vec<&str> = if ix.contains_key("MAX_AF") { + vec!["MAX_AF"] + } else { + ["gnomADe_AF", "gnomADg_AF"] + .into_iter() + .filter(|f| ix.contains_key(f)) + .collect() + }; + let afs: Vec = entries + .iter() + .flat_map(|v| { + fields + .iter() + .flat_map(|f| { + get(v, f) + .split('&') + .filter_map(|a| pycompat::num::parse_float(a).ok()) + .collect::>() + }) + .collect::>() + }) + .collect(); + let freq = if fields.is_empty() { + "none" + } else { + match text::py_max(afs) { + None => "unknown", + Some(m) if m < COMMON_AF => "rare", + Some(_) => "common", + } + }; + let out = chosen + .iter() + .map(|v| CsqEntry { + impact: get(v, "IMPACT"), + gene: get(v, "SYMBOL"), + consequence: get(v, "Consequence").replace('&', ","), + }) + .collect(); + (cls, out, freq) +} + +/// The genotype check of one call: weak, the reasons, GQ, DP and AD. +pub fn call_check( + fmt: &str, + sample: &str, + floor: &ClinicalFloor, +) -> ( + bool, + Vec, + Option, + Option, + Option, +) { + let f = format_map(fmt, sample); + let gq = digits(f.get("GQ").copied()); + let dp = digits(f.get("DP").copied()); + let ad = f.get("AD").copied().unwrap_or(""); + let mut why = Vec::new(); + if let Some(g) = gq + && g < floor.call.gq + { + why.push(format!("GQ {g} < {}", floor.call.gq)); + } + if let Some(d) = dp + && d < floor.call.dp + { + why.push(format!("DP {d} < {}", floor.call.dp)); + } + let gt = f.get("GT").copied().unwrap_or("").replace('|', "/"); + let al: Vec<&str> = gt.split('/').collect(); + let depths: Vec<&str> = ad.split(',').collect(); + if al.len() == 2 + && al.iter().all(|a| isdigit(a)) + && depths.len() > 1 + && depths.iter().all(|x| isdigit(x)) + { + let n: Vec = depths.iter().filter_map(|x| crate::int(x).ok()).collect(); + let tot: i128 = n.iter().sum(); + if tot != 0 { + let ab = n[1..].iter().sum::() as f64 / tot as f64; + let (lo, hi) = floor.het_ab; + if (al[0] == "0") != (al[1] == "0") && !(lo <= ab && ab <= hi) { + why.push(format!( + "alt reads {} of the depth on a heterozygote", + pycompat::fmt::fixed(ab, 2) + )); + } else if al[0] == al[1] && al[1] != "0" && ab < floor.hom_ab { + why.push(format!( + "alt reads {} of the depth on a homozygous call", + pycompat::fmt::fixed(ab, 2) + )); + } + } + } + ( + !why.is_empty(), + why, + gq, + dp, + (!ad.is_empty()).then(|| ad.to_owned()), + ) +} + +/// One record of the clinical VCF with a HIGH-impact ranked entry. +#[derive(Debug, Clone, PartialEq)] +pub struct HighRecord { + pub genotype: String, + pub ps: Option, + /// Gene to its first HIGH consequence. + pub genes: IndexMap, + pub transcript: &'static str, + pub frequency: &'static str, + pub weak: bool, + pub why: Vec, + pub gq: Option, + pub dp: Option, + pub ad: Option, +} + +/// The clinical VCF read once: its records count and its HIGH-impact +/// records. An error is the note of a file that cannot be read. +#[derive(Debug, Clone)] +pub struct ClinicalVcf { + pub records: usize, + /// Keyed by `(chrom, pos, ref, alt)`; an error is the collector's note + /// (a POS that is no integer on a HIGH-impact record, a bad floor). + pub high: Result, String>, +} + +impl ClinicalVcf { + pub fn read(path: &str) -> Result { + let lines = text::lines(path).map_err(|e| e.unreadable(path))?; + let records = lines.iter().filter(|l| !l.starts_with('#')).count(); + Ok(ClinicalVcf { + records, + high: high_impact(&lines), + }) + } +} + +/// `csq_high_impact_genes`. +fn high_impact(lines: &[String]) -> Result, String> { + let floor = ClinicalFloor::read(lines)?; + let mut fmt: Option> = None; + let mut out = IndexMap::new(); + for line in lines { + if line.starts_with("##INFO= = line.split('\t').map(str::to_owned).collect(); + if r.len() < 8 { + continue; + } + let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); + let mut genes: IndexMap = IndexMap::new(); + for e in entries.into_iter().filter(|e| e.impact == "HIGH") { + genes.entry(e.gene).or_insert(e.consequence); + } + if genes.is_empty() { + continue; + } + let gt = text::gt_of(&r).to_owned(); + let (weak, why, gq, dp, ad) = if r.len() > 9 { + call_check(&r[8], &r[9], &floor) + } else { + (false, Vec::new(), None, None, None) + }; + let pos = text::pos(&r)?; + out.insert( + (r[0].clone(), pos, r[3].clone(), r[4].clone()), + HighRecord { + genotype: gt, + ps: gt_field(&r, "PS"), + genes, + transcript: cls, + frequency: freq, + weak, + why, + gq, + dp, + ad, + }, + ); + } + Ok(out) +} + +/// A source of a candidate variant. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + ClinVar, + High, + Sv, +} + +impl Kind { + fn source(self) -> &'static str { + match self { + Kind::ClinVar => "ClinVar", + Kind::High => "HIGH impact", + Kind::Sv => "AnnotSV", + } + } +} + +/// A variant key the verdicts are counted by: a small variant, an SV by +/// its span, or a gene's called deletion. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum AKey { + Small(Key), + Sv(String, String, String, String), + Deletion(String), +} + +/// One candidate: the fields it is printed with (in their order), its key, +/// copies and ploidy, and its call's GT and phase set. +#[derive(Debug, Clone)] +struct Cand { + kind: Kind, + gene: String, + fields: IndexMap, + key: AKey, + copies: usize, + ploidy: usize, + gt: String, + ps: Option, +} + +/// `sf_verdicts`: for each distinct variant of a gene, `None` (reported) or +/// the reason it is not, and the partial-deficiency note of a reported gene +/// with a hypomorph allele. +fn sf_verdicts( + gene: &str, + rule: &Rule, + alleles: &IndexMap, +) -> (HashMap>, Option) { + let sev = |k: &AKey| match k { + AKey::Small(k) => rule.severity.get(k), + _ => None, + }; + let mut out: IndexMap> = IndexMap::new(); + let mut kept: IndexMap = IndexMap::new(); + for (k, cp) in alleles { + match sev(k) { + Some(a) if a.kind.as_deref() == Some("pseudodeficiency") => { + out.insert( + k.clone(), + Some(format!( + "{}: a pseudodeficiency allele, not counted", + a.name + )), + ); + } + _ => { + kept.insert(k.clone(), *cp); + } + } + } + let n: usize = kept.values().map(|(c, _)| c).sum(); + let ri = rule.report_if.as_str(); + let hemi = kept.values().any(|(_, p)| *p == 1); + for (k, (c, p)) in &kept { + let good = match ri { + "any" | "het_female_too" => true, + "biallelic" => n >= 2, + "hemizygous" => hemi || n >= 2, + _ => matches!(k, AKey::Small(s) if rule.variants.contains_key(s)) && *c >= 2 && *p == 2, + }; + let why = if good { + None + } else if ri == "hom_variant" { + Some(format!("{gene} is reported only for: {}", rule.table_text)) + } else if ri == "hemizygous" { + Some(format!( + "one heterozygous variant; {gene} is reported for {}", + rule.table_text + )) + } else { + Some(format!( + "one heterozygous variant; {gene} is reported with {}", + rule.table_text + )) + }; + out.insert(k.clone(), why); + } + let hypo: Vec<&str> = out + .iter() + .filter(|(_, why)| why.is_none()) + .filter_map(|(k, _)| sev(k).filter(|a| a.kind.as_deref() == Some("hypomorph"))) + .map(|a| a.name.as_str()) + .collect(); + let partial = (!hypo.is_empty()).then(|| { + format!( + "partial, not profound: {} is a hypomorph allele, so the deficiency it causes is partial", + hypo.join("; ") + ) + }); + (out.into_iter().collect(), partial) +} + +/// `sf_structural`: step 05's class 4 and 5 SVs in genes whose rule allows +/// them, and the class 4 or 5 deletions over a gene's sv_regions. +fn sf_structural( + inp: &DeletionInputs, + rules: &IndexMap, +) -> (Vec, bool, Option) { + let rows = match &inp.annotsv { + Err(e) => { + return ( + Vec::new(), + false, + Some(format!("the AnnotSV table could not be read: {e}")), + ); + } + Ok(None) => return (Vec::new(), false, None), + Ok(Some(rows)) => rows, + }; + let mut out = Vec::new(); + for x in rows { + if x.common || !matches!(x.class(), "4" | "5") { + continue; + } + let (chrom, span) = x.sv.split_once(':').unwrap_or((&x.sv, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + let chrom = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let mut hits: IndexMap> = IndexMap::new(); + for g in &x.genes { + if rules.get(g).is_some_and(|r| r.sv_cn) { + hits.insert(g.clone(), None); + } + } + let sv_type = x.sv_type.as_deref().unwrap_or("None"); + let (an, bn) = ( + isdigit(a).then(|| crate::int(a).ok()).flatten(), + isdigit(b).then(|| crate::int(b).ok()).flatten(), + ); + for (g, rule) in rules { + for (c, rs, re, name) in &rule.sv_regions { + if let (Some(an), Some(bn)) = (an, bn) + && sv_type == "DEL" + && *c == chrom + && an <= *re + && bn >= *rs + && !hits.contains_key(g) + { + hits.insert(g.clone(), Some(name.clone())); + } + } + } + hits.sort_keys(); + for (g, via) in hits { + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.clone())); + f.insert( + "variant".into(), + Value::Str(format!("{sv_type} {chrom}:{a}-{b}")), + ); + f.insert("acmg_class".into(), Value::Str(x.class().to_owned())); + f.insert("caller".into(), opt_s(x.caller.as_deref())); + f.insert("support".into(), opt_int(x.support)); + f.insert("via".into(), opt_s(via.as_deref())); + out.push(Cand { + kind: Kind::Sv, + gene: g, + fields: f, + key: AKey::Sv( + chrom.clone(), + a.to_owned(), + b.to_owned(), + sv_type.to_owned(), + ), + copies: 1, + ploidy: 2, + gt: String::new(), + ps: None, + }); + } + } + (out, true, None) +} + +fn zyg_map(gt: &str) -> String { + match gt { + "0/1" | "1/0" | "0|1" | "1|0" => "het".into(), + "1/1" | "1|1" => "hom".into(), + _ => gt.into(), + } +} + +/// What the ACMG tier reads. +pub struct Inputs<'a> { + pub assets: &'a str, + /// Step 06's floor and hits, or the note of a hits VCF that cannot be + /// read; `None` without one. + pub hits: Option<&'a Result>, + /// The clinical VCF, or its note; `None` without one. + pub clinical: Option<&'a Result>, + pub deletions: &'a DeletionInputs, +} + +/// `acmg_sf_or_note`: the tier, or `Ok` of the unavailable form when a +/// list or an input cannot be read. An `Err` is the collector's note of the +/// whole clinical section. +pub fn acmg_sf(i: &Inputs) -> Result { + match tier(i)? { + Ok(v) => Ok(v), + Err(why) => Ok(obj([ + ("version", Value::Str(ACMG_SF_VERSION.into())), + ("unavailable", Value::Str(why)), + ("clinvar_hits", Value::Array(Vec::new())), + ("high_impact", Value::Array(Vec::new())), + ("structural", Value::Array(Vec::new())), + ("not_reported", Value::Array(Vec::new())), + ])), + } +} + +/// `acmg_sf_tier`: the outer error is the section's note, the inner one +/// the reason the tier is unavailable. +fn tier(i: &Inputs) -> Result, String> { + let rules = match acmg_sf_rules(i.assets) { + Ok(r) => r, + Err(e) => return Ok(Err(e)), + }; + let mut clinvar: Vec = Vec::new(); + if let Some(h) = i.hits { + let (floor, recs) = match h { + Ok(x) => x, + Err(e) => return Ok(Err(e.clone())), + }; + for r in recs.iter().filter(|r| r.len() >= 8) { + let row = Hit::parse(r, *floor)?; + let gt = text::gt_of(r).to_owned(); + let (copies, ploidy) = allele_copies(>); + for g in row.gene.split(',') { + if !rules.contains_key(g) { + continue; + } + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.to_owned())); + f.insert("variant".into(), Value::Str(row.variant())); + f.insert("genotype".into(), Value::Str(row.genotype.clone())); + f.insert("significance".into(), Value::Str(row.significance.clone())); + f.insert("stars".into(), Value::Int(row.stars.into())); + f.insert("low_confidence".into(), Value::Bool(row.low_confidence)); + clinvar.push(Cand { + kind: Kind::ClinVar, + gene: g.to_owned(), + fields: f, + key: AKey::Small(row.key()), + copies, + ploidy, + gt: gt.clone(), + ps: gt_field(r, "PS"), + }); + } + } + } + let mut high: Vec = Vec::new(); + if let Some(c) = i.clinical { + let c = match c { + Ok(c) => c, + Err(e) => return Ok(Err(e.clone())), + }; + let mut recs: Vec<(&Key, &HighRecord)> = + c.high.as_ref().map_err(Clone::clone)?.iter().collect(); + recs.sort_by(|a, b| a.0.cmp(b.0)); + for ((ch, pos, r, a), x) in recs { + if x.frequency == "common" { + continue; + } + let mut genes: Vec<(&String, &String)> = x.genes.iter().collect(); + genes.sort(); + for (g, cons) in genes { + if !rules.contains_key(g) { + continue; + } + let (copies, ploidy) = allele_copies(&x.genotype); + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.clone())); + f.insert("variant".into(), Value::Str(format!("{ch}:{pos} {r}>{a}"))); + f.insert("genotype".into(), Value::Str(zyg_map(&x.genotype))); + f.insert("consequence".into(), Value::Str(cons.clone())); + f.insert("transcript".into(), Value::Str(x.transcript.into())); + f.insert("frequency".into(), Value::Str(x.frequency.into())); + f.insert( + "frequency_unknown".into(), + Value::Bool(x.frequency == "unknown"), + ); + f.insert("weak_call".into(), Value::Bool(x.weak)); + f.insert( + "call_note".into(), + if x.why.is_empty() { + Value::Null + } else { + Value::Str(x.why.join("; ")) + }, + ); + f.insert("gq".into(), opt_int(x.gq)); + f.insert("dp".into(), opt_int(x.dp)); + f.insert("ad".into(), opt_s(x.ad.as_deref())); + high.push(Cand { + kind: Kind::High, + gene: g.clone(), + fields: f, + key: AKey::Small((ch.clone(), *pos, r.clone(), a.clone())), + copies, + ploidy, + gt: x.genotype.clone(), + ps: x.ps.clone(), + }); + } + } + } + let (svs, sv_checked, sv_note) = sf_structural(i.deletions, &rules); + + // One verdict per gene, over its distinct variants. + let mut alleles: IndexMap> = IndexMap::new(); + for x in clinvar.iter().chain(&high) { + let p = if x.ploidy == 0 { 2 } else { x.ploidy }; + alleles + .entry(x.gene.clone()) + .or_default() + .entry(x.key.clone()) + .or_insert((x.copies.max(1), p)); + } + for x in &svs { + alleles + .entry(x.gene.clone()) + .or_default() + .entry(x.key.clone()) + .or_insert((1, 2)); + } + let (spans, two_hit_note): (Spans, Option) = match carrier_genes(i.assets) { + Ok(g) => (g.into_iter().map(|(k, v)| (k, v.span)).collect(), None), + Err(e) => ( + IndexMap::new(), + Some(format!( + "A deletion as the second hit: only AnnotSV's gene names were read; step 35's \ + deletions need the gene spans of {e}" + )), + ), + }; + let (dels, _, _) = gene_deletions(i.deletions, &spans); + let mut seen = HashSet::new(); + let mut variants = Vec::new(); + for x in clinvar.iter().chain(&high) { + let AKey::Small(k) = &x.key else { continue }; + let pseudo = rules[&x.gene].severity_kind(k) == Some("pseudodeficiency"); + if !pseudo && seen.insert((x.gene.clone(), k.clone())) { + variants.push(Variant { + gene: x.gene.clone(), + key: k.clone(), + chrom: k.0.clone(), + pos: k.1, + text: match &x.fields["variant"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }, + genotype: match &x.fields["genotype"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }, + copies: x.copies, + ploidy: if x.ploidy == 0 { 2 } else { x.ploidy }, + gt: x.gt.clone(), + ps: x.ps.clone(), + allele: None, + }); + } + } + let found = two_hits(&variants, &dels); + let mut two: HashMap<(String, Key), String> = HashMap::new(); + for g in found.entries.keys() { + if matches!(rules[g].report_if.as_str(), "biallelic" | "hemizygous") { + if let Some(a) = alleles.get_mut(g) { + a.insert(AKey::Deletion(g.clone()), (1, 2)); + } + two.extend( + found + .pairs + .iter() + .filter(|(k, _)| &k.0 == g) + .map(|(k, t)| (k.clone(), t.clone())), + ); + } + } + let verdict: HashMap<&String, _> = alleles + .iter() + .map(|(g, a)| (g, sf_verdicts(g, &rules[g], a))) + .collect(); + let mut reported: [Vec; 3] = Default::default(); + let mut not_reported = Vec::new(); + for x in clinvar.into_iter().chain(high).chain(svs) { + let (whys, partial) = &verdict[&x.gene]; + let why = whys.get(&x.key).cloned().flatten(); + let rule = &rules[&x.gene]; + let mut f = x.fields; + if let AKey::Small(k) = &x.key + && found.hemi.contains_key(&(x.gene.clone(), k.clone())) + { + f.insert("genotype".into(), Value::Str("hemizygous".into())); + } + let slot = match x.kind { + Kind::ClinVar => 0, + Kind::High => 1, + Kind::Sv => 2, + }; + match why { + None => { + f.insert("rule".into(), Value::Str(rule.table_text.clone())); + f.insert("rule_note".into(), opt_s(rule.note.as_deref())); + f.insert("severity".into(), opt_s(partial.as_deref())); + if x.kind != Kind::Sv + && let AKey::Small(k) = &x.key + && let Some(t) = two.get(&(x.gene.clone(), k.clone())) + { + f.insert("two_hit".into(), Value::Str(t.clone())); + } + reported[slot].push(Value::Object(f)); + } + Some(reason) => not_reported.push(obj([ + ("gene", Value::Str(x.gene.clone())), + ("variant", f["variant"].clone()), + ( + "genotype", + f.get("genotype").cloned().unwrap_or(Value::Null), + ), + ("source", Value::Str(x.kind.source().into())), + ("reason", Value::Str(reason)), + ])), + } + } + let [clinvar_rows, high_rows, sv_rows] = reported; + let flag = + |x: &Value, k: &str| matches!(x, Value::Object(m) if m.get(k) == Some(&Value::Bool(true))); + let sure = high_rows + .iter() + .filter(|x| !(flag(x, "weak_call") || flag(x, "frequency_unknown"))) + .count(); + let weak = high_rows.iter().filter(|x| flag(x, "weak_call")).count(); + let unknown = high_rows + .iter() + .filter(|x| flag(x, "frequency_unknown")) + .count(); + Ok(Ok(obj([ + ("version", Value::Str(ACMG_SF_VERSION.into())), + ("genes_on_list", Value::Int(rules.len() as i128)), + ("rules", Value::Str("assets/acmg_sf.tsv".into())), + ("clinvar_hits", Value::Array(clinvar_rows)), + ("clinvar_checked", Value::Bool(i.hits.is_some())), + ("high_impact", Value::Array(high_rows)), + ("high_impact_checked", Value::Bool(i.clinical.is_some())), + ("high_impact_rare", Value::Int(sure as i128)), + ("high_impact_weak", Value::Int(weak as i128)), + ("high_impact_frequency_unknown", Value::Int(unknown as i128)), + ("structural", Value::Array(sv_rows)), + ("structural_checked", Value::Bool(sv_checked)), + ("structural_note", opt_s(sv_note.as_deref())), + ("not_reported", Value::Array(not_reported)), + ("two_hit_note", opt_s(two_hit_note.as_deref())), + ]))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fmt(s: &str) -> Vec { + s.split('|').map(str::to_owned).collect() + } + + #[test] + fn mane_then_canonical_then_any() { + let f = fmt("Allele|IMPACT|SYMBOL|Consequence|CANONICAL|MANE_SELECT|gnomADe_AF"); + let (c, e, q) = csq_rank( + &f, + "T|HIGH|A|stop_gained|YES||0.5,T|LOW|A|synonymous_variant||NM_1|0.001", + ); + assert_eq!((c, q), ("MANE", "common")); + assert_eq!(e.len(), 1); + assert_eq!(e[0].impact, "LOW"); + let (c, _, q) = csq_rank(&f, "T|HIGH|A|stop_gained&splice|YES||"); + assert_eq!((c, q), ("CANONICAL", "unknown")); + let f2 = fmt("Allele|IMPACT|SYMBOL|Consequence"); + let (c, e, q) = csq_rank(&f2, "T|HIGH|A|a&b"); + assert_eq!((c, q), ("ANY", "none")); + assert_eq!(e[0].consequence, "a,b"); + } + + #[test] + fn the_genotype_check() { + let f = ClinicalFloor::default(); + let (weak, why, gq, dp, ad) = call_check("GT:GQ:DP:AD", "0/1:20:30:25,5", &f); + assert!(weak); + assert_eq!( + why, + [ + "GQ 20 < 30", + "alt reads 0.17 of the depth on a heterozygote" + ] + ); + assert_eq!((gq, dp, ad.as_deref()), (Some(20), Some(30), Some("25,5"))); + let (weak, why, ..) = call_check("GT:AD", "1|1:5,15", &f); + assert!(weak); + assert_eq!(why, ["alt reads 0.75 of the depth on a homozygous call"]); + let (weak, ..) = call_check("GT:GQ:DP:AD", "0/1:40:30:15,15", &f); + assert!(!weak); + } + + #[test] + fn the_floors_of_the_header() { + let l = |s: &str| vec![s.to_owned(), "#CHROM".to_owned()]; + let f = ClinicalFloor::read(&l( + "##PGP_CALL_FLOOR=GQ=20,DP=5,HET_AB=0.25-0.75,HOM_AB=0.9", + )) + .unwrap(); + assert_eq!(f.call, Floor { gq: 20, dp: 5 }); + assert_eq!((f.het_ab, f.hom_ab), ((0.25, 0.75), 0.9)); + assert!(ClinicalFloor::read(&l("##PGP_CALL_FLOOR=HOM_AB=0..9")).is_err()); + } +} diff --git a/rust/crates/og-summary/src/collect/assets.rs b/rust/crates/og-summary/src/collect/assets.rs new file mode 100644 index 000000000..a71eeb613 --- /dev/null +++ b/rust/crates/og-summary/src/collect/assets.rs @@ -0,0 +1,297 @@ +//! The two gene lists the clinical sections read from `assets/`: +//! `acmg_sf.tsv` (the ACMG SF v3.3 genes and the reporting rule of each, +//! Lee et al., Genet Med 2025, PMID 40568962) and `carrier_genes.tsv` (the +//! genes of the carrier list and their spans). +//! +//! A missing or broken list is an error with the words the reports print; +//! the section that needs it says the list is unavailable and keeps its +//! other values. + +use crate::{basename, join}; +use indexmap::IndexMap; +use pycompat::csv::{DictReader, DictRow, Quoting}; + +/// The list version both reports print. +pub const ACMG_SF_VERSION: &str = "ACMG SF v3.3"; +/// The rules a gene of the ACMG list can have. +pub const SF_REPORT_IF: [&str; 5] = [ + "any", + "biallelic", + "hom_variant", + "hemizygous", + "het_female_too", +]; +/// The kinds of a severity entry. +pub const SEVERITY_KINDS: [&str; 2] = ["hypomorph", "pseudodeficiency"]; + +/// A variant as `(chrom, pos, ref, alt)`. +pub type Key = (String, i128, String, String); + +/// What a severity entry or a variants entry names. +#[derive(Debug, Clone, PartialEq)] +pub struct Allele { + /// `hypomorph` or `pseudodeficiency`; `None` in a variants cell. + pub kind: Option, + pub name: String, +} + +/// The reporting rule of one ACMG gene. +#[derive(Debug, Clone)] +pub struct Rule { + pub report_if: String, + pub variants: IndexMap, + /// AnnotSV's class 4 and 5 SVs count in this gene. + pub sv_cn: bool, + /// Regions where a deletion counts for the gene: chrom, start, end, name. + pub sv_regions: Vec<(String, i128, i128, String)>, + pub severity: IndexMap, + pub table_text: String, + pub note: Option, +} + +impl Rule { + pub fn severity_kind(&self, key: &Key) -> Option<&str> { + self.severity.get(key).and_then(|a| a.kind.as_deref()) + } +} + +/// One gene of the carrier list. +#[derive(Debug, Clone)] +pub struct CarrierGene { + pub inheritance: String, + pub condition: String, + pub severity: IndexMap, + /// GRCh38, 1-based, inclusive. + pub span: (String, i128, i128), +} + +/// The rows of `assets/` without its `#` lines, cells as written +/// (no quoting). +pub fn asset_rows(assets: &str, name: &str) -> Result, String> { + let p = join(assets, name); + let missing = || { + format!( + "{name} is not in {assets}: the report needs the repository's assets/ folder \ + (render_report.py --assets)" + ) + }; + if !crate::is_file(&p) { + return Err(missing()); + } + let bytes = std::fs::read(&p).map_err(|e| crate::os_error(&e, &p))?; + let text = String::from_utf8(bytes).map_err(|e| { + format!( + "UnicodeDecodeError: 'utf-8' codec can't decode {}: {e}", + basename(&p) + ) + })?; + let lines = text + .split_inclusive('\n') + .filter(|l| !l.starts_with('#')) + .map(|l| Ok::<_, std::io::Error>(l.to_owned())); + DictReader::new(lines, '\t', Quoting::None) + .collect::, _>>() + .map_err(|e| format!("{}: {e}", e.exception())) +} + +/// `row[key]`, its error as `str(KeyError)`. +fn col<'a>(r: &'a DictRow, key: &str) -> Result, String> { + r.get(key).ok_or_else(|| crate::repr(key)) +} + +fn int(s: Option<&str>) -> Result { + match s { + None => Err( + "int() argument must be a string, a bytes-like object or a real number, not 'NoneType'" + .into(), + ), + Some(s) => crate::int(s), + } +} + +/// `'chr6:26092913:G:A'` as a key. +pub fn allele_key(locus: &str) -> Result { + let f: Vec<&str> = locus.split(':').collect(); + match f.as_slice() { + [c, pos, r, a] => Ok(( + (*c).to_owned(), + crate::int(pos)?, + (*r).to_owned(), + (*a).to_owned(), + )), + _ if f.len() < 4 => Err(format!( + "not enough values to unpack (expected 4, got {})", + f.len() + )), + _ => Err(format!( + "too many values to unpack (expected 4, got {})", + f.len() + )), + } +} + +/// The variants or severity cell: `.` or entries separated by `;`, each +/// `[kind ]chrom:pos:ref:alt name`. +pub fn parse_alleles(text: Option<&str>, kinds: bool) -> Result, String> { + let mut out = IndexMap::new(); + for e in text.filter(|t| !t.is_empty()).unwrap_or(".").split(';') { + let e = pycompat::text::strip(e, None); + if e.is_empty() || e == "." { + continue; + } + let mut parts: Vec<&str> = e.splitn(if kinds { 3 } else { 2 }, ' ').collect(); + if kinds && parts.len() < 2 { + return Err(format!( + "severity entry needs a kind and a chrom:pos:ref:alt: {e}" + )); + } + let kind = if kinds { + let k = parts.remove(0); + if !SEVERITY_KINDS.contains(&k) { + return Err(format!( + "severity '{k}' is not one of {}: {e}", + SEVERITY_KINDS.join(", ") + )); + } + Some(k.to_owned()) + } else { + None + }; + let name = parts.get(1).copied().unwrap_or(parts[0]).to_owned(); + out.insert(allele_key(parts[0])?, Allele { kind, name }); + } + Ok(out) +} + +/// `{gene: rule}` of `acmg_sf.tsv`, in the file's order. +pub fn acmg_sf_rules(assets: &str) -> Result, String> { + let mut rules: IndexMap = IndexMap::new(); + for (n, r) in asset_rows(assets, "acmg_sf.tsv")?.iter().enumerate() { + let one = || -> Result<(String, Rule), String> { + let g = col(r, "gene")?.unwrap_or("None").to_owned(); + let report_if = col(r, "report_if")?; + let sv_cn = col(r, "sv_cn")?; + let ri_ok = report_if.is_some_and(|x| SF_REPORT_IF.contains(&x)); + if !ri_ok || !matches!(sv_cn, Some("yes" | "no")) { + return Err(format!( + "report_if must be one of {}, sv_cn yes or no", + SF_REPORT_IF.join(", ") + )); + } + let report_if = report_if.unwrap_or_default(); + if sv_cn == Some("yes") && !matches!(report_if, "any" | "het_female_too") { + return Err( + "sv_cn yes needs a rule that one variant meets (any, het_female_too)".into(), + ); + } + let mut regions = Vec::new(); + for e in col(r, "sv_regions")? + .filter(|x| !x.is_empty()) + .unwrap_or(".") + .split(';') + { + let e = pycompat::text::strip(e, None); + if e.is_empty() || e == "." { + continue; + } + let (loc, name) = e.split_once(' ').unwrap_or((e, "")); + let (c, span) = loc.split_once(':').unwrap_or((loc, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + regions.push(( + c.to_owned(), + crate::int(a)?, + crate::int(b)?, + if name.is_empty() { loc } else { name }.to_owned(), + )); + } + let variants = parse_alleles(col(r, "variants")?, false)?; + if report_if == "hom_variant" && variants.is_empty() { + return Err("report_if hom_variant needs its variants".into()); + } + if rules.contains_key(&g) { + return Err("the gene is on two rows".into()); + } + let inheritance = col(r, "inheritance")?; + let _ = inheritance; + let severity = parse_alleles(col(r, "severity")?, true)?; + let table_text = col(r, "table_text")?.unwrap_or("None").to_owned(); + let note = col(r, "note")? + .filter(|x| !matches!(*x, "" | ".")) + .map(str::to_owned); + Ok(( + g, + Rule { + report_if: report_if.to_owned(), + variants, + sv_cn: sv_cn == Some("yes"), + sv_regions: regions, + severity, + table_text, + note, + }, + )) + }; + let (g, rule) = one().map_err(|e| format!("acmg_sf.tsv row {}: {e}", n + 1))?; + rules.insert(g, rule); + } + if rules.is_empty() { + return Err("acmg_sf.tsv has no gene".into()); + } + Ok(rules) +} + +/// `{gene: row}` of `carrier_genes.tsv`; a gene on two rows keeps its +/// first place and its last row. +pub fn carrier_genes(assets: &str) -> Result, String> { + let mut out = IndexMap::new(); + for (n, r) in asset_rows(assets, "carrier_genes.tsv")?.iter().enumerate() { + let one = || -> Result<(String, CarrierGene), String> { + let g = col(r, "gene")?.unwrap_or("None").to_owned(); + let inheritance = col(r, "inheritance")?.unwrap_or("None").to_owned(); + col(r, "lists")?; + let condition = col(r, "condition")?.unwrap_or("None").to_owned(); + let severity = parse_alleles(r.get("severity").flatten(), true)?; + let chrom = col(r, "chrom")?.unwrap_or("None").to_owned(); + let span = (chrom, int(col(r, "start")?)?, int(col(r, "end")?)?); + Ok(( + g, + CarrierGene { + inheritance, + condition, + severity, + span, + }, + )) + }; + let (g, x) = one().map_err(|e| format!("carrier_genes.tsv row {}: {e}", n + 1))?; + out.insert(g, x); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn alleles_read_with_and_without_kinds() { + let v = parse_alleles(Some("chr6:10:G:A C282Y; chr6:20:C:G"), false).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!( + v[&("chr6".into(), 10, "G".into(), "A".into())].name, + "C282Y" + ); + assert_eq!( + v[&("chr6".into(), 20, "C".into(), "G".into())].name, + "chr6:20:C:G" + ); + let s = parse_alleles(Some("hypomorph chr1:5:A:T p.X"), true).unwrap(); + assert_eq!( + s.values().next().unwrap().kind.as_deref(), + Some("hypomorph") + ); + assert!(parse_alleles(Some("odd chr1:5:A:T"), true).is_err()); + assert!(parse_alleles(Some("chr1:5:A"), false).is_err()); + assert!(parse_alleles(None, true).unwrap().is_empty()); + } +} diff --git a/rust/crates/og-summary/src/collect/carrier.rs b/rust/crates/og-summary/src/collect/carrier.rs new file mode 100644 index 000000000..852e35bf3 --- /dev/null +++ b/rust/crates/og-summary/src/collect/carrier.rs @@ -0,0 +1,146 @@ +//! The sample's own carrier list: step 06's ClinVar hits in the genes of +//! `assets/carrier_genes.tsv`, one person, no partner needed. An allele the +//! list marks hypomorph or pseudodeficiency is set aside, never a carrier +//! finding on its own. The genes of autosomal recessive conditions that a +//! called deletion and a hit both reach are listed as two hits, and a hit +//! called homozygous inside a deletion is shown as hemizygous. + +use super::assets::carrier_genes; +use super::clinvar::{Floor, Hit, HitRecords, allele_copies, carrier_genotype, obj}; +use super::text::{self, gt_field}; +use super::two_hits::{DeletionInputs, Spans, Variant, gene_deletions, two_hits}; +use indexmap::IndexMap; +use pycompat::json::Value; + +/// The section's data: the list, or why it could not be made. +#[derive(Debug, Clone, PartialEq)] +pub struct Carrier(pub Value); + +impl Carrier { + /// `carrier_list_or_note`. An `Err` is the collector's note (a hit + /// whose POS is no integer). + pub fn read( + assets: &str, + hits: &Result, + deletions: &DeletionInputs, + ) -> Result { + let unavailable = |e: String| Carrier(obj([("unavailable", Value::Str(e))])); + let genes = match carrier_genes(assets) { + Ok(g) => g, + Err(e) => return Ok(unavailable(e)), + }; + let recs = match hits { + Ok((_, r)) => r, + Err(e) => return Ok(unavailable(e.clone())), + }; + let mut items: Vec> = Vec::new(); + // Which list each item is on: carriers (false) or set aside (true). + let mut aside: Vec = Vec::new(); + let mut variants: Vec<(Variant, usize)> = Vec::new(); + for r in recs.iter().filter(|r| r.len() >= 8) { + let row = Hit::parse(r, Floor::default())?; + let gt = text::gt_of(r).to_owned(); + let (copies, ploidy) = allele_copies(>); + let key = row.key(); + for g in row.gene.split(',') { + let Some(x) = genes.get(g) else { continue }; + let mut item = IndexMap::new(); + item.insert("gene".into(), Value::Str(g.to_owned())); + item.insert("variant".into(), Value::Str(row.variant())); + item.insert("genotype".into(), Value::Str(carrier_genotype(>, &row))); + item.insert("significance".into(), Value::Str(row.significance.clone())); + item.insert("stars".into(), Value::Int(row.stars.into())); + item.insert("inheritance".into(), Value::Str(x.inheritance.clone())); + item.insert("condition".into(), Value::Str(x.condition.clone())); + let sev = x.severity.get(&key); + if let Some(s) = sev { + item.insert( + "kind".into(), + s.kind.clone().map_or(Value::Null, Value::Str), + ); + item.insert("name".into(), Value::Str(s.name.clone())); + } + let genotype = match &item["genotype"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }; + items.push(item); + aside.push(sev.is_some()); + if sev.is_none_or(|s| s.kind.as_deref() == Some("hypomorph")) { + variants.push(( + Variant { + gene: g.to_owned(), + key: key.clone(), + chrom: row.chrom.clone(), + pos: row.pos, + text: row.variant(), + genotype, + copies, + ploidy, + gt: gt.clone(), + ps: gt_field(r, "PS"), + allele: sev.map(|s| s.name.clone()), + }, + items.len() - 1, + )); + } + } + } + let spans: Spans = genes + .iter() + .map(|(g, x)| (g.clone(), x.span.clone())) + .collect(); + let (dels, read, unread) = gene_deletions(deletions, &spans); + let vs: Vec = variants.iter().map(|(v, _)| v.clone()).collect(); + let found = two_hits(&vs, &dels); + for (v, at) in &variants { + if let Some(t) = found.hemi.get(&(v.gene.clone(), v.key.clone())) { + let item = &mut items[*at]; + item.insert("genotype".into(), Value::Str("hemizygous".into())); + item.insert( + "hemizygous".into(), + Value::Str(format!("called homozygous inside {t}")), + ); + } + } + let mut rows: Vec<(&String, _)> = found.entries.iter().collect(); + rows.sort_by(|a, b| a.0.cmp(b.0)); + let two: Vec = rows + .into_iter() + .filter(|(g, _)| genes[*g].inheritance == "AR") + .map(|(g, e)| { + let mut m = e.to_json(); + m.insert("condition".into(), Value::Str(genes[g].condition.clone())); + m.insert("inheritance".into(), Value::Str("AR".into())); + Value::Object(m) + }) + .collect(); + let (mut carriers, mut set_aside) = (Vec::new(), Vec::new()); + for (item, a) in items.into_iter().zip(aside) { + if a { + set_aside.push(Value::Object(item)); + } else { + carriers.push(Value::Object(item)); + } + } + let strs = |v: Vec| Value::Array(v.into_iter().map(Value::Str).collect()); + Ok(Carrier(obj([ + ("genes_on_list", Value::Int(genes.len() as i128)), + ("list", Value::Str("assets/carrier_genes.tsv".into())), + ("carriers", Value::Array(carriers)), + ("set_aside", Value::Array(set_aside)), + ( + "two_hits", + obj([ + ("genes", Value::Array(two)), + ("deletions_read", strs(read)), + ("deletions_not_read", strs(unread)), + ]), + ), + ]))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} diff --git a/rust/crates/og-summary/src/collect/clinical.rs b/rust/crates/og-summary/src/collect/clinical.rs new file mode 100644 index 000000000..6dc161381 --- /dev/null +++ b/rust/crates/og-summary/src/collect/clinical.rs @@ -0,0 +1,208 @@ +//! The clinical variant filter (step 23) with its ACMG secondary-findings +//! tier, and the variant prioritization (step 31, slivar) with its +//! compound-het pairs phased from the reads. +//! +//! The bash pipeline writes summary tables; the Nextflow report gets the +//! VCFs instead, so each section reads the table when there is one and +//! counts the VCF's records otherwise. + +use super::acmg::{self, ClinicalVcf}; +use super::clinvar::obj; +use super::text::{self, cell, digits}; +use indexmap::IndexMap; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use std::collections::BTreeSet; + +/// The trans pairs the report lists by name; the rest are counted. +pub const COMPHET_TRANS_SHOWN: usize = 20; + +fn rows_or_note(path: &str) -> Result, String> { + text::rows(path).map_err(|e| e.note()) +} + +fn int(n: usize) -> Value { + Value::Int(n as i128) +} + +/// The data of the clinical section. +#[derive(Debug, Clone, PartialEq)] +pub struct Clinical(pub Value); + +impl Clinical { + /// `sec_clinical`: `table` is step 23's summary table, `vcf` the + /// clinical VCF read once (both full paths). + pub fn read( + table: Option<&str>, + vcf: Option<&Result>, + tier: &acmg::Inputs, + ) -> Result { + let Some(table) = table else { + // The Nextflow report: the record count only (one record per + // variant, as in the table). + let v = match vcf { + Some(Ok(v)) => v, + Some(Err(e)) => return Err(e.clone()), + None => return Ok(Clinical(Value::Null)), + }; + return Ok(Clinical(obj([ + ("variants", int(v.records)), + ("acmg_sf", acmg::acmg_sf(tier)?), + ]))); + }; + let rows = rows_or_note(table)?; + let mut by: IndexMap = IndexMap::new(); + for r in &rows { + let k = cell(r, "IMPACT").filter(|v| !v.is_empty()).unwrap_or("."); + *by.entry(k.to_owned()).or_default() += 1; + } + let genes: BTreeSet<&str> = rows + .iter() + .filter_map(|r| cell(r, "GENE")) + .filter(|g| !matches!(*g, "" | ".")) + .collect(); + let mut m: IndexMap = IndexMap::new(); + m.insert("variants".into(), int(rows.len())); + m.insert( + "by_impact".into(), + Value::Object(by.into_iter().map(|(k, v)| (k, int(v))).collect()), + ); + m.insert("genes".into(), int(genes.len())); + m.insert("acmg_sf".into(), acmg::acmg_sf(tier)?); + // Step 23's tier and call check columns; an older table has neither. + if let Some(r) = rows.first() { + if r.get("FREQ_TIER").is_some() { + let n = rows + .iter() + .filter(|r| cell(r, "FREQ_TIER") == Some("unknown")) + .count(); + m.insert("frequency_unknown".into(), int(n)); + } + if r.get("CALL_CHECK").is_some() { + let n = rows + .iter() + .filter(|r| cell(r, "CALL_CHECK") == Some("weak")) + .count(); + m.insert("weak_calls".into(), int(n)); + } + } + Ok(Clinical(Value::Object(m))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} + +/// The data of the slivar section. +#[derive(Debug, Clone, PartialEq)] +pub struct Slivar(pub Value); + +impl Slivar { + /// `sec_slivar` with step 31's summary table (`table`) or, for the + /// Nextflow report, its prioritized VCF (`vcf`); full paths. + pub fn read(table: Option<&str>, vcf: Option<&str>, s: &str) -> Result { + let mut m: IndexMap = IndexMap::new(); + let Some(p) = table else { + let Some(v) = vcf else { + return Ok(Slivar(Value::Null)); + }; + let n = text::vcf_records(v).map_err(|e| e.unreadable(v))?.len(); + m.insert("prioritized".into(), int(n)); + comphet_phase( + &crate::join(text::dirname(v), &format!("{s}_compound_het_pairs.tsv")), + &mut m, + )?; + return Ok(Slivar(Value::Object(m))); + }; + let rows = rows_or_note(p)?; + m.insert("prioritized".into(), int(rows.len())); + // Step 31's frequency tier: prioritized for the compound-het search, not rare. + if rows.first().is_some_and(|r| r.get("FREQ_TIER").is_some()) { + let n = rows + .iter() + .filter(|r| cell(r, "FREQ_TIER") == Some("unknown")) + .count(); + m.insert("frequency_unknown".into(), int(n)); + } + let dir = text::dirname(p); + let ch = crate::join(dir, &format!("{s}_compound_hets.tsv")); + if crate::is_file(&ch) { + let c = rows_or_note(&ch)?; + let genes: std::collections::HashSet<&str> = c + .iter() + .filter_map(|r| cell(r, "GENE")) + .filter(|g| !g.is_empty()) + .collect(); + m.insert("compound_het_variants".into(), int(c.len())); + m.insert("compound_het_genes".into(), int(genes.len())); + } + comphet_phase( + &crate::join(dir, &format!("{s}_compound_het_pairs.tsv")), + &mut m, + )?; + Ok(Slivar(Value::Object(m))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} + +/// Step 31's pairs table: the pairs by phase from the reads, and the trans +/// ones (both copies of the gene hit) by name. A run from before the table +/// adds none of these keys. +fn comphet_phase(path: &str, m: &mut IndexMap) -> Result<(), String> { + if !crate::is_file(path) { + return Ok(()); + } + let rows = rows_or_note(path)?; + let phase = obj(["trans", "cis", "unknown"].map(|k| { + ( + k, + int(rows.iter().filter(|r| cell(r, "PHASE") == Some(k)).count()), + ) + })); + let py = |r: &DictRow, k: &str| cell(r, k).unwrap_or("None").to_owned(); + let n = |r: &DictRow, k: &str| digits(cell(r, k)).unwrap_or(0); + let trans: Vec = rows + .iter() + .filter(|r| cell(r, "PHASE") == Some("trans")) + .take(COMPHET_TRANS_SHOWN) + .map(|r| { + let gene = match r.get("GENE") { + None => Value::Str(String::new()), + Some(v) => v.map_or(Value::Null, |g| Value::Str(g.to_owned())), + }; + obj([ + ("gene", gene), + ( + "variants", + Value::Str(format!( + "{}:{} {}>{} + {}:{} {}>{}", + py(r, "CHROM"), + py(r, "POS_1"), + py(r, "REF_1"), + py(r, "ALT_1"), + py(r, "CHROM"), + py(r, "POS_2"), + py(r, "REF_2"), + py(r, "ALT_2") + )), + ), + ( + "reads", + Value::Str(format!( + "{} of {}", + py(r, "TRANS_READS"), + n(r, "TRANS_READS") + n(r, "CIS_READS") + )), + ), + ]) + }) + .collect(); + m.insert("compound_het_pairs".into(), int(rows.len())); + m.insert("compound_het_phase".into(), phase); + m.insert("compound_het_trans".into(), Value::Array(trans)); + Ok(()) +} diff --git a/rust/crates/og-summary/src/collect/clinvar.rs b/rust/crates/og-summary/src/collect/clinvar.rs new file mode 100644 index 000000000..7350c5a6b --- /dev/null +++ b/rust/crates/og-summary/src/collect/clinvar.rs @@ -0,0 +1,799 @@ +//! The ClinVar screen (step 06): each P/LP hit with its review stars, its +//! call quality against step 06's floor, gnomAD AF and rsIDs from step 13's +//! VEP output, and the cDNA read check, the cDNA scan and the always-report +//! deep intronic sites when step 06 wrote them. +//! +//! The hits VCF is small and three sections read it (this one, the ACMG +//! tier of the clinical section and the carrier list), so it is read once +//! ([`HitsVcf`]) and each section parses the rows it needs. + +use super::text::{self, ReadError, digits, format_map, gt_of, info_map, py_max}; +use indexmap::IndexMap; +use og_clinvar::{ReviewStatus, combined_note, half_call, zygosity}; +use pycompat::json::Value; +use std::collections::HashMap; + +/// gnomAD AF above which a P/LP hit is marked common. +pub const COMMON_AF: f64 = 0.01; + +/// Step 06's call floor: a hit whose GQ or DP is below it is low confidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Floor { + pub gq: i128, + pub dp: i128, +} + +impl Default for Floor { + fn default() -> Self { + Floor { gq: 30, dp: 10 } + } +} + +impl Floor { + fn to_json(self) -> Value { + obj([("GQ", Value::Int(self.gq)), ("DP", Value::Int(self.dp))]) + } +} + +/// `(GQ|DP)=(\d+)` over a header line: every match, the last one wins. +pub fn floor_values(line: &str, word_start: bool, floor: &mut Floor) { + let b = line.as_bytes(); + for key in ["GQ=", "DP="] { + let mut from = 0; + while let Some(i) = line[from..].find(key).map(|i| i + from) { + from = i + 1; + if word_start && i > 0 && (b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_') { + continue; + } + let rest = &line[i + 3..]; + let n = rest.bytes().take_while(u8::is_ascii_digit).count(); + if n > 0 + && let Ok(v) = crate::int(&rest[..n]) + { + // Python takes the matches in text order; with one key per + // pass, the last of each key is the one that stays. + if key == "GQ=" { + floor.gq = v; + } else { + floor.dp = v; + } + } + } + } +} + +/// A variant as `(chrom, pos, ref, alt)`. +pub type VarKey = (String, i128, String, String); +/// The hits VCF's call floor and data lines. +pub type HitRecords = (Floor, Vec>); + +/// The hits VCF, read once: its call floor and its data lines. +#[derive(Debug, Clone)] +pub struct HitsVcf { + /// The path (full) and the path relative to the sample folder. + pub path: String, + pub rel: String, + /// The floor and the records, or the note of a file that cannot be read. + pub read: Result, +} + +impl HitsVcf { + pub fn read(path: String, rel: String) -> HitsVcf { + let read = text::lines(&path) + .map(|lines| { + let mut floor = Floor::default(); + for l in lines.iter().take_while(|l| l.starts_with("##")) { + if l.starts_with("##PGP_CALL_FLOOR=") { + floor_values(l, false, &mut floor); + } + } + let recs = lines + .iter() + .filter(|l| !l.starts_with('#')) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + (floor, recs) + }) + .map_err(|e: ReadError| e.unreadable(&path)); + HitsVcf { path, rel, read } + } +} + +/// One hit as `clinvar_row` reads it. +#[derive(Debug, Clone, PartialEq)] +pub struct Hit { + pub stars: u8, + pub chrom: String, + pub pos: i128, + pub reference: String, + pub alt: String, + pub genotype: String, + pub gene: String, + pub significance: String, + pub review_status: String, + pub combined_allele: String, + pub dp: Option, + pub gq: Option, + pub low_confidence: bool, + pub frequency: Option, + pub cdna: Option, +} + +/// gnomAD and rsIDs of one hit from VEP. +#[derive(Debug, Clone, PartialEq)] +pub struct Frequency { + pub exome: Option, + pub genome: Option, + pub rsids: Vec, +} + +impl Hit { + /// `clinvar_row`: an error is the collector's note (a POS that is no + /// integer). + pub fn parse(r: &[String], floor: Floor) -> Result { + let info = if r.len() > 7 { + info_map(&r[7]) + } else { + HashMap::new() + }; + let get = |k: &str| info.get(k).copied().unwrap_or(""); + let rev = get("CLNREVSTAT"); + let genes: Vec<&str> = get("GENEINFO") + .split('|') + .filter(|g| !g.is_empty()) + .map(|g| g.split(':').next().unwrap_or("")) + .collect(); + let gt = gt_of(r); + let fmt = if r.len() > 9 { + format_map(&r[8], &r[9]) + } else { + HashMap::new() + }; + let dp = digits(fmt.get("DP").copied()); + let gq = digits(fmt.get("GQ").copied()); + let pos = text::pos(r)?; + let s = |v: &str| v.replace('_', " "); + let or_dot = |v: String| if v.is_empty() { ".".to_owned() } else { v }; + Ok(Hit { + stars: ReviewStatus::parse(rev.as_bytes()).stars(), + chrom: r[0].clone(), + pos, + reference: r[3].clone(), + alt: r[4].clone(), + genotype: String::from_utf8_lossy(&zygosity(gt.as_bytes())).into_owned(), + gene: or_dot(genes.join(",")), + significance: or_dot(s(get("CLNSIG"))), + review_status: or_dot(s(rev)), + combined_allele: String::from_utf8_lossy(&combined_note( + get("PGP_JOINED_FROM").as_bytes(), + get("PGP_PART_OF").as_bytes(), + get("PGP_ONE_ALLELE").as_bytes(), + gt.as_bytes(), + )) + .into_owned(), + dp, + gq, + low_confidence: gq.is_some_and(|v| v < floor.gq) || dp.is_some_and(|v| v < floor.dp), + frequency: None, + cdna: None, + }) + } + + pub fn key(&self) -> (String, i128, String, String) { + ( + self.chrom.clone(), + self.pos, + self.reference.clone(), + self.alt.clone(), + ) + } + + /// `chrom:pos ref>alt`. + pub fn variant(&self) -> String { + format!( + "{}:{} {}>{}", + self.chrom, self.pos, self.reference, self.alt + ) + } + + fn to_json(&self) -> Value { + let mut m = IndexMap::new(); + let put = |m: &mut IndexMap, k: &str, v: Value| { + m.insert(k.to_owned(), v); + }; + put(&mut m, "stars", Value::Int(self.stars.into())); + put(&mut m, "chrom", Value::Str(self.chrom.clone())); + put(&mut m, "pos", Value::Int(self.pos)); + put(&mut m, "ref", Value::Str(self.reference.clone())); + put(&mut m, "alt", Value::Str(self.alt.clone())); + put(&mut m, "genotype", Value::Str(self.genotype.clone())); + put(&mut m, "gene", Value::Str(self.gene.clone())); + put( + &mut m, + "significance", + Value::Str(self.significance.clone()), + ); + put( + &mut m, + "review_status", + Value::Str(self.review_status.clone()), + ); + put( + &mut m, + "combined_allele", + Value::Str(self.combined_allele.clone()), + ); + put(&mut m, "dp", opt_int(self.dp)); + put(&mut m, "gq", opt_int(self.gq)); + put(&mut m, "low_confidence", Value::Bool(self.low_confidence)); + if let Some(f) = &self.frequency { + let af = py_max(f.exome.into_iter().chain(f.genome)); + put(&mut m, "gnomad_exome_af", opt_f(f.exome)); + put(&mut m, "gnomad_genome_af", opt_f(f.genome)); + put( + &mut m, + "rsids", + Value::Array(f.rsids.iter().cloned().map(Value::Str).collect()), + ); + put(&mut m, "af", opt_f(af)); + put( + &mut m, + "common", + Value::Bool(af.is_some_and(|a| a > COMMON_AF)), + ); + } + if let Some(c) = &self.cdna { + put(&mut m, "cdna", c.clone()); + } + Value::Object(m) + } +} + +/// A hit's genotype as the carrier list words it. +pub fn carrier_genotype(gt: &str, hit: &Hit) -> String { + if half_call(gt.as_bytes()) { + hit.genotype.clone() + } else { + let (copies, ploidy) = allele_copies(gt); + zygosity_word(copies, ploidy).to_owned() + } +} + +/// `(alt copies, ploidy)` of a GT: a missing allele is a copy not called, +/// not a missing copy, so the half of a split call is never hemizygous. +pub fn allele_copies(gt: &str) -> (usize, usize) { + let a: Vec<&str> = gt.split(['/', '|']).filter(|x| !x.is_empty()).collect(); + ( + a.iter().filter(|x| !matches!(**x, "0" | ".")).count(), + a.len(), + ) +} + +pub fn zygosity_word(copies: usize, ploidy: usize) -> &'static str { + if copies == 0 { + "unknown" + } else if ploidy == 1 { + "hemizygous" + } else if copies >= 2 { + "hom" + } else { + "het" + } +} + +pub fn obj<'a>(pairs: impl IntoIterator) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +pub fn opt_int(x: Option) -> Value { + x.map_or(Value::Null, Value::Int) +} + +pub fn opt_f(x: Option) -> Value { + x.map_or(Value::Null, Value::Float) +} + +pub fn opt_s(x: Option<&str>) -> Value { + x.map_or(Value::Null, |s| Value::Str(s.to_owned())) +} + +// --- the cDNA read check, the scan, the deep intronic sites ---------------------- + +/// A row key of a cDNA check table: chrom and the alleles may be missing. +pub type CdnaKey = (Option, i128, Option, Option); + +/// `clinvar_cdna_checks`: the read check of each row, keyed by variant. +pub fn cdna_checks(path: &str) -> Result, String> { + let rows = text::rows(path).map_err(|e| match e { + ReadError::Os(_) => e.unreadable(path), + ReadError::Other(n) => n, + })?; + let mut out = HashMap::new(); + for r in &rows { + let Some(pos) = digits(text::cell(r, "pos")) else { + continue; + }; + let c = |k: &str| text::cell(r, k).map(str::to_owned); + let or = |k: &str, d: &str| { + Value::Str( + text::cell(r, k) + .filter(|v| !v.is_empty()) + .unwrap_or(d) + .to_owned(), + ) + }; + let n = |k: &str| opt_int(digits(text::cell(r, k))); + out.insert( + (c("chrom"), pos, c("ref"), c("alt")), + obj([ + ("check", or("cdna_check", "not_checked")), + ("note", or("cdna_note", "")), + ("alt_reads", n("alt_reads")), + ("alt_clipped", n("alt_clipped")), + ("ref_reads", n("ref_reads")), + ("ref_clipped", n("ref_clipped")), + ( + "construct", + opt_s(text::cell(r, "construct_site").filter(|v| !v.is_empty())), + ), + ]), + ); + } + Ok(out) +} + +/// The check of a variant the table does not hold. +pub fn not_in_table() -> Value { + obj([ + ("check", Value::Str("not_checked".into())), + ("note", Value::Str("not in the read-check table".into())), + ("alt_reads", Value::Null), + ("alt_clipped", Value::Null), + ("ref_reads", Value::Null), + ("ref_clipped", Value::Null), + ]) +} + +pub fn cdna_key(chrom: &str, pos: i128, r: &str, a: &str) -> CdnaKey { + ( + Some(chrom.to_owned()), + pos, + Some(r.to_owned()), + Some(a.to_owned()), + ) +} + +pub fn check_word(c: &Value) -> Option<&str> { + match c { + Value::Object(m) => match m.get("check") { + Some(Value::Str(s)) => Some(s), + _ => None, + }, + _ => None, + } +} + +fn has_construct(c: &Value) -> bool { + matches!(c, Value::Object(m) if m.get("construct").is_some_and(pycompat::value::truthy)) +} + +/// `clinvar_cdna_scan`: the scan's note and loci. +fn cdna_scan(path: &str) -> Result { + let lines = text::lines(path).map_err(|e| e.unreadable(path))?; + let note = lines + .first() + .and_then(|l| l.strip_prefix("# cdna_scan: ")) + .unwrap_or("") + .to_owned(); + let body = lines + .iter() + .filter(|l| !l.starts_with('#')) + .map(|l| Ok::<_, std::io::Error>(l.clone())); + let mut loci = Vec::new(); + for r in pycompat::csv::DictReader::new(body, '\t', pycompat::csv::Quoting::Minimal) { + let r = r.map_err(|e| format!("{}: {e}", e.exception()))?; + let Some(start) = digits(text::cell(&r, "start")) else { + continue; + }; + let int0 = |k: &str| -> Result { + match text::cell(&r, k).filter(|v| !v.is_empty()) { + None => Ok(Value::Int(0)), + Some(v) => crate::int(v) + .map(Value::Int) + .map_err(|e| format!("ValueError: {e}")), + } + }; + loci.push(obj([ + ("chrom", opt_s(text::cell(&r, "chrom"))), + ("start", Value::Int(start)), + ("end", int0("end")?), + ("junctions", int0("junctions")?), + ("reads", int0("reads")?), + ( + "junction_list", + Value::Str(text::cell_or(&r, "junction_list").to_owned()), + ), + ])); + } + Ok(obj([ + ("note", Value::Str(note)), + ("loci", Value::Array(loci)), + ])) +} + +/// `clinvar_deep_intronic`: one row per always-report site. +fn deep_intronic(path: &str) -> Result { + let rows = text::rows(path).map_err(|e| match e { + ReadError::Os(_) => e.unreadable(path), + ReadError::Other(n) => n, + })?; + let mut out = Vec::new(); + for r in &rows { + let Some(pos) = digits(text::cell(r, "pos")) else { + continue; + }; + let or = |k: &str, d: &str| { + Value::Str( + text::cell(r, k) + .filter(|v| !v.is_empty()) + .unwrap_or(d) + .to_owned(), + ) + }; + out.push(obj([ + ("chrom", opt_s(text::cell(r, "chrom"))), + ("pos", Value::Int(pos)), + ("ref", opt_s(text::cell(r, "ref"))), + ("alt", opt_s(text::cell(r, "alt"))), + ("gene", or("gene", ".")), + ("hgvs", or("hgvs", "")), + ("clinvar_id", or("clinvar_id", "")), + ("call", or("call", "no call")), + ("detail", or("detail", "")), + ])); + } + Ok(Value::Array(out)) +} + +// --- gnomAD AF and rsIDs from step 13's VEP output ----------------------------------- + +/// Where the frequencies came from, or why there are none. +#[derive(Debug, Clone, PartialEq)] +pub struct FrequencySource { + pub source: Option, + pub note: Option, +} + +/// The VEP output's frequencies of every hit, in one pass per contig over +/// the hits sorted by position: one indexed query per hit position, served +/// by the reader's block cache when hits share a block. +pub fn vep_frequencies(vep: &str, keys: &[VarKey]) -> Result, String> { + let mut fmt: Option> = None; + { + let mut lines = og_io::Lines::new( + og_io::open(vep).map_err(|e| crate::os_error(&e, vep))?, + og_io::Utf8::Replace, + ); + let mut line = String::new(); + while lines.read_line(&mut line).map_err(|e| e.to_string())? { + if !line.starts_with('#') { + break; + } + if line.starts_with("##INFO= = keys.iter().collect(); + sorted.sort(); + sorted.dedup(); + let mut out = HashMap::new(); + if sorted.is_empty() { + return Ok(out); + } + let mut reader = + og_io::tabix::Reader::with_index(vep, format!("{vep}.tbi")).map_err(|e| e.to_string())?; + for key in sorted { + let (chrom, pos, reference, alt) = key; + let Ok(p) = u64::try_from(*pos) else { continue }; + if p == 0 { + continue; + } + let lines: Vec> = reader + .query(chrom, p - 1, p) + .collect::>() + .map_err(|e| e.to_string())?; + for raw in lines { + let line = String::from_utf8_lossy(&raw); + let r: Vec<&str> = line.split('\t').collect(); + if r.len() < 3 || r[1].parse::().ok() != Some(*pos) { + continue; + } + let alts: Vec<&str> = if r.len() > 7 { + r[4].split(',').collect() + } else { + Vec::new() + }; + if r.get(3) != Some(&reference.as_str()) || !alts.contains(&alt.as_str()) { + continue; + } + // VEP's Allele drops the base an indel's alleles share. + let first = |s: &str| s.chars().next(); + let snv = reference.chars().count() == 1 && alts.iter().all(|a| a.chars().count() == 1); + let allele = if snv || alts.iter().any(|a| first(a) != first(reference)) { + alt.clone() + } else { + let rest: String = alt.chars().skip(1).collect(); + if rest.is_empty() { + "-".to_owned() + } else { + rest + } + }; + let mut got = [None, None]; + let mut ids: Vec = Vec::new(); + for tr in info_map(r[7]).get("CSQ").copied().unwrap_or("").split(',') { + let v: Vec<&str> = tr.split('|').collect(); + if v.len() != fmt.len() || (alts.len() > 1 && v[allele_ix] != allele) { + continue; + } + for (i, c) in cols.iter().enumerate() { + if let Some(c) = c + && got[i].is_none() + { + got[i] = py_max( + v[*c] + .split('&') + .filter_map(|x| pycompat::num::parse_float(x).ok()), + ); + } + } + if let Some(e) = ev { + for x in v[e].split('&') { + if is_rsid(x) { + let x = x.to_lowercase(); + if !ids.contains(&x) { + ids.push(x); + } + } + } + } + } + ids.sort_by_key(|x| x[2..].parse::().unwrap_or(u128::MAX)); + out.insert( + key.clone(), + Frequency { + exome: got[0], + genome: got[1], + rsids: ids, + }, + ); + } + } + Ok(out) +} + +/// `(?i)rs\d+`, the whole field. +fn is_rsid(x: &str) -> bool { + let b = x.as_bytes(); + b.len() > 2 && b[..2].eq_ignore_ascii_case(b"rs") && b[2..].iter().all(u8::is_ascii_digit) +} + +// --- the section -------------------------------------------------------------------- + +/// The data of the ClinVar section. +#[derive(Debug, Clone, PartialEq)] +pub struct ClinVar { + pub hits: Vec, + pub floor: Floor, + pub frequency: FrequencySource, + pub cdna: bool, + pub cdna_scan: Option, + pub deep_intronic: Option, +} + +impl ClinVar { + /// `sec_clinvar` over the hits read once. `side(rel)` is the file of + /// another of step 06's or 13's outputs, found in the folder. + pub fn read( + hits: &HitsVcf, + dir: &str, + side: &dyn Fn(&str) -> Option<(String, String)>, + s: &str, + ) -> Result { + let (floor, recs) = hits.read.clone()?; + let mut rows = Vec::new(); + for r in recs.iter().filter(|r| r.len() >= 8) { + rows.push(Hit::parse(r, floor)?); + } + rows.sort_by(|a, b| { + b.stars + .cmp(&a.stars) + .then_with(|| a.chrom.cmp(&b.chrom)) + .then_with(|| a.pos.cmp(&b.pos)) + }); + let frequency = frequencies(dir, side, s, &mut rows); + let mut cdna = false; + if let Some((p, _)) = side(&format!("clinvar/{s}_clinvar_cdna_check.tsv")) { + let table = cdna_checks(&p)?; + for x in &mut rows { + x.cdna = Some( + table + .get(&cdna_key(&x.chrom, x.pos, &x.reference, &x.alt)) + .cloned() + .unwrap_or_else(not_in_table), + ); + } + cdna = true; + } + let cdna_scan = match side(&format!("clinvar/{s}_clinvar_cdna_scan.tsv")) { + Some((p, _)) => Some(cdna_scan(&p)?), + None => None, + }; + let deep_intronic = match side(&format!("clinvar/{s}_clinvar_deep_intronic.tsv")) { + Some((p, _)) => Some(deep_intronic(&p)?), + None => None, + }; + Ok(ClinVar { + hits: rows, + floor, + frequency, + cdna, + cdna_scan, + deep_intronic, + }) + } + + pub fn to_json(&self) -> Value { + let sure: Vec<&Hit> = self.hits.iter().filter(|x| !x.low_confidence).collect(); + let by = (0..=4u8) + .rev() + .map(|k| { + ( + k.to_string(), + Value::Int(sure.iter().filter(|x| x.stars == k).count() as i128), + ) + }) + .collect(); + let mut m: IndexMap = IndexMap::new(); + m.insert("count".into(), Value::Int(sure.len() as i128)); + m.insert("by_stars".into(), Value::Object(by)); + m.insert( + "hits".into(), + Value::Array(self.hits.iter().map(Hit::to_json).collect()), + ); + m.insert( + "low_confidence".into(), + Value::Int((self.hits.len() - sure.len()) as i128), + ); + m.insert("call_floor".into(), self.floor.to_json()); + m.insert( + "frequency".into(), + obj([ + ("source", opt_s(self.frequency.source.as_deref())), + ("note", opt_s(self.frequency.note.as_deref())), + ("common_af", Value::Float(COMMON_AF)), + ]), + ); + if self.cdna { + let checks: Vec<&Value> = self.hits.iter().filter_map(|x| x.cdna.as_ref()).collect(); + m.insert( + "cdna_flagged".into(), + Value::Int( + checks + .iter() + .filter(|c| check_word(c) == Some("flagged")) + .count() as i128, + ), + ); + m.insert( + "construct_sites".into(), + Value::Int(checks.iter().filter(|c| has_construct(c)).count() as i128), + ); + } + if let Some(v) = &self.cdna_scan { + m.insert("cdna_scan".into(), v.clone()); + } + if let Some(v) = &self.deep_intronic { + m.insert("deep_intronic".into(), v.clone()); + } + Value::Object(m) + } +} + +/// `clinvar_frequencies`: puts the frequencies on each hit and says where +/// they came from. +fn frequencies( + dir: &str, + side: &dyn Fn(&str) -> Option<(String, String)>, + s: &str, + rows: &mut [Hit], +) -> FrequencySource { + let vep = side(&format!("vep/{s}_vep.vcf.gz")); + let _ = dir; + let mut note = None; + let mut found = HashMap::new(); + match &vep { + None => note = Some("no VEP output (step 13) in this folder".to_owned()), + Some((p, rel)) if !std::path::Path::new(&format!("{p}.tbi")).exists() => { + note = Some(format!("{rel} has no .tbi index")); + } + Some((p, rel)) => { + let keys: Vec<_> = rows.iter().map(Hit::key).collect(); + match vep_frequencies(p, &keys) { + Ok(f) => found = f, + Err(e) => note = Some(format!("{rel} could not be read: {e}")), + } + } + } + for x in rows.iter_mut() { + x.frequency = Some(found.get(&x.key()).cloned().unwrap_or(Frequency { + exome: None, + genome: None, + rsids: Vec::new(), + })); + } + FrequencySource { + source: if note.is_none() { + vep.map(|(_, rel)| rel) + } else { + None + }, + note, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn floor_takes_the_last_value_of_each_key() { + let mut f = Floor::default(); + floor_values("##PGP_CALL_FLOOR=GQ=20,DP=8,GQ=25", false, &mut f); + assert_eq!(f, Floor { gq: 25, dp: 8 }); + let mut f = Floor::default(); + floor_values("##PGP_CALL_FLOOR=XGQ=5,DP=7", true, &mut f); + assert_eq!(f, Floor { gq: 30, dp: 7 }); + } + + #[test] + fn copies_and_words() { + assert_eq!(allele_copies("0/1"), (1, 2)); + assert_eq!(allele_copies("1|1"), (2, 2)); + assert_eq!(allele_copies("1"), (1, 1)); + assert_eq!(allele_copies("1/."), (1, 2)); + assert_eq!(allele_copies(""), (0, 0)); + assert_eq!(zygosity_word(1, 1), "hemizygous"); + assert_eq!(zygosity_word(0, 2), "unknown"); + } + + #[test] + fn rsid_fields() { + assert!(is_rsid("rs123")); + assert!(is_rsid("RS9")); + assert!(!is_rsid("rs")); + assert!(!is_rsid("COSV1")); + } +} diff --git a/rust/crates/og-summary/src/collect/cpsr.rs b/rust/crates/og-summary/src/collect/cpsr.rs new file mode 100644 index 000000000..ffd178d9c --- /dev/null +++ b/rust/crates/og-summary/src/collect/cpsr.rs @@ -0,0 +1,264 @@ +//! Cancer predisposition (step 17, CPSR): the class counts, and CPSR's own +//! class beside the final class and ClinVar's, so a final P/LP class that +//! ClinVar set over CPSR's own benign call is not hidden. + +use super::clinvar::{CdnaKey, Floor, cdna_checks, check_word, not_in_table, opt_int, opt_s}; +use super::text::{self, cell, digits, stripped}; +use indexmap::IndexMap; +use og_bam::cdna::{cpsr_variant, norm_class}; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use std::collections::HashMap; + +const PLP: [&str; 2] = ["pathogenic", "likely pathogenic"]; +const CLASS_COLUMNS: [&str; 3] = [ + "CLASSIFICATION", + "FINAL_CLASSIFICATION", + "CPSR_CLASSIFICATION", +]; + +#[derive(Debug, Clone, PartialEq)] +pub struct Cpsr { + pub html_report: bool, + /// The class column and the counts, when the table was read. + pub table: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Table { + /// `None`: the table has no class column. + pub column: Option, + pub classification: IndexMap, + pub compare: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Compare { + pub plp: Vec, + pub disagreements: Vec, + pub own_benign: usize, + pub own: Option, + pub floor_dp: i128, + pub cdna: bool, +} + +/// Where the inputs beside the CPSR table are. +pub struct Inputs { + pub html: Option, + /// The classification table (full path). + pub tsv: Option, + /// Step 06's floor, the default without a hits VCF; an error is the + /// note of a hits VCF that cannot be read. + pub floor: Result, + /// Step 17's cDNA read check (full path). + pub cdna: Option, +} + +impl Cpsr { + pub fn read(i: &Inputs) -> Result { + let html_report = i.html.is_some(); + let Some(tsv) = &i.tsv else { + return Ok(Cpsr { + html_report, + table: None, + }); + }; + let rows = text::rows(tsv).map_err(|e| match e { + text::ReadError::Os(_) => e.unreadable(tsv), + text::ReadError::Other(n) => n, + })?; + // `rows[0].keys()`: the header, and the None key of a long first row. + let cols: Vec<&str> = rows + .first() + .map(|r| r.iter().map(|(k, _)| k).collect()) + .unwrap_or_default(); + let col = CLASS_COLUMNS.iter().find(|c| cols.contains(c)).copied(); + let Some(col) = col else { + return Ok(Cpsr { + html_report, + table: Some(Table { + column: None, + classification: IndexMap::new(), + compare: None, + }), + }); + }; + let mut counts: IndexMap = IndexMap::new(); + for r in &rows { + let k = cell(r, col).filter(|v| !v.is_empty()).unwrap_or("."); + *counts.entry(k.to_owned()).or_default() += 1; + } + counts.sort_by(|_, a, _, b| b.cmp(a)); + let compare = compare(i, &rows, &cols, col)?; + Ok(Cpsr { + html_report, + table: Some(Table { + column: Some(col.to_owned()), + classification: counts, + compare: Some(compare), + }), + }) + } + + pub fn to_json(&self) -> Value { + let mut m: IndexMap = IndexMap::new(); + m.insert("html_report".into(), Value::Bool(self.html_report)); + let Some(t) = &self.table else { + return Value::Object(m); + }; + m.insert("classification_column".into(), opt_s(t.column.as_deref())); + if t.column.is_some() { + m.insert( + "classification".into(), + Value::Object( + t.classification + .iter() + .map(|(k, v)| (k.clone(), Value::Int(*v as i128))) + .collect(), + ), + ); + } + if let Some(c) = &t.compare { + let field = |x: &Value, k: &str| match x { + Value::Object(m) => m.get(k).cloned(), + _ => None, + }; + let low = c + .plp + .iter() + .filter(|x| field(x, "low_confidence") == Some(Value::Bool(true))) + .count(); + m.insert("plp".into(), Value::Array(c.plp.clone())); + m.insert( + "disagreements".into(), + Value::Array(c.disagreements.clone()), + ); + m.insert( + "final_plp_own_benign".into(), + Value::Int(c.own_benign as i128), + ); + m.insert("own_class_column".into(), opt_s(c.own.as_deref())); + m.insert("low_confidence".into(), Value::Int(low as i128)); + m.insert("call_floor_dp".into(), Value::Int(c.floor_dp)); + if c.cdna { + let flagged = c + .plp + .iter() + .filter(|x| field(x, "cdna").as_ref().and_then(check_word) == Some("flagged")) + .count(); + m.insert("cdna_flagged".into(), Value::Int(flagged as i128)); + } + } + Value::Object(m) + } +} + +/// `cpsr_compare`. +fn compare(i: &Inputs, rows: &[DictRow], cols: &[&str], col: &str) -> Result { + let own = (cols.contains(&"CPSR_CLASSIFICATION") && col != "CPSR_CLASSIFICATION") + .then_some("CPSR_CLASSIFICATION"); + let floor = i.floor.clone()?; + let cdna: Option> = match &i.cdna { + Some(p) => Some(cdna_checks(p)?), + None => None, + }; + let has_clinvar_col = cols.contains(&"CLINVAR_CLASSIFICATION"); + let (mut plp, mut disagree, mut own_benign) = (Vec::new(), Vec::new(), 0); + for r in rows { + let final_ = norm_class(cell(r, col)); + let mine = own.map(|o| norm_class(cell(r, o))).unwrap_or_default(); + let authority = stripped(r, "ASSERTION_AUTHORITY"); + let clinvar_raw = if has_clinvar_col { + stripped(r, "CLINVAR_CLASSIFICATION") + } else if authority.to_lowercase() == "clinvar" { + stripped(r, col) + } else { + "" + }; + let clinvar = norm_class(Some(clinvar_raw)); + let is = |c: &str| PLP.contains(&c); + if is(&final_) && matches!(mine.as_str(), "benign" | "likely benign") { + own_benign += 1; + } + let is_plp = is(&final_) || is(&mine); + let differ = + own.is_some() && !clinvar.is_empty() && !mine.is_empty() && is(&clinvar) != is(&mine); + if !(is_plp || differ) { + continue; + } + let v = cpsr_variant(cell(r, "VAR_ID"), cell(r, "GENOMIC_CHANGE")) + .map_err(|e| format!("ValueError: {e}"))?; + let dp = digits(Some(stripped(r, "DP_CONTROL"))); + let or_dot = + |x: Option<&str>| Value::Str(x.filter(|v| !v.is_empty()).unwrap_or(".").to_owned()); + let mut x: IndexMap = IndexMap::new(); + x.insert( + "variant".into(), + match &v { + Some(h) => Value::Str(format!("{}:{} {}>{}", h.chrom, h.pos, h.reference, h.alt)), + None => or_dot(cell(r, "GENOMIC_CHANGE")), + }, + ); + x.insert("gene".into(), or_dot(cell(r, "SYMBOL"))); + x.insert("final".into(), or_dot(cell(r, col))); + x.insert( + "cpsr".into(), + own.map_or(Value::Null, |o| or_dot(cell(r, o))), + ); + x.insert( + "clinvar".into(), + if clinvar.is_empty() { + Value::Null + } else { + Value::Str(clinvar_raw.to_owned()) + }, + ); + x.insert( + "authority".into(), + opt_s(Some(authority).filter(|a| !a.is_empty())), + ); + x.insert( + "acmg_codes".into(), + opt_s(Some(stripped(r, "ACMG_CODE")).filter(|a| !a.is_empty())), + ); + x.insert("dp".into(), opt_int(dp)); + x.insert( + "low_confidence".into(), + Value::Bool(dp.is_some_and(|d| d < floor.dp)), + ); + if let Some(t) = &cdna + && is_plp + { + let found = v.as_ref().and_then(|h| { + t.get(&( + Some(h.chrom.clone()), + i128::from(h.pos), + Some(h.reference.clone()), + Some(h.alt.clone()), + )) + }); + x.insert( + "cdna".into(), + found + .filter(|c| pycompat::value::truthy(c)) + .cloned() + .unwrap_or_else(not_in_table), + ); + } + let x = Value::Object(x); + if differ { + disagree.push(x.clone()); + } + if is_plp { + plp.push(x); + } + } + Ok(Compare { + plp, + disagreements: disagree, + own_benign, + own: own.map(str::to_owned), + floor_dp: floor.dp, + cdna: cdna.is_some(), + }) +} diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs new file mode 100644 index 000000000..cb0ba8a1e --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -0,0 +1,284 @@ +//! Group b of `og summary collect`: the clinical sections. ClinVar, the +//! carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical filter +//! with its ACMG secondary-findings tier, slivar, and the `cyp2d6` block. +//! +//! Each input is read once: the ClinVar hits VCF feeds the ClinVar +//! section, the ACMG tier, the carrier list and CPSR's call floor; the +//! clinical VCF feeds the record count and the ACMG tier; AnnotSV's table +//! and step 35's copy-number tables feed both two-hit readings. The +//! ClinVar section (with its VEP lookups) and the clinical section (with +//! the clinical VCF) are read on their own threads. + +use super::acmg::{self, ClinicalVcf}; +use super::carrier::Carrier; +use super::clinical::{Clinical, Slivar}; +use super::clinvar::{ClinVar, Floor, HitsVcf}; +use super::cpsr::{self, Cpsr}; +use super::model::{Manifest, RunStatus, Section, Spec, Summary}; +use super::pgx::{Cpic, Cyp2d6, Cyrius, PharmCat, Pypgx}; +use super::scan::FolderIndex; +use super::sections::{Data, SampleQc}; +use super::two_hits::{self, DeletionInputs}; +use super::{Input, Read, read_at, read_optional, with_mtime}; +use std::path::Path; + +/// The sections `og summary collect --only b` reads, in summary.json's order. +pub const GROUP_B: [Spec; 9] = [ + Spec { + key: "clinvar", + title: "ClinVar screen", + step: "06", + }, + Spec { + key: "carrier", + title: "Carrier list", + step: "06", + }, + Spec { + key: "pharmcat", + title: "PharmCAT", + step: "07", + }, + Spec { + key: "cpic", + title: "CPIC drug-gene recommendations", + step: "27", + }, + Spec { + key: "pypgx", + title: "pypgx", + step: "32", + }, + Spec { + key: "cyrius", + title: "Cyrius (CYP2D6)", + step: "21", + }, + Spec { + key: "cpsr", + title: "Cancer predisposition (CPSR)", + step: "17", + }, + Spec { + key: "clinical", + title: "Clinical variant filter", + step: "23", + }, + Spec { + key: "slivar", + title: "Variant prioritization (slivar)", + step: "31", + }, +]; + +/// Reads the folder into the summary of group b. An error is a run status +/// or a manifest that is there and cannot be read. +pub fn collect(input: &Input) -> Result { + let dir = Path::new(&input.sample_dir); + let d = input.sample_dir.as_str(); + let index = FolderIndex::walk(dir); + let run = + read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); + let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + let s = pycompat::mtime::escape(&input.sample); + let raw = input.sample.as_str(); + let file = |rel: &str| -> (String, String) { (crate::join(d, rel), rel.to_owned()) }; + let find = |pats: &[String]| index.first(pats).map(file); + let one = |p: String| find(&[p]); + + // The shared inputs, each read once. + let hits = one(format!("clinvar/{s}_clinvar_hits.vcf")).map(|(p, rel)| HitsVcf::read(p, rel)); + let clinical_table = one(format!("clinical/{s}_clinical_summary.tsv")); + let clinical_vcf_at = one(format!("clinical/{s}_clinical.vcf.gz")); + let deletions = DeletionInputs { + annotsv: two_hits::read_annotsv( + one(format!("annotsv/{s}_sv_class4_5_checked.tsv")), + one(format!("sv_merged/{s}_sv_consensus.vcf.gz")).map(|(p, _)| p), + ), + step35: two_hits::read_step35(d, raw, &input.assets), + }; + let side = |rel: &str| one(rel.to_owned()); + + let (clinvar, clinical, rest) = std::thread::scope(|scope| { + let clinvar = scope.spawn(|| -> Read { + let Some(h) = &hits else { return Ok(None) }; + let data = ClinVar::read(h, d, &side, &s)?; + with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data)) + }); + let clinical = scope.spawn(|| -> Read { + let at = clinical_table.clone().or_else(|| clinical_vcf_at.clone()); + let Some((p, rel)) = at else { return Ok(None) }; + let vcf = clinical_vcf_at.as_ref().map(|(v, _)| ClinicalVcf::read(v)); + let tier = acmg::Inputs { + assets: &input.assets, + hits: hits.as_ref().map(|h| &h.read), + clinical: vcf.as_ref(), + deletions: &deletions, + }; + let data = Clinical::read( + clinical_table.as_ref().map(|(p, _)| p.as_str()), + vcf.as_ref(), + &tier, + )?; + with_mtime(&p, rel, Data::Clinical(data)) + }); + let rest = read_rest(input, &s, &index, &find, &hits, &deletions); + let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { + h.join() + .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) + }; + (join(clinvar), join(clinical), rest) + }); + let [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] = rest; + let reads = [ + clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, + ]; + let sections: Vec
= GROUP_B + .iter() + .zip(reads) + .map(|(spec, r)| Section::new(*spec, r, run.as_ref())) + .collect(); + let sec = |k: &str| sections.iter().find(|x| x.spec.key == k); + let cyp2d6 = match (sec("cpic"), sec("pypgx"), sec("cyrius")) { + (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new( + c, + p, + y, + one(format!("pgx_consensus/{s}_pgx_consensus.tsv")), + d, + )), + _ => None, + }; + + // The run block is group a's: the declared sex from the command line, + // the run status, the manifest or step 33's table, the first that has one. + let qc_declared = one(format!("qc/{s}_sample_qc.tsv")).and_then(|(p, _)| { + let rows = super::sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = [ + input.declared_sex.clone(), + run.as_ref().and_then(|r| r.declared_sex.clone()), + manifest + .as_ref() + .and_then(|m| m.value("run", "declared_sex").map(str::to_owned)), + qc_declared, + ] + .into_iter() + .flatten() + .find(|v| !v.is_empty()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + Ok(Summary { + sample: input.sample.clone(), + generated_utc: super::time::utc_minute(now).unwrap_or_default(), + run, + manifest, + declared_sex, + sections, + cyp2d6, + }) +} + +/// Finds a section's file: the full path and the path in the folder. +type Find<'a> = dyn Fn(&[String]) -> Option<(String, String)> + 'a; + +/// The seven sections read on the calling thread, in order. +fn read_rest( + input: &Input, + s: &str, + index: &FolderIndex, + find: &Find, + hits: &Option, + deletions: &DeletionInputs, +) -> [Read; 7] { + let one = |p: String| find(&[p]); + let carrier = match hits { + None => Ok(None), + Some(h) => Carrier::read(&input.assets, &h.read, deletions) + .and_then(|c| with_mtime(&h.path, h.rel.clone(), Data::Carrier(c))), + }; + let pharmcat = pharmcat_at(index, input, s) + .and_then(|at| read_at(at, |p| PharmCat::read(p).map(Data::PharmCat))); + let cpic = read_at(one(format!("cpic/{s}_phenotypes.tsv")), |p| { + Cpic::read(p, &input.sample).map(Data::Cpic) + }); + let pypgx = read_at(one(format!("pypgx/{s}_pypgx_summary.tsv")), |p| { + Pypgx::read(p, &input.sample).map(Data::Pypgx) + }); + let cyrius = read_at( + find(&[ + format!("cyrius/{s}_cyp2d6.tsv"), + "cyrius/*_cyp2d6.tsv".to_owned(), + ]), + |p| Cyrius::read(p).map(Data::Cyrius), + ); + let html = find(&[ + format!("cpsr/{s}.cpsr.grch38.html"), + "cpsr/*.cpsr.*.html".to_owned(), + ]); + let tsv = find(&[ + format!("cpsr/{s}.cpsr.grch38.classification.tsv.gz"), + "cpsr/*.cpsr.*.classification.tsv.gz".to_owned(), + ]); + let cpsr = read_at(tsv.clone().or_else(|| html.clone()), |_| { + let floor = match hits { + None => Ok(Floor::default()), + Some(h) => h.read.as_ref().map(|(f, _)| *f).map_err(Clone::clone), + }; + Cpsr::read(&cpsr::Inputs { + html: html.as_ref().map(|(p, _)| p.clone()), + tsv: tsv.as_ref().map(|(p, _)| p.clone()), + floor, + cdna: one(format!("cpsr/{s}_cpsr_cdna_check.tsv")).map(|(p, _)| p), + }) + .map(Data::Cpsr) + }); + let slivar_table = one(format!("slivar/{s}_slivar_summary.tsv")); + let slivar_vcf = if slivar_table.is_none() { + one(format!("slivar/{s}_prioritized.vcf.gz")) + } else { + None + }; + let slivar = read_at(slivar_table.clone().or(slivar_vcf.clone()), |_| { + Slivar::read( + slivar_table.as_ref().map(|(p, _)| p.as_str()), + slivar_vcf.as_ref().map(|(p, _)| p.as_str()), + &input.sample, + ) + .map(Data::Slivar) + }); + [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] +} + +/// PharmCAT's report: the newest of the files its four names match. +fn pharmcat_at( + index: &FolderIndex, + input: &Input, + s: &str, +) -> Result, String> { + let d = input.sample_dir.as_str(); + let found: Vec<&str> = [ + format!("pharmcat/{s}.report.json"), + "pharmcat/*.report.json".to_owned(), + format!("vcf/{s}.report.json"), + "vcf/*.report.json".to_owned(), + ] + .iter() + .flat_map(|p| index.all(p)) + .collect(); + let paths: Vec<(String, &str)> = found + .iter() + .map(|rel| (crate::join(d, rel), *rel)) + .collect(); + let newest = pycompat::mtime::newest(paths.iter().map(|(p, _)| p.as_str())) + .map_err(|e| e.to_string())?; + Ok(newest.map(|p| { + let rel = paths.iter().find(|(q, _)| q == p).map_or("", |(_, r)| r); + (p.to_owned(), rel.to_owned()) + })) +} diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index 689d8c087..c6d6e22a3 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -295,6 +295,8 @@ pub struct Summary { pub manifest: Option, pub declared_sex: Option, pub sections: Vec
, + /// The CYP2D6 callers side by side (group b). + pub cyp2d6: Option, } impl Summary { @@ -334,6 +336,9 @@ impl Summary { .collect(), ), ); + if let Some(c) = &self.cyp2d6 { + m.insert("cyp2d6".into(), c.to_json()); + } Value::Object(m) } } diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs new file mode 100644 index 000000000..458660493 --- /dev/null +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -0,0 +1,618 @@ +//! The pharmacogenomics sections: PharmCAT (step 07), the CPIC table (step +//! 27), pypgx (step 32) and Cyrius (step 21), and the summary's `cyp2d6` +//! block, which sets the three CYP2D6 callers side by side by step 36's +//! rule (`og_steps::pgx_rules`). + +use super::clinvar::{obj, opt_int, opt_s}; +use super::model::{Section, State}; +use super::sections::Data; +use super::text::{self, cell, stripped}; +use indexmap::IndexMap; +use og_steps::pgx_rules::{NO_CALL, is_call, same_diplotype}; +use og_steps::pypgx_summary::{LOW_GQ, NOT_CALLABLE}; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use pycompat::text::{lower, strip, upper}; + +/// The label of pypgx's CYP2D6 call when it is the only usable one. +pub const PROBABLE: &str = "probable (one caller: pypgx)"; +/// The rows of the CYP2D6 depth check the reports show. +pub const DEPTH_CHECK_KEYS: [&str; 5] = [ + "status", + "gene_mapq1_share", + "gene_mapq20_share", + "body_mapq1_share", + "body_mapq20_share", +]; + +fn rows_or_note(path: &str) -> Result, String> { + text::rows(path).map_err(|e| e.note()) +} + +/// `row.get(key, "")`: `""` without the column, `None` for a short row. +fn get_default(r: &DictRow, key: &str) -> Value { + match r.get(key) { + None => Value::Str(String::new()), + Some(v) => opt_s(v), + } +} + +/// `row[key]` as JSON: a `KeyError` note without the column. +fn need(r: &DictRow, key: &str) -> Result { + text::need(r, key).map(opt_s) +} + +// --- PharmCAT ------------------------------------------------------------------------ + +#[derive(Debug, Clone, PartialEq)] +pub struct PharmCat { + pub version: String, + pub html_report: bool, +} + +impl PharmCat { + pub fn read(path: &str) -> Result { + let raw = std::fs::read(path) + .map_err(|e| format!("{}: {}", crate::basename(path), crate::os_error(&e, path)))?; + let body = String::from_utf8(raw) + .map_err(|e| format!("{}: 'utf-8' codec can't decode: {e}", crate::basename(path)))?; + let data = + pycompat::json::loads(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; + let version = match &data { + Value::Object(m) => match m.get("pharmcatVersion") { + None => String::new(), + Some(v) => pycompat::value::py_str(v).map_err(|e| e.to_string())?, + }, + _ => String::new(), + }; + let html = path.strip_suffix(".json").map(|p| format!("{p}.html")); + Ok(PharmCat { + version, + html_report: html.is_some_and(crate::is_file), + }) + } + + pub fn to_json(&self) -> Value { + obj([ + ("version", Value::Str(self.version.clone())), + ("html_report", Value::Bool(self.html_report)), + ]) + } +} + +// --- CPIC ---------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct CpicGene { + pub gene: Value, + pub diplotype: Value, + pub phenotype: Value, + pub status: Value, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Cpic { + pub genes: Vec, + pub parse_failed: bool, + pub warnings: Vec, +} + +impl Cpic { + /// `sec_cpic`: the phenotype table and, beside it, the recommendation + /// text's parse failure and warnings. + pub fn read(path: &str, s: &str) -> Result { + let genes: Vec = rows_or_note(path)? + .iter() + .map(|r| CpicGene { + gene: get_default(r, "Gene"), + diplotype: get_default(r, "Diplotype"), + phenotype: get_default(r, "Phenotype"), + status: get_default(r, "Status"), + }) + .collect(); + let rec = crate::join( + text::dirname(path), + &format!("{s}_cpic_recommendations.txt"), + ); + let (mut warnings, mut failed) = (Vec::new(), false); + if crate::is_file(&rec) { + for line in text::lines(&rec).map_err(|e| e.note())? { + if line.contains("PARSING FAILED") { + failed = true; + } + let t = strip(&line, None); + if let Some(w) = t.strip_prefix("WARNING:") { + warnings.push(strip(w, None).to_owned()); + } + } + } + let parse_failed = failed || genes.is_empty(); + Ok(Cpic { + genes, + parse_failed, + warnings, + }) + } + + fn count(&self, status: &str) -> Value { + Value::Int( + self.genes + .iter() + .filter(|g| matches!(&g.status, Value::Str(s) if s == status)) + .count() as i128, + ) + } + + /// The CYP2D6 row, if the table has one. + pub fn cyp2d6(&self) -> Option<&CpicGene> { + self.genes + .iter() + .find(|g| matches!(&g.gene, Value::Str(s) if s == "CYP2D6")) + } + + pub fn to_json(&self) -> Value { + obj([ + ( + "genes", + Value::Array( + self.genes + .iter() + .map(|g| { + obj([ + ("gene", g.gene.clone()), + ("diplotype", g.diplotype.clone()), + ("phenotype", g.phenotype.clone()), + ("status", g.status.clone()), + ]) + }) + .collect(), + ), + ), + ("non_normal", self.count("non-normal")), + ("ambiguous", self.count("ambiguous")), + ("not_called", self.count("not called")), + ("unclassified", self.count("unclassified")), + ("parse_failed", Value::Bool(self.parse_failed)), + ( + "warnings", + Value::Array(self.warnings.iter().cloned().map(Value::Str).collect()), + ), + ]) + } +} + +// --- pypgx --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct Pypgx { + pub genes_total: usize, + pub genes_called: usize, + pub cyp2d6: String, + /// Gene (as JSON's key text) to the copy-number call or why none. + pub cnv_call: IndexMap, + pub not_callable: Vec, + pub low_gq: Vec, + /// phenotype, defining_filter, quality of the CYP2D6 row. + pub cyp2d6_evidence: Option<[String; 3]>, + pub depth_check: Option>, + /// conflicts, one_tool_only, concordant. + pub comparison: Option<[usize; 3]>, +} + +/// A JSON object key from a cell that may be `None` (`json.dumps` writes `null`). +fn key_text(v: Option<&str>) -> String { + v.unwrap_or("null").to_owned() +} + +impl Pypgx { + pub fn read(path: &str, s: &str) -> Result { + let rows = rows_or_note(path)?; + let called = rows + .iter() + .filter(|r| !NO_CALL.contains(&lower(stripped(r, "Diplotype")).as_str())) + .count(); + let cyp_row = rows.iter().find(|r| cell(r, "Gene") == Some("CYP2D6")); + let cyp = cyp_row.and_then(|r| cell(r, "Diplotype")).unwrap_or(""); + let mut cnv_call = IndexMap::new(); + for r in &rows { + let status = stripped(r, "Status"); + if upper(stripped(r, "Source")) != "BAM" || status == NOT_CALLABLE { + continue; + } + let v = stripped(r, "CNV_call"); + let gene = key_text(text::need(r, "Gene")?); + let call = if !matches!(v, "" | ".") && !v.starts_with("N/A") { + v.to_owned() + } else if lower(status) == "failed" || upper(stripped(r, "Diplotype")) == "FAILED" { + "not called: pypgx failed".to_owned() + } else if status.contains("depth check") || v.contains("depth check") { + "not called: the CYP2D6 depth check failed".to_owned() + } else { + "not called: pypgx gave no copy-number call".to_owned() + }; + cnv_call.insert(gene, call); + } + let mut not_callable = Vec::new(); + for r in rows + .iter() + .filter(|r| stripped(r, "Status") == NOT_CALLABLE) + { + not_callable.push(need(r, "Gene")?); + } + let mut low_gq = Vec::new(); + for r in rows.iter().filter(|r| stripped(r, "Quality") == LOW_GQ) { + let gene = need(r, "Gene")?; + let min = stripped(r, "Min_GQ"); + let min_gq = if pycompat::text::isdigit(min) { + opt_int(crate::int(min).ok()) + } else { + Value::Null + }; + low_gq.push(obj([ + ("gene", gene), + ("diplotype", Value::Str(stripped(r, "Diplotype").to_owned())), + ("min_gq", min_gq), + ( + "weakest", + Value::Str(stripped(r, "Weakest_variant").to_owned()), + ), + ])); + } + let cyp2d6_evidence = cyp_row.map(|r| { + [ + stripped(r, "Phenotype").to_owned(), + stripped(r, "Defining_filter").to_owned(), + stripped(r, "Quality").to_owned(), + ] + }); + let dir = text::dirname(path); + let dc = crate::join(dir, &format!("{s}_cyp2d6_depth_check.tsv")); + let depth_check = if crate::is_file(&dc) { + let mut kv: IndexMap = IndexMap::new(); + for line in text::lines(&dc).map_err(|e| e.note())? { + if let Some((k, v)) = line.split_once('\t') { + kv.insert(k.to_owned(), v.to_owned()); + } + } + Some( + DEPTH_CHECK_KEYS + .iter() + .filter_map(|k| kv.get(*k).map(|v| ((*k).to_owned(), v.clone()))) + .collect(), + ) + } else { + None + }; + let comp = crate::join(dir, &format!("{s}_pharmcat_comparison.tsv")); + let comparison = if crate::is_file(&comp) { + let c = rows_or_note(&comp)?; + let n = + |f: &dyn Fn(Option<&str>) -> bool| c.iter().filter(|r| f(cell(r, "Match"))).count(); + Some([ + n(&|m| m == Some("No")), + n(&|m| matches!(m, Some("pypgx only" | "PharmCAT only"))), + n(&|m| m == Some("Yes")), + ]) + } else { + None + }; + Ok(Pypgx { + genes_total: rows.len(), + genes_called: called, + cyp2d6: if cyp.is_empty() { "not called" } else { cyp }.to_owned(), + cnv_call, + not_callable, + low_gq, + cyp2d6_evidence, + depth_check, + comparison, + }) + } + + pub fn to_json(&self) -> Value { + let mut m: IndexMap = IndexMap::new(); + m.insert("genes_total".into(), Value::Int(self.genes_total as i128)); + m.insert("genes_called".into(), Value::Int(self.genes_called as i128)); + m.insert("cyp2d6".into(), Value::Str(self.cyp2d6.clone())); + m.insert( + "cnv_call".into(), + Value::Object( + self.cnv_call + .iter() + .map(|(k, v)| (k.clone(), Value::Str(v.clone()))) + .collect(), + ), + ); + m.insert( + "not_callable".into(), + Value::Array(self.not_callable.clone()), + ); + m.insert("low_gq".into(), Value::Array(self.low_gq.clone())); + if let Some([p, f, q]) = &self.cyp2d6_evidence { + m.insert( + "cyp2d6_evidence".into(), + obj([ + ("phenotype", Value::Str(p.clone())), + ("defining_filter", Value::Str(f.clone())), + ("quality", Value::Str(q.clone())), + ]), + ); + } + if let Some(d) = &self.depth_check { + m.insert( + "depth_check".into(), + Value::Object( + d.iter() + .map(|(k, v)| (k.clone(), Value::Str(v.clone()))) + .collect(), + ), + ); + } + if let Some([c, o, y]) = self.comparison { + m.insert( + "comparison".into(), + obj([ + ("conflicts", Value::Int(c as i128)), + ("one_tool_only", Value::Int(o as i128)), + ("concordant", Value::Int(y as i128)), + ]), + ); + } + Value::Object(m) + } +} + +// --- Cyrius -------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct Cyrius { + pub genotype: String, + /// `None` for a row cut short of the column. + pub filter: Option, +} + +impl Cyrius { + pub fn read(path: &str) -> Result { + let rows = rows_or_note(path)?; + let r = rows.first(); + let genotype = r.and_then(|r| cell(r, "Genotype")).unwrap_or(""); + let filter = match r.map(|r| r.get("Filter")) { + None | Some(None) => Some(String::new()), + Some(Some(v)) => v.map(str::to_owned), + }; + Ok(Cyrius { + genotype: if genotype.is_empty() { + "none" + } else { + genotype + } + .to_owned(), + filter, + }) + } + + pub fn to_json(&self) -> Value { + obj([ + ("genotype", Value::Str(self.genotype.clone())), + ("filter", opt_s(self.filter.as_deref())), + ]) + } +} + +// --- the cyp2d6 block ---------------------------------------------------------------- + +/// Step 36's CYP2D6 row, as the summary keeps it. +#[derive(Debug, Clone, PartialEq)] +pub struct Consensus { + pub result: String, + pub passed_to_pharmcat: bool, + pub reason: String, + pub source: Option, +} + +/// pypgx's call alone, labelled. +#[derive(Debug, Clone, PartialEq)] +pub struct Probable { + pub diplotype: String, + pub phenotype: String, +} + +/// The `cyp2d6` block of summary.json. +#[derive(Debug, Clone, PartialEq)] +pub struct Cyp2d6 { + pub pharmcat: Value, + pub pypgx: Value, + pub cyrius: Value, + pub agree: Option, + pub consensus: Option, + pub probable: Option, +} + +/// `probable_cyp2d6`: pypgx's call when it is the only usable one. +fn probable( + diplotype: &str, + evidence: Option<&[String; 3]>, + cyrius: Option<&Cyrius>, + depth_status: Option<&str>, +) -> Option { + if !is_call(diplotype) { + return None; + } + if let Some(c) = cyrius + && is_call(&c.genotype) + && c.filter.as_deref() == Some("PASS") + { + return None; + } + if depth_status.is_some_and(|d| d != "ok") { + return None; + } + let [phenotype, filter, quality] = evidence.cloned().unwrap_or_default(); + // A row without the evidence passes None, which neither list holds. + let have = evidence.is_some(); + if !have + || !matches!(filter.as_str(), "PASS" | "none") + || !matches!(quality.as_str(), "ok" | "N/A") + { + return None; + } + Some(Probable { + diplotype: strip(diplotype, None).to_owned(), + phenotype: strip(&phenotype, None).to_owned(), + }) +} + +/// `read_consensus`: step 36's CYP2D6 row, or `None` when the table is +/// missing, older than a caller's result it was made from, or has no +/// CYP2D6 row. +fn read_consensus( + found: Option<(String, String)>, + dir: &str, + sources: [Option<&str>; 2], +) -> Result, String> { + let Some((p, rel)) = found else { + return Ok(None); + }; + let mtime = |x: &str| pycompat::mtime::mtime(x).map_err(|e| crate::os_error(&e, x)); + let own = mtime(&p)?; + for src in sources.into_iter().flatten() { + if mtime(&crate::join(dir, src))? > own { + return Ok(None); + } + } + let rows = text::rows(&p).map_err(|e| e.text())?; + let Some(row) = rows.iter().find(|r| stripped(r, "Gene") == "CYP2D6") else { + return Ok(None); + }; + Ok(Some(Consensus { + result: text::cell_or(row, "Result").to_owned(), + passed_to_pharmcat: cell(row, "Outside_call") == Some("yes"), + reason: text::cell_or(row, "Reason").to_owned(), + source: Some(rel), + })) +} + +impl Cyp2d6 { + /// `cyp2d6_block` over the three caller sections. + pub fn new( + cpic: &Section, + pypgx: &Section, + cyrius: &Section, + consensus_at: Option<(String, String)>, + dir: &str, + ) -> Cyp2d6 { + let pc = match &cpic.data { + Some(Data::Cpic(c)) => c.cyp2d6(), + _ => None, + }; + let pd = match (&pypgx.data, pypgx.state) { + (Some(Data::Pypgx(p)), State::Ok) => Some(p), + _ => None, + }; + let cy = match (&cyrius.data, cyrius.state) { + (Some(Data::Cyrius(c)), State::Ok) => Some(c), + _ => None, + }; + let failed = |s: &Section| { + (s.state == State::Failed).then(|| Value::Str(format!("failed (step {})", s.spec.step))) + }; + let pg = pd.map(|p| p.cyp2d6.as_str()); + let pg_ok = pg.is_some_and(is_call); + let cy_ok = cy.is_some_and(|c| is_call(&c.genotype) && c.filter.as_deref() == Some("PASS")); + let cy_txt = cy.map(|c| { + let mut t = c.genotype.clone(); + if is_call(&t) && !cy_ok { + let f = c + .filter + .as_deref() + .filter(|f| !f.is_empty()) + .unwrap_or("missing"); + t.push_str(&format!(" (not usable: Filter {f})")); + } + t + }); + let pharmcat = match pc { + Some(g) => { + let st = &g.status; + if matches!(st, Value::Str(s) if s == "not called" || s == "ambiguous") { + st.clone() + } else { + g.diplotype.clone() + } + } + None => failed(cpic).unwrap_or(Value::Null), + }; + let agree = match (pg_ok && cy_ok, pg, cy) { + (true, Some(a), Some(c)) => Some(same_diplotype(a, &c.genotype)), + _ => None, + }; + let consensus = match read_consensus( + consensus_at, + dir, + [pypgx.source.as_deref(), cyrius.source.as_deref()], + ) { + Ok(c) => c, + Err(e) => Some(Consensus { + result: "unreadable".into(), + passed_to_pharmcat: false, + reason: e, + source: None, + }), + }; + let probable = match (pg_ok, pd) { + (true, Some(p)) => probable( + &p.cyp2d6, + p.cyp2d6_evidence.as_ref(), + cy, + p.depth_check + .as_ref() + .and_then(|d| d.get("status")) + .map(String::as_str), + ), + _ => None, + }; + Cyp2d6 { + pharmcat, + pypgx: failed(pypgx).unwrap_or_else(|| opt_s(pg)), + cyrius: failed(cyrius).unwrap_or_else(|| opt_s(cy_txt.as_deref())), + agree, + consensus, + probable, + } + } + + pub fn to_json(&self) -> Value { + obj([ + ( + "calls", + obj([ + ("PharmCAT", self.pharmcat.clone()), + ("pypgx", self.pypgx.clone()), + ("Cyrius", self.cyrius.clone()), + ]), + ), + ("agree", self.agree.map_or(Value::Null, Value::Bool)), + ( + "consensus", + self.consensus.as_ref().map_or(Value::Null, |c| { + obj([ + ("result", Value::Str(c.result.clone())), + ("passed_to_pharmcat", Value::Bool(c.passed_to_pharmcat)), + ("reason", Value::Str(c.reason.clone())), + ("source", opt_s(c.source.as_deref())), + ]) + }), + ), + ( + "probable", + self.probable.as_ref().map_or(Value::Null, |p| { + obj([ + ("diplotype", Value::Str(p.diplotype.clone())), + ("phenotype", Value::Str(p.phenotype.clone())), + ("label", Value::Str(PROBABLE.into())), + ]) + }), + ), + ]) + } +} diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 26d7c0087..2792c67d6 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -87,6 +87,25 @@ impl FolderIndex { } None } + + /// Every file one pattern names, in sorted order (`glob` with the + /// order made stable), for the readers that take the newest of them. + pub fn all(&self, pattern: &str) -> Vec<&str> { + if !has_magic(pattern) { + return self + .files + .get(pattern) + .map(String::as_str) + .into_iter() + .collect(); + } + let pat: Vec<&str> = pattern.split('/').collect(); + self.sorted + .iter() + .filter(|f| matches(&f.split('/').collect::>(), &pat)) + .map(String::as_str) + .collect() + } } fn has_magic(s: &str) -> bool { diff --git a/rust/crates/og-summary/src/collect/sections.rs b/rust/crates/og-summary/src/collect/sections.rs index 8aa04d534..a719a593a 100644 --- a/rust/crates/og-summary/src/collect/sections.rs +++ b/rust/crates/og-summary/src/collect/sections.rs @@ -20,6 +20,15 @@ pub enum Data { MosaicSexLoss(Mosaic), ReadCount(ReadCount), Variants(Variants), + ClinVar(super::clinvar::ClinVar), + Carrier(super::carrier::Carrier), + PharmCat(super::pgx::PharmCat), + Cpic(super::pgx::Cpic), + Pypgx(super::pgx::Pypgx), + Cyrius(super::pgx::Cyrius), + Cpsr(super::cpsr::Cpsr), + Clinical(super::clinical::Clinical), + Slivar(super::clinical::Slivar), } impl Data { @@ -32,6 +41,15 @@ impl Data { Data::MosaicSexLoss(d) => d.to_json(), Data::ReadCount(d) => d.to_json(), Data::Variants(d) => d.to_json(), + Data::ClinVar(d) => d.to_json(), + Data::Carrier(d) => d.to_json(), + Data::PharmCat(d) => d.to_json(), + Data::Cpic(d) => d.to_json(), + Data::Pypgx(d) => d.to_json(), + Data::Cyrius(d) => d.to_json(), + Data::Cpsr(d) => d.to_json(), + Data::Clinical(d) => d.to_json(), + Data::Slivar(d) => d.to_json(), } } } diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs new file mode 100644 index 000000000..a215514b0 --- /dev/null +++ b/rust/crates/og-summary/src/collect/text.rs @@ -0,0 +1,195 @@ +//! The small readers the clinical sections (group b) share: a VCF's data +//! lines as cells, INFO and FORMAT as maps, a table whose read errors keep +//! their kind, and Python's way of reading a digit cell. + +use crate::{basename, os_error, text_lines}; +use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::text::{isdigit, strip}; +use std::collections::HashMap; + +/// Why a file could not be read: an `OSError` (its `str()`), or a fault of +/// another kind, already written as the section's note. +#[derive(Debug, Clone)] +pub enum ReadError { + Os(String), + Other(String), +} + +impl ReadError { + /// The note when the Python wrapped the error in `Unreadable(f"{name}: {e}")`. + pub fn unreadable(&self, path: &str) -> String { + match self { + ReadError::Os(e) => format!("{}: {e}", basename(path)), + ReadError::Other(n) => n.clone(), + } + } + + /// The note of the collector's catch-all: `type(e).__name__: e`. + pub fn note(&self) -> String { + match self { + ReadError::Os(e) => format!("{}: {e}", os_kind(e)), + ReadError::Other(n) => n.clone(), + } + } + + /// `str(e)`, as a message that embeds the error prints it. + pub fn text(&self) -> String { + match self { + ReadError::Os(e) => e.clone(), + ReadError::Other(n) => n.split_once(": ").map_or(n.clone(), |(_, m)| m.to_owned()), + } + } +} + +fn os_kind(e: &str) -> &'static str { + if e.contains("No such file") { + "FileNotFoundError" + } else if e.contains("Permission denied") { + "PermissionError" + } else if e.contains("Is a directory") { + "IsADirectoryError" + } else { + "OSError" + } +} + +/// The lines of a text file (gzip by its magic, universal newlines, bad +/// UTF-8 as U+FFFD), each without its line ending. +pub fn lines(path: &str) -> Result, ReadError> { + let it = og_io::Lines::new( + og_io::open(path).map_err(|e| ReadError::Os(os_error(&e, path)))?, + og_io::Utf8::Replace, + ); + it.map(|l| l.map_err(|e| ReadError::Os(gz_text(&e)))) + .collect() +} + +/// gzip's own words for a stream cut short; the reader's otherwise. +fn gz_text(e: &std::io::Error) -> String { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + "Compressed file ended before the end-of-stream marker was reached".to_owned() + } else { + e.to_string() + } +} + +/// The data lines of a VCF split at tabs (`vcf_records`). +pub fn vcf_records(path: &str) -> Result>, ReadError> { + Ok(lines(path)? + .into_iter() + .filter(|l| !l.starts_with('#')) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect()) +} + +/// The rows of a tab-separated table with a header (`csv.DictReader`). +pub fn rows(path: &str) -> Result, ReadError> { + let mut lines = text_lines(path).map_err(ReadError::Os)?; + let iter = std::iter::from_fn(move || { + let mut s = String::new(); + match lines.read_line(&mut s) { + Ok(true) => Some(Ok(s)), + Ok(false) => None, + Err(e) => Some(Err(e)), + } + }); + DictReader::new(iter, '\t', Quoting::Minimal) + .collect::, _>>() + .map_err(|e| match e.exception() { + "OSError" => ReadError::Os(e.to_string()), + kind => ReadError::Other(format!("{kind}: {e}")), + }) +} + +/// The header of a table as `csv.reader`'s first row reads it. +pub fn columns(path: &str) -> Result, ReadError> { + let first = lines(path)?.into_iter().next().unwrap_or_default(); + if first.is_empty() { + return Ok(Vec::new()); + } + Ok(first.split('\t').map(str::to_owned).collect()) +} + +/// `row.get(key)` where a missing key and a short row both read as `None`. +pub fn cell<'a>(row: &'a DictRow, key: &str) -> Option<&'a str> { + row.get(key).flatten() +} + +/// `row.get(key) or ""`. +pub fn cell_or<'a>(row: &'a DictRow, key: &str) -> &'a str { + cell(row, key).unwrap_or("") +} + +/// `(row.get(key) or "").strip()`. +pub fn stripped<'a>(row: &'a DictRow, key: &str) -> &'a str { + strip(cell_or(row, key), None) +} + +/// `row[key]`: a `KeyError` note when the table has no such column. A +/// short row gives `None`. +pub fn need<'a>(row: &'a DictRow, key: &str) -> Result, String> { + row.get(key) + .ok_or_else(|| format!("KeyError: {}", crate::repr(key))) +} + +/// `int(x) if x.isdigit() else None`. +pub fn digits(x: Option<&str>) -> Option { + x.filter(|v| isdigit(v)).and_then(|v| crate::int(v).ok()) +} + +/// `info_map`: a VCF INFO field as a map, a later key over an earlier one. +pub fn info_map(field: &str) -> HashMap<&str, &str> { + field + .split(';') + .map(|kv| kv.split_once('=').unwrap_or((kv, ""))) + .collect() +} + +/// `dict(zip(FORMAT.split(":"), SAMPLE.split(":")))`. +pub fn format_map<'a>(fmt: &'a str, sample: &'a str) -> HashMap<&'a str, &'a str> { + fmt.split(':').zip(sample.split(':')).collect() +} + +/// One FORMAT field of the first sample, `None` when absent, empty or `.`. +pub fn gt_field(r: &[String], key: &str) -> Option { + if r.len() < 10 { + return None; + } + format_map(&r[8], &r[9]) + .get(key) + .filter(|v| !v.is_empty() && **v != ".") + .map(|v| (*v).to_owned()) +} + +/// The GT of the first sample, `""` without one. +pub fn gt_of(r: &[String]) -> &str { + r.get(9).map_or("", |s| s.split(':').next().unwrap_or("")) +} + +/// `int(r[1])`, as the note of the collector's catch-all when it is not one. +pub fn pos(r: &[String]) -> Result { + crate::int(&r[1]).map_err(|e| format!("ValueError: {e}")) +} + +/// Python's `max()` over floats: the first of the largest, NaN kept only +/// when it comes first. +pub fn py_max(xs: impl IntoIterator) -> Option { + xs.into_iter().fold(None, |best, x| match best { + Some(b) if x.partial_cmp(&b) != Some(std::cmp::Ordering::Greater) => Some(b), + _ => Some(x), + }) +} + +/// `os.path.relpath(path, base)` for a path found under `base`. +pub fn relpath(base: &str, path: &str) -> String { + let b = base.trim_end_matches('/'); + path.strip_prefix(b) + .and_then(|r| r.strip_prefix('/')) + .unwrap_or(path) + .to_owned() +} + +/// `os.path.dirname(p)`. +pub fn dirname(p: &str) -> &str { + p.rfind('/').map_or("", |i| &p[..i]) +} diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs new file mode 100644 index 000000000..52d2184b8 --- /dev/null +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -0,0 +1,980 @@ +//! Two hits in one gene across callers: a called deletion plus a small +//! variant. +//! +//! Small variants alone miss a gene hit twice when one hit is a deletion: +//! an exon deletion of one copy of MUTYH plus a pathogenic variant on the +//! other, an HBA deletion plus an HBA point variant, one SMN1 copy plus an +//! SMN1 variant, a 1q21.1 deletion plus the RBM8A low-expression allele +//! (TAR syndrome: Albers et al., Nat Genet 2012, PMID 22366785). The +//! deletions come from the engines that call them: AnnotSV's class 4 and 5 +//! deletions (step 05), step 35's interval table, its HBA count and +//! Parascopy's SMN1 count. +//! +//! The two are in trans (biallelic) only when the reads show it: the +//! variant sits inside the deletion and is called homozygous (one copy is +//! left, and it carries the variant), or both calls carry read-backed phase +//! (a `|` GT and the same PS) on different copies. On the same copy (cis) +//! they are no second hit; otherwise they are possibly in trans. +//! +//! The readers here take from step 05's and step 35's tables only what the +//! rule needs. The AnnotSV and Parascopy sections themselves are read by +//! their own modules. + +use super::clinvar::obj; +use super::text::{self, ReadError, cell, columns, digits, rows}; +use indexmap::IndexMap; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use pycompat::text::isdigit; +use std::collections::{BTreeSet, HashMap}; + +/// Gene spans: chrom, start, end (1-based, inclusive). +pub type Spans = IndexMap; + +/// A called deletion that reaches a gene. +#[derive(Debug, Clone, PartialEq)] +pub struct Deletion { + pub chrom: String, + pub start: i128, + pub end: i128, + pub text: String, + /// A variant inside the span is on the one copy left. False for the + /// HBA count, which cannot say which gene of the pair is gone. + pub inside: bool, + /// The call's GT and phase set (AnnotSV only). + pub gt: Option, + pub ps: Option, +} + +/// A small variant of a gene, as the rule reads it. +#[derive(Debug, Clone, PartialEq)] +pub struct Variant { + pub gene: String, + pub key: VKey, + pub chrom: String, + pub pos: i128, + pub text: String, + pub genotype: String, + pub copies: usize, + pub ploidy: usize, + pub gt: String, + pub ps: Option, + /// The severity name of a hypomorph allele. + pub allele: Option, +} + +/// A variant key: a small variant `(chrom, pos, ref, alt)`. +pub type VKey = (String, i128, String, String); + +/// How a deletion and a small variant in one gene pair up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Verdict { + /// Both copies are hit. + Biallelic, + /// Phase unknown, or the calls disagree: possibly both copies. + PossiblyBiallelic, + /// Both on the same copy: no second hit. + Cis, +} + +impl Verdict { + pub fn word(self) -> &'static str { + match self { + Verdict::Biallelic => "biallelic", + Verdict::PossiblyBiallelic => "possibly biallelic", + Verdict::Cis => "cis", + } + } +} + +/// The best pair of a gene. +#[derive(Debug, Clone, PartialEq)] +pub struct Entry { + pub gene: String, + pub verdict: Verdict, + pub why: String, + pub deletion: String, + pub variant: String, + pub genotype: String, + pub allele: Option, +} + +impl Entry { + pub fn to_json(&self) -> IndexMap { + let Value::Object(m) = obj([ + ("gene", Value::Str(self.gene.clone())), + ("verdict", Value::Str(self.verdict.word().into())), + ("why", Value::Str(self.why.clone())), + ("deletion", Value::Str(self.deletion.clone())), + ("variant", Value::Str(self.variant.clone())), + ("genotype", Value::Str(self.genotype.clone())), + ( + "allele", + self.allele.clone().map_or(Value::Null, Value::Str), + ), + ]) else { + unreachable!() + }; + m + } +} + +/// The copies a phased GT puts the alt allele on, or `None` when the GT +/// is not phased or a copy is not called (the missing copy could carry the +/// other allele, so no cis or trans can be read from it). +pub fn phase_haps(gt: Option<&str>) -> Option> { + let gt = gt.filter(|g| !g.is_empty() && g.contains('|'))?; + let alleles: Vec<&str> = gt.split('|').collect(); + if alleles.contains(&".") { + return None; + } + Some( + alleles + .iter() + .enumerate() + .filter(|(_, a)| **a != "0") + .map(|(i, _)| i) + .collect(), + ) +} + +/// The verdict of one deletion and one small variant in one gene, and why. +pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { + let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; + // Read-backed phase first: on the same copy (cis) is no second hit, + // wherever the variant sits. + let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); + let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); + let mut phased: Option<(Verdict, String)> = None; + if let (Some(hd), Some(hv)) = (&hd, &hv) + && let Some(ps) = dl.ps.as_deref().filter(|p| !p.is_empty()) + && Some(ps) == v.ps.as_deref() + { + if hd == hv { + return ( + Verdict::Cis, + format!("in cis: read-backed phase puts both on the same copy (phase set {ps})"), + ); + } + phased = Some(if hd.is_disjoint(hv) { + ( + Verdict::Biallelic, + format!( + "in trans: read-backed phase puts them on different copies (phase set {ps})" + ), + ) + } else { + ( + Verdict::PossiblyBiallelic, + format!( + "read-backed phase puts the variant on both copies (phase set {ps}); check the reads" + ), + ) + }); + } + if inside && v.copies >= 2 { + return ( + Verdict::Biallelic, + "the variant is called homozygous inside the deletion: it is hemizygous, on the one \ + copy the deletion leaves" + .into(), + ); + } + if inside && v.ploidy == 2 { + return ( + Verdict::PossiblyBiallelic, + "a heterozygous call inside a called deletion: one of the two calls may be wrong; \ + check the reads" + .into(), + ); + } + if let Some(p) = phased { + return p; + } + ( + Verdict::PossiblyBiallelic, + "phase unknown, possibly in trans (no read-backed phase links the two calls)".into(), + ) +} + +/// What [`two_hits`] finds. +#[derive(Debug, Default)] +pub struct TwoHits { + /// The best pair of each gene hit twice, in the order genes were found. + pub entries: IndexMap, + /// Variants called homozygous inside a deletion (hemizygous): the + /// deletion's text. + pub hemi: HashMap<(String, VKey), String>, + /// The best pair of each variant that has one: ` with + /// : `. + pub pairs: HashMap<(String, VKey), String>, +} + +/// The genes hit twice. +pub fn two_hits(variants: &[Variant], dels: &HashMap>) -> TwoHits { + let mut out = TwoHits::default(); + let mut best: HashMap<(String, VKey), Verdict> = HashMap::new(); + for v in variants { + let k = (v.gene.clone(), v.key.clone()); + for dl in dels.get(&v.gene).map_or(&[][..], Vec::as_slice) { + let (verdict, why) = pair_phase(dl, v); + if verdict == Verdict::Biallelic && why.starts_with("the variant is called homozygous") + { + out.hemi.insert(k.clone(), dl.text.clone()); + } + if verdict == Verdict::Cis { + continue; + } + let e = Entry { + gene: v.gene.clone(), + verdict, + why: why.clone(), + deletion: dl.text.clone(), + variant: v.text.clone(), + genotype: if out.hemi.contains_key(&k) { + "hemizygous".into() + } else { + v.genotype.clone() + }, + allele: v.allele.clone(), + }; + match out.entries.get(&v.gene) { + Some(old) if old.verdict <= verdict => {} + _ => { + out.entries.insert(v.gene.clone(), e); + } + } + if best.get(&k).is_none_or(|b| verdict < *b) { + best.insert(k.clone(), verdict); + out.pairs.insert( + k.clone(), + format!("{} with {}: {why}", verdict.word(), dl.text), + ); + } + } + } + out +} + +// --- the deletions -------------------------------------------------------------------- + +/// One AnnotSV row as the two-hit and ACMG rules read it. +#[derive(Debug, Clone, PartialEq)] +pub struct SvRow { + /// `SV_chrom:SV_start-SV_end` as AnnotSV writes it. + pub sv: String, + pub sv_type: Option, + pub gt: Option, + pub ps: Option, + pub acmg_class: Option, + pub genes: Vec, + pub caller: Option, + /// Callers supporting it in step 22's consensus; `None` without one. + pub support: Option, + pub common: bool, +} + +impl SvRow { + /// `str(acmg_class)`. + pub fn class(&self) -> &str { + self.acmg_class.as_deref().unwrap_or("None") + } +} + +/// Step 05's checked class 4 and 5 table: its class 4 and 5 rows, `None` +/// when there is no table, or the error text the rules print. +pub fn read_annotsv( + at: Option<(String, String)>, + consensus: Option, +) -> Result>, String> { + let Some((p, _)) = at else { + return Ok(None); + }; + let size = std::fs::metadata(&p) + .map_err(|e| crate::os_error(&e, &p))? + .len(); + let rows = if size > 0 { + rows(&p).map_err(|e| e.text())? + } else { + Vec::new() + }; + const NEED: [&str; 9] = [ + "ACMG_class", + "Caller", + "Common", + "Exon_edge", + "Population_AF", + "SV_chrom", + "SV_end", + "SV_start", + "SV_type", + ]; + if let Some(r) = rows.first() + && NEED.iter().any(|k| r.get(k).is_none()) + { + let list: Vec = NEED.iter().map(|k| crate::repr(k)).collect(); + return Err(format!( + "{} lacks a column of [{}]", + crate::basename(&p), + list.join(", ") + )); + } + let support = consensus_support(&rows, consensus.as_deref()); + let py = |v: Option<&str>| v.unwrap_or("None").to_owned(); + let out = rows + .iter() + .zip(support) + .map(|(r, support)| { + let fmt = sample_format(r); + let some = |k: &str| { + fmt.get(k) + .filter(|v| !v.is_empty()) + .map(|v| (*v).to_owned()) + }; + SvRow { + sv: format!( + "{}:{}-{}", + py(cell(r, "SV_chrom")), + py(cell(r, "SV_start")), + py(cell(r, "SV_end")) + ), + sv_type: cell(r, "SV_type").map(str::to_owned), + gt: some("GT"), + ps: some("PS"), + acmg_class: cell(r, "ACMG_class").map(str::to_owned), + genes: text::cell_or(r, "Gene_name") + .split(';') + .filter(|g| !g.is_empty()) + .map(str::to_owned) + .collect(), + caller: cell(r, "Caller").map(str::to_owned), + support, + common: cell(r, "Common") == Some("yes"), + } + }) + .filter(|x| matches!(x.class(), "4" | "5")) + .collect(); + Ok(Some(out)) +} + +/// The first sample's FORMAT values of an AnnotSV row: the column after +/// FORMAT in the header, none when that is `Annotation_mode` or absent. +fn sample_format(r: &DictRow) -> HashMap<&str, &str> { + let keys: Vec<&str> = r.iter().map(|(k, _)| k).collect(); + let Some(i) = keys.iter().position(|k| *k == "FORMAT") else { + return HashMap::new(); + }; + match keys.get(i + 1) { + Some(&k) if k != "Annotation_mode" => { + text::format_map(text::cell_or(r, "FORMAT"), text::cell_or(r, k)) + } + _ => HashMap::new(), + } +} + +/// How far a consensus record may sit from an AnnotSV call, each end. +const CONSENSUS_BP: i128 = 1000; + +fn chrom_key(c: &str) -> &str { + if c.len() >= 3 && c[..3].eq_ignore_ascii_case("chr") { + &c[3..] + } else { + c + } +} + +fn info_value<'a>(info: &'a str, key: &str) -> &'a str { + info.split(';') + .find_map(|kv| { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + (k == key).then_some(v) + }) + .unwrap_or("") +} + +fn dup_or(t: &str) -> String { + let t = t.to_uppercase(); + if t.starts_with("DUP") { + "DUP".into() + } else { + t + } +} + +/// The support of each row from step 22's consensus: the nearest record of +/// the same contig and type within 1 kb at both ends; 1 without one; `None` +/// for every row when there is no consensus or it cannot be read. +fn consensus_support(rows: &[DictRow], path: Option<&str>) -> Vec> { + let none = || vec![None; rows.len()]; + let Some(path) = path.filter(|p| crate::is_file(p)) else { + return none(); + }; + let Ok(lines) = text::lines(path) else { + return none(); + }; + // (contig, type) to the records' (pos, end, supp). + type Records = HashMap<(String, String), Vec<(i128, i128, i128)>>; + let mut by: Records = HashMap::new(); + for l in lines.iter().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 8 { + continue; + } + let num = |s: &str| crate::int(s).ok(); + let Some(pos) = num(c[1]) else { continue }; + let end = match info_value(c[7], "END") { + "" => Some(pos), + e => num(e), + }; + let supp = match info_value(c[7], "SUPP") { + "" => Some(0), + e => num(e), + }; + let (Some(end), Some(supp)) = (end, supp) else { + continue; + }; + by.entry(( + chrom_key(c[0]).to_owned(), + dup_or(info_value(c[7], "SVTYPE")), + )) + .or_default() + .push((pos, end, supp)); + } + rows.iter() + .map(|r| { + let (Some(Ok(start)), Some(Ok(end))) = ( + cell(r, "SV_start").map(crate::int), + cell(r, "SV_end").map(crate::int), + ) else { + return Some(1); + }; + let key = ( + chrom_key(cell(r, "SV_chrom").unwrap_or("")).to_owned(), + dup_or(cell(r, "SV_type").unwrap_or("")), + ); + let mut best: Option<(i128, i128)> = None; + for &(p, e, supp) in by.get(&key).map_or(&[][..], Vec::as_slice) { + let (dp, de) = ((p - start).abs(), (e - end).abs()); + if dp <= CONSENSUS_BP && de <= CONSENSUS_BP && best.is_none_or(|(d, _)| dp + de < d) + { + best = Some((dp + de, supp)); + } + } + Some(best.map_or(1, |(_, s)| s)) + }) + .collect() +} + +/// What step 35 wrote that the rule reads. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Step35 { + /// The interval table's deletions and duplications; `None` without it. + pub intervals: Option>>, + /// The HBA table (metric to value); `None` without it. + pub hba: Option>, + /// Parascopy's resolved SMN1 copy number at exon 7. + pub smn1: Option, +} + +/// SMN1 exon 7, c.840, GRCh38 1-based. +pub const SMN1_EXON7: (&str, i128) = ("chr5", 70951946); +/// The tables in paralogs/ that are not Parascopy catalog tables. +const NOT_CATALOG_TABLES: [&str; 2] = ["hba", "paraphase"]; +const PARASCOPY_COLUMNS: [&str; 9] = [ + "chrom", + "start", + "end", + "agCN", + "agCN_filter", + "agCN_qual", + "psCN", + "psCN_qual", + "psCN_filter", +]; +const INTERVAL_COLUMNS: [&str; 9] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", +]; +const DOSAGE_COLUMNS: [&str; 6] = ["locus", "status", "copies", "estimate", "reason", "reading"]; + +fn has_all(path: &str, cols: &[&str]) -> Result { + let have = columns(path).map_err(|e| e.text())?; + Ok(cols.iter().all(|c| have.iter().any(|h| h == c))) +} + +fn short_row(path: &str, rows: &[DictRow], cols: &[&str]) -> Result<(), String> { + if let Some(i) = rows + .iter() + .position(|r| cols.iter().any(|k| cell(r, k).is_none())) + { + return Err(format!( + "{} line {} lacks a field", + crate::basename(path), + i + 2 + )); + } + Ok(()) +} + +/// Step 35's HBA or CYP21A2 table as metric to value; a table without a +/// status row reads as not assessed. +fn read_hba(path: &str) -> Result, String> { + let mut out = IndexMap::new(); + for l in text::lines(path).map_err(|e| e.text())? { + if let Some((k, v)) = l.split_once('\t') + && k != "metric" + { + out.insert(k.to_owned(), v.to_owned()); + } + } + if !matches!( + out.get("status").map(String::as_str), + Some("called" | "not assessed") + ) { + out = IndexMap::from([ + ("status".to_owned(), "not assessed".to_owned()), + ( + "message".to_owned(), + format!("not assessed: {} has no status row", crate::basename(path)), + ), + ]); + } + Ok(out) +} + +/// The psCN of SMN1 at exon 7 when Parascopy resolved it (filter PASS, +/// quality 20 or more, a whole number), as `parascopy_cn_at` reads it. +fn smn1_cn(rows: &[DictRow]) -> Option { + let (chrom, pos) = SMN1_EXON7; + for r in rows { + let (Some(c), Some(Ok(s)), Some(Ok(e))) = ( + cell(r, "chrom"), + cell(r, "start").map(crate::int), + cell(r, "end").map(crate::int), + ) else { + continue; + }; + let mut regions = vec![(c.to_owned(), s + 1, e)]; + for h in text::cell(r, "homologous_regions") + .filter(|v| !v.is_empty()) + .unwrap_or("*") + .split(',') + { + let h = pycompat::text::strip(h, None); + if let Some((hc, rest)) = h.split_once(':') { + let a: String = rest.chars().take_while(char::is_ascii_digit).collect(); + let rest = &rest[a.len()..]; + if let Some(rest) = rest.strip_prefix('-') { + let b: String = rest.chars().take_while(char::is_ascii_digit).collect(); + if !hc.is_empty() && !a.is_empty() && !b.is_empty() { + regions.push((hc.to_owned(), crate::int(&a).ok()?, crate::int(&b).ok()?)); + } + } + } + } + let Some(idx) = regions + .iter() + .position(|(c, b, e)| c == chrom && *b <= pos && pos <= *e) + else { + continue; + }; + let cns: Vec<&str> = text::cell_or(r, "psCN").split(',').collect(); + let quals: Vec<&str> = text::cell_or(r, "psCN_qual").split(',').collect(); + let cn = cns.get(idx).copied().unwrap_or("?"); + let q = quals + .get(idx) + .and_then(|q| pycompat::num::parse_float(q).ok()); + let filt = cell(r, "psCN_filter") + .filter(|v| !v.is_empty()) + .unwrap_or("."); + let ok = filt == "PASS" && q.is_some_and(|q| q >= 20.0) && isdigit(cn); + return if ok { crate::int(cn).ok() } else { None }; + } + None +} + +/// What the rule takes from step 35's tables, in the order the Parascopy +/// section reads them, so the same fault stops it. `Ok(None)`: step 35 +/// wrote no table. +pub fn read_step35(dir: &str, s: &str, assets: &str) -> Result, String> { + let paralogs = crate::join(dir, "paralogs"); + let mut tables: Vec<(String, String)> = Vec::new(); + if let Ok(entries) = std::fs::read_dir(¶logs) { + for e in entries.flatten() { + let Ok(name) = e.file_name().into_string() else { + continue; + }; + let Some(t) = name + .strip_prefix(&format!("{s}_")) + .and_then(|n| n.strip_suffix("_copy_number.tsv")) + else { + continue; + }; + let p = crate::join(¶logs, &name); + if !NOT_CATALOG_TABLES.contains(&t) && crate::is_file(&p) { + tables.push((t.to_owned(), p)); + } + } + } + tables.sort_by(|a, b| pycompat::mtime::py_cmp(&a.1, &b.1)); + let side = |n: &str| crate::join(¶logs, &format!("{s}_{n}")); + let (hp, ip, cp, dp) = ( + side("hba_copy_number.tsv"), + side("cn_intervals.tsv"), + side("cyp21a2_psv.tsv"), + side("cn_dosage.tsv"), + ); + let smn = tables + .iter() + .find(|(t, _)| t == "smn") + .map(|(_, p)| p.clone()); + let any = + smn.is_some() || !tables.is_empty() || [&hp, &ip, &cp, &dp].iter().any(crate::is_file); + if !any { + return Ok(None); + } + let mut smn1 = None; + if let Some(p) = &smn { + let r = rows(p).map_err(|e| e.text())?; + if !has_all(p, &PARASCOPY_COLUMNS)? { + return Err(format!( + "{} lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or \ + psCN_filter column", + crate::basename(p) + )); + } + smn1 = smn1_cn(&r); + } + let others: Vec<&(String, String)> = tables.iter().filter(|(t, _)| t != "smn").collect(); + if !others.is_empty() { + let cat = super::assets::asset_rows(assets, "paralog_loci.tsv")?; + for r in &cat { + if r.get("table").is_none() { + return Err("'table'".into()); + } + } + } + let hba = if crate::is_file(&hp) { + Some(read_hba(&hp)?) + } else { + None + }; + let intervals = if crate::is_file(&ip) { + let r = rows(&ip).map_err(|e| e.text())?; + if !has_all(&ip, &INTERVAL_COLUMNS)? { + return Err(format!( + "{} lacks the locus, name, status, copies, expected or change column", + crate::basename(&ip) + )); + } + short_row(&ip, &r, &INTERVAL_COLUMNS)?; + Some( + r.iter() + .filter(|x| matches!(cell(x, "change"), Some("deletion" | "duplication"))) + .map(|x| { + [ + "locus", "name", "chrom", "start", "end", "copies", "expected", "change", + ] + .iter() + .map(|k| ((*k).to_owned(), text::cell_or(x, k).to_owned())) + .collect() + }) + .collect(), + ) + } else { + None + }; + if crate::is_file(&cp) { + read_hba(&cp)?; + } + if crate::is_file(&dp) { + if !has_all(&dp, &DOSAGE_COLUMNS)? { + return Err(format!( + "{} lacks the {} or reading column", + crate::basename(&dp), + DOSAGE_COLUMNS[..5].join(", ") + )); + } + let r = rows(&dp).map_err(|e| e.text())?; + short_row(&dp, &r, &DOSAGE_COLUMNS)?; + } + for (_, p) in others { + rows(p).map_err(|e: ReadError| e.text())?; + columns(p).map_err(|e| e.text())?; + } + Ok(Some(Step35 { + intervals, + hba, + smn1, + })) +} + +/// The deletion inputs of the sample, read once for every rule that needs +/// them. +#[derive(Debug, Clone)] +pub struct DeletionInputs { + pub annotsv: Result>, String>, + pub step35: Result, String>, +} + +/// The genes a called deletion reaches: the deletions of each gene, the +/// engines read, and the engines not read with why. +pub fn gene_deletions( + inp: &DeletionInputs, + spans: &Spans, +) -> (HashMap>, Vec, Vec) { + let mut out: HashMap> = HashMap::new(); + let (mut read, mut unread) = (Vec::new(), Vec::new()); + let over = |chrom: &str, a: i128, b: i128| -> Vec { + spans + .iter() + .filter(|(_, (c, gs, ge))| c == chrom && *gs <= b && *ge >= a) + .map(|(g, _)| g.clone()) + .collect() + }; + let mut add = |g: String, x: Deletion| out.entry(g).or_default().push(x); + match &inp.annotsv { + Err(e) => unread.push(format!("AnnotSV (step 05): {e}")), + Ok(None) => unread.push("AnnotSV (step 05): no table".into()), + Ok(Some(rows)) => { + read.push("AnnotSV class 4 and 5 deletions (step 05)".into()); + for x in rows { + let (chrom, span) = x.sv.split_once(':').unwrap_or((&x.sv, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + if x.sv_type.as_deref() != Some("DEL") || x.common || !(isdigit(a) && isdigit(b)) { + continue; + } + // A call genotyped 0/0 is no deletion in this sample. + if let Some(gt) = x.gt.as_deref() + && gt.replace('|', "/").split('/').all(|a| a == "0") + { + continue; + } + let chrom = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let (Ok(a_n), Ok(b_n)) = (crate::int(a), crate::int(b)) else { + continue; + }; + // SV_start is the base before the deleted bases. + let first = a_n + 1; + let mut genes: BTreeSet = over(&chrom, first, b_n).into_iter().collect(); + genes.extend(x.genes.iter().filter(|g| !spans.contains_key(*g)).cloned()); + for g in genes { + add( + g, + Deletion { + chrom: chrom.clone(), + start: first, + end: b_n, + inside: true, + text: format!( + "DEL {chrom}:{a}-{b} (AnnotSV class {}, {})", + x.class(), + x.caller.as_deref().unwrap_or("None") + ), + gt: x.gt.clone(), + ps: x.ps.clone(), + }, + ); + } + } + } + } + let st = match &inp.step35 { + Ok(Some(st)) => st, + Ok(None) => { + unread.push("step 35's copy-number tables: none".into()); + return (out, read, unread); + } + Err(e) => { + unread.push(format!("step 35's copy-number tables: {e}")); + return (out, read, unread); + } + }; + match &st.intervals { + Some(rows) => { + read.push("step 35's interval table (read depth)".into()); + for r in rows { + let f = |k: &str| r.get(k).map_or("", String::as_str); + if f("change") != "deletion" || !(isdigit(f("start")) && isdigit(f("end"))) { + continue; + } + let (Ok(a), Ok(b)) = (crate::int(f("start")), crate::int(f("end"))) else { + continue; + }; + for g in over(f("chrom"), a + 1, b) { + add( + g, + Deletion { + chrom: f("chrom").to_owned(), + start: a + 1, + end: b, + inside: true, + gt: None, + ps: None, + text: format!( + "{} deletion, {} of {} copies (read depth, step 35)", + f("locus"), + f("copies"), + f("expected") + ), + }, + ); + } + } + } + None => unread.push("step 35's interval table: none".into()), + } + let hba = st.hba.clone().unwrap_or_default(); + let n = hba.get("alpha_globin_genes").map_or("", String::as_str); + if hba.get("status").map(String::as_str) == Some("called") && isdigit(n) { + read.push("step 35's HBA1/HBA2 count".into()); + // 0 genes: no alpha-globin gene is left to carry a variant. + if digits(Some(n)).is_some_and(|k| (1..4).contains(&k)) { + for g in ["HBA1", "HBA2"] { + if let Some((c, a, b)) = spans.get(g) { + add( + g.to_owned(), + Deletion { + chrom: c.clone(), + start: *a, + end: *b, + inside: false, + gt: None, + ps: None, + text: format!("{n} of 4 alpha-globin genes (read depth, step 35)"), + }, + ); + } + } + } + } else { + unread.push(format!( + "step 35's HBA1/HBA2 count: {}", + hba.get("message") + .filter(|m| !m.is_empty()) + .map_or("not called", String::as_str) + )); + } + match st.smn1 { + Some(cn) => { + read.push("Parascopy's SMN1 count (step 35)".into()); + if cn == 1 + && let Some((c, a, b)) = spans.get("SMN1") + { + add( + "SMN1".into(), + Deletion { + chrom: c.clone(), + start: *a, + end: *b, + inside: true, + gt: None, + ps: None, + text: "one SMN1 copy (Parascopy, step 35)".into(), + }, + ); + } + } + None => unread.push("Parascopy's SMN1 count: not resolved".into()), + } + (out, read, unread) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn del(gt: Option<&str>, ps: Option<&str>) -> Deletion { + Deletion { + chrom: "chr1".into(), + start: 100, + end: 200, + text: "DEL".into(), + inside: true, + gt: gt.map(str::to_owned), + ps: ps.map(str::to_owned), + } + } + + fn var(pos: i128, gt: &str, ps: Option<&str>) -> Variant { + let (copies, ploidy) = super::super::clinvar::allele_copies(gt); + Variant { + gene: "G".into(), + key: ("chr1".into(), pos, "A".into(), "T".into()), + chrom: "chr1".into(), + pos, + text: format!("chr1:{pos} A>T"), + genotype: "het".into(), + copies, + ploidy, + gt: gt.into(), + ps: ps.map(str::to_owned), + allele: None, + } + } + + #[test] + fn phase_reads_only_called_phased_copies() { + assert_eq!(phase_haps(Some("0|1")), Some(BTreeSet::from([1]))); + assert_eq!(phase_haps(Some("1|0")), Some(BTreeSet::from([0]))); + assert_eq!(phase_haps(Some("0/1")), None); + assert_eq!(phase_haps(Some(".|1")), None); + assert_eq!(phase_haps(None), None); + } + + #[test] + fn read_backed_phase_decides_outside_the_deletion() { + // Outside the deletion's span: phase alone decides. + let trans = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "1|0", Some("7"))); + assert_eq!(trans.0, Verdict::Biallelic); + assert!(trans.1.starts_with("in trans")); + let cis = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "0|1", Some("7"))); + assert_eq!(cis.0, Verdict::Cis); + // Another phase set links nothing. + let other = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "1|0", Some("8"))); + assert_eq!(other.0, Verdict::PossiblyBiallelic); + assert!(other.1.starts_with("phase unknown")); + } + + #[test] + fn a_phased_cis_pair_is_no_second_hit_even_inside_the_deletion() { + let v = var(150, "0|1", Some("7")); + let cis = pair_phase(&del(Some("0|1"), Some("7")), &v); + assert_eq!(cis.0, Verdict::Cis); + let found = two_hits( + &[v], + &HashMap::from([("G".to_owned(), vec![del(Some("0|1"), Some("7"))])]), + ); + assert!(found.entries.is_empty()); + } + + #[test] + fn homozygous_inside_is_hemizygous() { + let v = var(150, "1/1", None); + let found = two_hits( + std::slice::from_ref(&v), + &HashMap::from([("G".to_owned(), vec![del(None, None)])]), + ); + let e = &found.entries["G"]; + assert_eq!(e.verdict, Verdict::Biallelic); + assert_eq!(e.genotype, "hemizygous"); + assert!(found.hemi.contains_key(&("G".to_owned(), v.key.clone()))); + } + + #[test] + fn the_best_pair_of_a_gene_wins() { + let v = var(500, "1|0", Some("7")); + let found = two_hits( + std::slice::from_ref(&v), + &HashMap::from([( + "G".to_owned(), + vec![del(None, None), del(Some("0|1"), Some("7"))], + )]), + ); + assert_eq!(found.entries["G"].verdict, Verdict::Biallelic); + assert!(found.pairs[&("G".to_owned(), v.key)].starts_with("biallelic with DEL: in trans")); + } +} From 2ad137d16f65f329b380dccce19c08bb05408a71 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:44:05 +0100 Subject: [PATCH 05/24] feat(summary): read the summary's group c sections in Rust og summary collect --only c reads copy number, HLA, the polygenic scores, ancestry, structural variants, repeats, telomeres, runs of homozygosity, the haplogroups, the opt-in step tables, chrM, traits with the APOL1 G2 deletion, and the couple screen, with the parts of the post pass that change them. og roh summary writes step 11's summary table; step 11 and the ROH module run it, so the bands live in one const. Bead: pgp-bs9u.43 --- modules/local/roh/main.nf | 23 +- rust/Cargo.lock | 3 + rust/crates/og-summary/Cargo.toml | 3 + rust/crates/og-summary/src/collect.rs | 125 +- rust/crates/og-summary/src/collect/group_c.rs | 278 +++ .../og-summary/src/collect/group_c/chrm.rs | 474 +++++ .../og-summary/src/collect/group_c/cn.rs | 743 ++++++++ .../og-summary/src/collect/group_c/hla.rs | 396 +++++ .../og-summary/src/collect/group_c/repeats.rs | 902 ++++++++++ .../og-summary/src/collect/group_c/roh.rs | 419 +++++ .../og-summary/src/collect/group_c/sv.rs | 461 +++++ .../og-summary/src/collect/group_c/tables.rs | 1536 +++++++++++++++++ .../og-summary/src/collect/group_c/tests.rs | 0 .../og-summary/src/collect/group_c/traits.rs | 570 ++++++ .../og-summary/src/collect/group_c/util.rs | 387 +++++ rust/crates/og-summary/src/collect/scan.rs | 17 + .../crates/og-summary/src/collect/sections.rs | 5 +- rust/crates/og-summary/src/collect/tests.rs | 1 + rust/og/src/main.rs | 10 +- scripts/11-roh-analysis.sh | 37 +- tests/test_collect_summary.py | 34 +- 21 files changed, 6348 insertions(+), 76 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/group_c.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/chrm.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/cn.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/hla.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/repeats.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/roh.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/sv.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/tables.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/tests.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/traits.rs create mode 100644 rust/crates/og-summary/src/collect/group_c/util.rs diff --git a/modules/local/roh/main.nf b/modules/local/roh/main.nf index bf7b9c3ca..86df6a0e2 100644 --- a/modules/local/roh/main.nf +++ b/modules/local/roh/main.nf @@ -45,24 +45,11 @@ process ROH { | bcftools roh \${ROH_FLAGS} -o ${meta.id}_roh.txt - # Summary: autosomal segments with 5 Mb or more outside GRCh38's - # centromere and heterochromatin bands (the acen, gvar and stalk rows of - # UCSC's cytoBand.hg38.txt, merged; the line of scripts/11-roh-analysis.sh - # and bin/collect_summary.py), the threshold of docs/11-roh-analysis.md; - # length_bp and length_mb are of that part. The bash step writes the same - # file. - MASKED_BANDS="chr1:121700000-143200000 chr2:91800000-96000000 chr3:87800000-98600000 chr4:48200000-51800000 chr5:46100000-51400000 chr6:58500000-62600000 chr7:58100000-62100000 chr8:43200000-47200000 chr9:42200000-61500000 chr10:38000000-41600000 chr11:51000000-55800000 chr12:33200000-37800000 chr13:0-18900000 chr14:0-18200000 chr15:0-20500000 chr16:35300000-47000000 chr17:22700000-27400000 chr18:15400000-21500000 chr19:19900000-31900000 chr20:25700000-30400000 chr21:0-13000000 chr22:0-17400000 chrX:58100000-63800000 chrY:10300000-10600000 chrY:26600000-57227415" - echo "# ROH Summary for ${meta.id}" > ${meta.id}_roh_summary.txt - echo "# Segments >=5MB on autosomes (excludes chrX/chrY)" >> ${meta.id}_roh_summary.txt - printf 'chrom\\tstart\\tend\\tlength_bp\\tlength_mb\\n' >> ${meta.id}_roh_summary.txt - awk -v bands="\$MASKED_BANDS" ' - BEGIN { n = split(bands, b, " "); for (i = 1; i <= n; i++) { split(b[i], x, /[:-]/); c[i] = x[1]; s[i] = x[2]; e[i] = x[3] } } - \$1 == "RG" && \$3 ~ /^chr([1-9]|1[0-9]|2[0-2])\$/ { - m = 0 - for (i = 1; i <= n; i++) if (c[i] == \$3) { o = (\$5 < e[i] ? \$5 : e[i]) - (\$4 - 1 > s[i] ? \$4 - 1 : s[i]); if (o > 0) m += o } - if (m > \$6) m = \$6 - l = \$6 - m - if (l >= 5000000) printf "%s\\t%s\\t%s\\t%d\\t%.1f\\n", \$3, \$4, \$5, l, l / 1e6 - }' ${meta.id}_roh.txt >> ${meta.id}_roh_summary.txt + # centromere and heterochromatin bands (MASKED_BANDS in og's step 11 + # code, the one copy), the threshold of docs/11-roh-analysis.md; + # length_bp and length_mb are of that part. The bash step runs the same + # command. + og roh summary --sample ${meta.id} --roh ${meta.id}_roh.txt --out ${meta.id}_roh_summary.txt cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d91a18ca..cdb0c64d0 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -839,6 +839,9 @@ dependencies = [ "flate2", "indexmap", "memchr", + "og-ancestry", + "og-chip", + "og-cn", "og-io", "og-steps", "pycompat", diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 1a7fa879c..c8191be00 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -10,6 +10,9 @@ publish.workspace = true pycompat.workspace = true og-io.workspace = true og-steps.workspace = true +og-ancestry.workspace = true +og-chip.workspace = true +og-cn.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 0ae779232..d85a9e5c2 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -7,7 +7,10 @@ //! switches to it. Group `a` is the model and the first seven sections: //! coverage, callability, sex_check, sample_qc, mosaic_sex_loss, //! read_count and variants, with the run status, the manifest and each -//! section's state. +//! section's state. Group `c` is copy number, HLA, the polygenic scores, +//! ancestry, structural variants, repeats, telomeres, runs of +//! homozygosity, the haplogroups, the opt-in step tables, chrM, traits with +//! the APOL1 G2 deletion, and the couple screen ([`group_c`]). //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -16,6 +19,7 @@ //! sections are read. A reader's error stays in its section, which is //! then `unreadable` with a note that names the file. +pub mod group_c; pub mod model; pub mod scan; pub mod sections; @@ -29,7 +33,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only a +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,c} --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] "; @@ -39,7 +43,7 @@ const OPTS: &[Opt] = &[ Opt("--declared-sex", Kind::Value, false), Opt("--out", Kind::Value, true), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a"]), true), + Opt("--only", Kind::Choice(&["a", "c"]), true), ]; /// What one collect reads. @@ -52,6 +56,15 @@ pub struct Input { pub declared_sex: Option, /// The repository's `assets/` folder. pub assets: String, + /// The group of sections to read: `a` or `c`. + pub group: Group, +} + +/// The group of sections a collect reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Group { + A, + C, } /// `og summary collect`: exit 0 when summary.json is written (whatever @@ -71,6 +84,11 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { sample_dir: a.text("--sample-dir").to_owned(), declared_sex: a.get("--declared-sex").map(str::to_owned), assets: a.get("--assets").map_or_else(default_assets, str::to_owned), + group: if a.text("--only") == "c" { + Group::C + } else { + Group::A + }, }; // A part of the sections must never replace the sample's complete // summary.json, so there is no default path while `--only` exists. @@ -127,6 +145,28 @@ pub fn collect(input: &Input) -> Result { let s = pycompat::mtime::escape(&input.sample); let find = |pats: &[String]| index.first(pats).map(str::to_owned); let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; + if input.group == Group::C { + // The declared sex the chrY filter of CNVpytor reads comes from + // step 33's table when the run status and manifest have none. + let qc = find(&[format!("qc/{s}_sample_qc.tsv")]).map(file); + let qc_declared = qc.and_then(|(p, _)| { + let rows = sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared.as_deref()); + let sections = read_group_c(input, &index, run.as_ref(), declared_sex.as_deref()); + return Ok(Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now()).unwrap_or_default(), + run, + manifest, + declared_sex, + sections, + }); + } let variants_at = find(&[format!("vcf/{s}.vcf.gz")]); let (variants, rest) = std::thread::scope(|scope| { @@ -172,25 +212,10 @@ pub fn collect(input: &Input) -> Result { Some(Data::SampleQc(q)) => q.declared_sex(), _ => None, }); - let declared_sex = [ - input.declared_sex.as_deref(), - run.as_ref().and_then(|r| r.declared_sex.as_deref()), - manifest - .as_ref() - .and_then(|m| m.value("run", "declared_sex")), - qc_declared, - ] - .into_iter() - .flatten() - .find(|v| !v.is_empty()) - .map(str::to_owned); - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0.0, |d| d.as_secs_f64()); + let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared); Ok(Summary { sample: input.sample.clone(), - generated_utc: time::utc_minute(now).unwrap_or_default(), + generated_utc: time::utc_minute(now()).unwrap_or_default(), run, manifest, declared_sex, @@ -198,6 +223,66 @@ pub fn collect(input: &Input) -> Result { }) } +fn now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) +} + +/// The declared sex: the command line's, the run status's, the +/// manifest's, then step 33's; the first that is not empty. +fn declared( + input: &Input, + run: Option<&RunStatus>, + manifest: Option<&Manifest>, + qc: Option<&str>, +) -> Option { + [ + input.declared_sex.as_deref(), + run.and_then(|r| r.declared_sex.as_deref()), + manifest.and_then(|m| m.value("run", "declared_sex")), + qc, + ] + .into_iter() + .flatten() + .find(|v| !v.is_empty()) + .map(str::to_owned) +} + +/// Reads the sections of group c, then runs the parts of the post pass +/// that change them. +fn read_group_c( + input: &Input, + index: &FolderIndex, + run: Option<&RunStatus>, + declared_sex: Option<&str>, +) -> Vec
{ + let c = group_c::Ctx { + sample: &input.sample, + dir: &input.sample_dir, + assets: &input.assets, + index, + }; + let reads = group_c::read_all(&c); + let mut sections: Vec
= group_c::GROUP_C + .iter() + .zip(reads) + .map(|(spec, r)| { + let r = r.and_then(|found| match found { + None => Ok(None), + Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), + }); + Section::new(*spec, r, run) + }) + .collect(); + let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections + .iter_mut() + .map(|s| (s.spec, s.state, s.data.as_mut())) + .collect(); + group_c::post(&c, &mut view, &group_c::Others::default(), declared_sex); + sections +} + /// What one reader gave: the file (relative), its mtime and its data. type Read = Result, String>; diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs new file mode 100644 index 000000000..75ef612d8 --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -0,0 +1,278 @@ +//! Group c of `og summary collect`: copy number, HLA, polygenic scores, +//! structural variants, repeats, telomeres, runs of homozygosity, +//! haplogroups, the step tables of the opt-in steps, chrM, traits with the +//! APOL1 G2 deletion, and the couple screen. +//! +//! Each section is read on its own, and an error stays in its section. The +//! three parts of the Python collector's post pass that change these +//! sections' data run here too, after every section is read: the repeat +//! readings with the traits and CPIC states, Locityper's primary calls, and +//! the chrY filter of CNVpytor's count for a declared female. What they +//! need from group b (the CPIC state, pypgx and Cyrius) comes in +//! [`Others`]; `--only c` reads none of it, so it is empty there. + +mod chrm; +mod cn; +mod hla; +mod repeats; +pub mod roh; +mod sv; +mod tables; +mod traits; +pub mod util; + +use super::model::{Spec, State}; +use super::scan::FolderIndex; +use super::sections::Data; +use pycompat::json::Value; + +/// The sections of group c, in summary.json's order. +pub const GROUP_C: [Spec; 32] = [ + spec("couple", "Couple carrier screen", "39"), + spec( + "parascopy", + "SMN1/SMN2 copy number (Parascopy, short reads)", + "35", + ), + spec("paralogs", "Paralog genes (Paraphase, PacBio HiFi)", "35b"), + spec("hla", "HLA typing (T1K)", "08"), + spec("prs", "Polygenic risk scores", "25"), + spec( + "ancestry_fine", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "26b", + ), + spec("manta", "Structural variants (Manta)", "04"), + spec("delly", "Structural variants (Delly)", "19"), + spec("cnvpytor", "Copy number (CNVpytor)", "18"), + spec("sv_consensus", "SV consensus merge", "22"), + spec( + "annotsv", + "Structural variants, class 4 and 5 (AnnotSV)", + "05", + ), + spec("expansions", "Repeat expansions (ExpansionHunter)", "09"), + spec("telomere", "Telomere content (TelomereHunter)", "10"), + spec("roh", "Runs of homozygosity", "11"), + spec("haplogroup", "Mitochondrial haplogroup", "12"), + spec("y_haplogroup", "Y-chromosome haplogroup (Yleaf)", "37"), + spec("y_tree", "Y placement on the current public tree", "37"), + spec("y_str", "Y-STR markers (HipSTR)", "45"), + spec( + "consumer_export", + "Raw-data export (23andMe v5 layout)", + "40", + ), + spec("bloodtype", "Blood groups (predicted)", "41"), + spec("lpa", "Lp(a): LPA KIV-2 repeat count", "46"), + spec("vntyper", "MUC1 VNTR frameshifts (VNtyper)", "52"), + spec("locityper", "Locityper second opinion at hard loci", "53"), + spec("mito", "Mitochondrial variants", "20"), + spec("traits", "Traits panel", "42"), + spec("archaic", "Archaic ancestry (IBDmix)", "43"), + spec("ancient", "Ancient populations (AADR PCA)", "44"), + spec("viral", "Viral DNA in the unmapped reads", "47"), + spec("exomiser", "Symptom-driven ranking (Exomiser)", "48"), + spec("chip", "Clonal haematopoiesis (CHIP)", "49"), + spec("mei", "Mobile element insertions (MEGAnE)", "50"), + spec("mosaic", "Mosaic chromosomal alterations (MoChA)", "51"), +]; + +const fn spec(key: &'static str, title: &'static str, step: &'static str) -> Spec { + Spec { key, title, step } +} + +/// A file a section found: its path and its path relative to the folder. +#[derive(Debug, Clone)] +pub struct Found { + pub path: String, + pub rel: String, +} + +/// What a reader sees: the sample, its folder, the assets and one walk of +/// the folder. +pub struct Ctx<'a> { + pub sample: &'a str, + pub dir: &'a str, + pub assets: &'a str, + pub index: &'a FolderIndex, +} + +impl Ctx<'_> { + /// The sample name as a file pattern matches it literally. + pub fn s(&self) -> String { + pycompat::mtime::escape(self.sample) + } + + /// `first_existing(d, patterns)`. + pub fn find(&self, pats: &[String]) -> Option { + self.index.first(pats).map(|rel| Found { + path: crate::join(self.dir, rel), + rel: rel.to_owned(), + }) + } + + /// `first_existing` of one pattern. + pub fn find1(&self, pat: String) -> Option { + self.find(&[pat]) + } + + /// `os.path.join(d, rel)`. + pub fn path(&self, rel: &str) -> String { + crate::join(self.dir, rel) + } + + /// `os.path.join(os.path.dirname(found), name)` when it is a file. + pub fn beside(&self, f: &Found, name: &str) -> Option { + let dir = |p: &str| p.rsplit_once('/').map_or(String::new(), |x| x.0.to_owned()); + let path = crate::join(&dir(&f.path), name); + crate::is_file(&path).then(|| Found { + rel: crate::join(&dir(&f.rel), name), + path, + }) + } + + /// `assets/`. + pub fn asset(&self, name: &str) -> String { + crate::join(self.assets, name) + } +} + +/// What a section reader gives: the file, its data, and for the sections +/// the post pass reads, what it needs, typed. +pub type Out = Result, String>; + +/// The states and data of group b the post pass reads. Empty under +/// `--only c`, which reads no group b section. +#[derive(Debug, Default, Clone)] +pub struct Others { + pub cpic: Option, + pub traits: Option, + /// pypgx's CYP2D6 call and its section state, when that section is ok + /// or stale. + pub pypgx: Option<(State, String)>, + /// Cyrius's genotype and Filter (either may be absent), when that + /// section is ok or stale. + pub cyrius: Option<(State, Option, Option)>, +} + +/// Reads every section of group c, in [`GROUP_C`] order. +pub fn read_all(c: &Ctx<'_>) -> Vec { + let readers: [fn(&Ctx<'_>) -> Out; 32] = [ + tables::couple, + cn::parascopy, + cn::paralogs, + hla::hla, + tables::prs, + tables::ancestry_fine, + sv::manta, + sv::delly, + sv::cnvpytor, + sv::sv_consensus, + sv::annotsv, + repeats::expansions, + tables::telomere, + roh::section, + tables::haplogroup, + tables::y_haplogroup, + tables::y_tree, + tables::y_str, + tables::consumer_export, + tables::bloodtype, + tables::lpa, + tables::vntyper, + tables::locityper, + chrm::mito, + traits::traits, + tables::archaic, + tables::ancient, + tables::viral, + tables::exomiser, + tables::chip, + tables::mei, + tables::mosaic, + ]; + // The readers are independent: one thread each, over a small pool, so a + // big input (a VCF, a table of thousands of rows) never holds up the + // others. A panic stays in its section. + let jobs = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); + let next = std::sync::atomic::AtomicUsize::new(0); + let slots: Vec>> = + (0..readers.len()).map(|_| std::sync::Mutex::new(None)).collect(); + std::thread::scope(|scope| { + for _ in 0..jobs { + scope.spawn(|| { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(r) = readers.get(i) else { break }; + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r(c))) + .unwrap_or_else(|_| Err("internal error: the reader stopped".to_owned())); + if let Ok(mut slot) = slots[i].lock() { + *slot = Some(out); + } + } + }); + } + }); + slots + .into_iter() + .map(|m| { + m.into_inner() + .ok() + .flatten() + .unwrap_or_else(|| Err("internal error: the reader stopped".to_owned())) + }) + .collect() +} + +/// The post pass over the read sections (`sections` in [`GROUP_C`] +/// order, each with its state and data): the repeat readings with the +/// traits and CPIC states, each Locityper locus's primary call, and the +/// chrY filter of CNVpytor's count. +pub fn post( + c: &Ctx<'_>, + sections: &mut [(Spec, State, Option<&mut Data>)], + others: &Others, + declared_sex: Option<&str>, +) { + let state = |key: &str, secs: &[(Spec, State, Option<&mut Data>)]| { + secs.iter() + .find(|(s, ..)| s.key == key) + .map_or(State::Missing, |x| x.1) + }; + let traits_state = others + .traits + .unwrap_or_else(|| state("traits", sections)); + let cpic_state = others.cpic.unwrap_or(State::Ok); + let hla_state = state("hla", sections); + let hla_data = sections + .iter() + .find(|(s, ..)| s.key == "hla") + .and_then(|x| x.2.as_deref()) + .and_then(json) + .cloned(); + for (spec, st, data) in sections.iter_mut() { + let live = matches!(st, State::Ok | State::Stale); + let Some(Data::Json(v)) = data.as_deref_mut() else { + continue; + }; + match spec.key { + "expansions" if live => repeats::post(c, v, traits_state, cpic_state), + "locityper" if live => { + tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + } + "cnvpytor" => sv::cnvpytor_sex_filter(v, declared_sex), + _ => {} + } + } +} + +fn json(d: &Data) -> Option<&Value> { + match d { + Data::Json(v) => Some(v), + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-summary/src/collect/group_c/chrm.rs b/rust/crates/og-summary/src/collect/group_c/chrm.rs new file mode 100644 index 000000000..4954a3302 --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/chrm.rs @@ -0,0 +1,474 @@ +//! Step 20's chrM section: the PASS calls and the heteroplasmic ones, the +//! poly-C tracts counted apart, the annotation table with its ClinVar +//! classes, the confirmed variants of `assets/mito_confirmed.tsv`, the large +//! deletions and the mtDNA copy number. + +use super::util::{Obj, asset_rows, cells, lines, opt_s, pairs, round, s, strs, vcf_lines}; +use super::{Ctx, Found, Out}; +use crate::collect::sections::py_error; +use indexmap::IndexMap; +use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::json::Value; + +/// The D-loop poly-C tracts, rCRS m.303-315 and m.16184-16193. +const POLYC_TRACTS: [(i128, i128); 2] = [(303, 315), (16184, 16193)]; +const MITO_ANNOT_COLS: [&str; 13] = [ + "chrom", + "pos", + "ref", + "alt", + "filter", + "af", + "clinvar_id", + "clinvar_significance", + "clinvar_review", + "clinvar_condition", + "gnomad_af_hom", + "gnomad_af_het", + "gnomad_max_hl", +]; +const CLINVAR_PLP: [&str; 3] = [ + "Pathogenic", + "Likely_pathogenic", + "Pathogenic/Likely_pathogenic", +]; +const CLINVAR_BENIGN: [&str; 4] = [ + "Benign", + "Likely_benign", + "Benign/Likely_benign", + "not_provided", +]; +const MITO_DEL_TOLERANCE: i128 = 50; +const MITO_DEL_LIST_PCT: f64 = 1.0; +const MITO_CN_COLS: [&str; 8] = [ + "status", + "reason", + "chrm_reads", + "chrm_length", + "autosomal_reads", + "autosomal_length", + "depth_ratio", + "mtdna_copies", +]; +const HET_LOW: f64 = 0.05; +const HET_HIGH: f64 = 0.95; + +fn n_chars(x: &str) -> i128 { + x.chars().count() as i128 +} + +/// The rCRS positions a chrM allele changes. +fn mito_span(pos: i128, r: &str, a: &str) -> (i128, i128) { + if n_chars(r) == n_chars(a) { + (pos, pos + n_chars(r) - 1) + } else { + (pos + 1, pos + (n_chars(r) - 1).max(1)) + } +} + +pub fn in_polyc(pos: i128, r: &str, a: &str) -> bool { + let (lo, hi) = mito_span(pos, r, a); + POLYC_TRACTS.iter().any(|(a, b)| lo <= *b && hi >= *a) +} + +/// `(pos, ref, alt)` without the bases both share at the end, then at the +/// start, one base always kept. +pub fn min_allele(mut pos: i128, r: &str, a: &str) -> (i128, String, String) { + let mut r: Vec = r.chars().collect(); + let mut a: Vec = a.chars().collect(); + while r.len() > 1 && a.len() > 1 && r.last() == a.last() { + r.pop(); + a.pop(); + } + let mut i = 0; + while r.len() - i > 1 && a.len() - i > 1 && r[i] == a[i] { + i += 1; + pos += 1; + } + ( + pos, + r[i..].iter().collect(), + a[i..].iter().collect(), + ) +} + +fn py_int(x: Option<&str>) -> Result { + match x { + None => Err(py_error( + "TypeError", + "int() argument must be a string, a bytes-like object or a real number, not 'NoneType'", + )), + Some(x) => crate::int(x).map_err(|e| py_error("ValueError", &e)), + } +} + +fn py_float(x: Option<&str>) -> Result { + match x { + None => Err(py_error( + "TypeError", + "float() argument must be a string or a real number, not 'NoneType'", + )), + Some(x) => crate::float(x).map_err(|e| py_error("ValueError", &e)), + } +} + +/// `row[k]`, the column required. +fn col<'a>(r: &'a DictRow, k: &str) -> Result, String> { + super::util::at(r, k) +} + +fn confirmed_fields(k: &DictRow) -> Result { + let mut o = Obj::new(); + for (key, c) in [ + ("hgvs", "hgvs"), + ("gene", "gene"), + ("disease", "disease"), + ("clinvar_id", "clinvar_id"), + ("pmids", "pmids"), + ("note", "heteroplasmy_note"), + ] { + o.put(key, opt_s(col(k, c)?)); + } + Ok(o) +} + +/// Python's `max` of floats: the first, replaced by each later one that is +/// greater. +fn py_max(xs: &[f64]) -> Option { + let mut it = xs.iter().copied(); + let mut cur = it.next()?; + for x in it { + if x > cur { + cur = x; + } + } + Some(cur) +} + +type Confirmed = (IndexMap<(i128, String, String), DictRow>, Vec); + +fn mito_confirmed(assets: &str) -> Result, String> { + let rows = match asset_rows(assets, "mito_confirmed.tsv") { + Ok(r) => r, + // A missing assets folder is a note, not an unreadable section. + Err(e) if e.starts_with("mito_confirmed.tsv is not in ") => return Ok(Err(e)), + Err(e) => return Err(e), + }; + let mut snv = IndexMap::new(); + let mut dels = Vec::new(); + for r in rows { + match col(&r, "kind")? { + Some("snv") => { + let key = ( + py_int(col(&r, "pos")?)?, + col(&r, "ref")?.unwrap_or("").to_owned(), + col(&r, "alt")?.unwrap_or("").to_owned(), + ); + snv.insert(key, r); + } + Some("deletion") => dels.push(r), + _ => {} + } + } + Ok(Ok((snv, dels))) +} + +fn mito_extended(c: &Ctx<'_>, vcf: &Found) -> Result { + let mut out = Obj::new(); + let (snv, dels) = match mito_confirmed(c.assets)? { + Ok((snv, dels)) => (Some(snv), dels), + Err(note) => { + out.put("confirmed_note", s(¬e)); + (None, Vec::new()) + } + }; + if let Some(snv) = &snv { + let mut records = Vec::new(); + vcf_lines(&vcf.path, |l| records.push(cells(l)))?; + let mut confirmed = Vec::new(); + for r in records { + if r.len() < 8 { + continue; + } + let (fmt, val): (Vec<&str>, Vec<&str>) = if r.len() > 9 { + (r[8].split(':').collect(), r[9].split(':').collect()) + } else { + (Vec::new(), Vec::new()) + }; + let afs: Vec<&str> = match fmt.iter().position(|f| *f == "AF") { + Some(i) if i < val.len() => val[i].split(',').collect(), + _ => Vec::new(), + }; + let pos = py_int(Some(&r[1]))?; + for (i, alt) in r[4].split(',').enumerate() { + if let Some(k) = snv.get(&min_allele(pos, &r[3], alt)) { + confirmed.push( + confirmed_fields(k)? + .set("pos", Value::Int(py_int(col(k, "pos")?)?)) + .set("filter", s(&r[6])) + .set("af", s(afs.get(i).copied().unwrap_or("."))) + .val(), + ); + } + } + } + out.put("confirmed", Value::Array(confirmed)); + } + let s_ = c.s(); + match c.find1(format!("mito/{s_}_chrM_deletions.tsv")) { + None => out.put( + "deletions_note", + s("no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)"), + ), + Some(p) => { + let rows = super::util::table(&p.path, false)?.rows; + let mut listed = Vec::new(); + for r in &rows { + let bkp5 = py_int(col(r, "bkp5")?)?; + let bkp3 = py_int(col(r, "bkp3")?)?; + let length = match col(r, "length")? { + Some(".") => Value::Null, + x => Value::Int(py_int(x)?), + }; + let level = py_float(col(r, "level_pct")?)?; + let mut known = None; + for k in &dels { + let kp = py_int(col(k, "pos")?)?; + if (bkp5 + 1 - kp).abs() <= MITO_DEL_TOLERANCE { + let ke = py_int(col(k, "end")?)?; + if (bkp3 - 1 - ke).abs() <= MITO_DEL_TOLERANCE { + known = Some(k); + break; + } + } + } + if known.is_some() || level >= MITO_DEL_LIST_PCT { + let mut row = Obj::new() + .set("bkp5", Value::Int(bkp5)) + .set("bkp3", Value::Int(bkp3)) + .set("length", length) + .set("level_pct", Value::Float(level)); + if let Some(k) = known { + row.put("known", confirmed_fields(k)?.val()); + } + listed.push(row.val()); + } + } + out.put( + "deletions", + Obj::new() + .set("source", s(&p.rel)) + .set("total", super::util::count(rows.len())) + .set("listed", Value::Array(listed)) + .set("list_floor_pct", Value::Float(MITO_DEL_LIST_PCT)) + .val(), + ); + } + } + match c.find1(format!("mito/{s_}_mtdna_copies.tsv")) { + Some(p) => { + let m: IndexMap = pairs(&p.path)?.into_iter().collect(); + out.put( + "copies", + MITO_CN_COLS + .iter() + .fold(Obj::new(), |o, k| { + o.set(k, opt_s(m.get(*k).map(String::as_str).filter(|v| *v != "."))) + }) + .val(), + ); + } + None => out.put( + "copies_note", + s("no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)"), + ), + } + Ok(out) +} + +fn mtime(p: &str) -> Result { + pycompat::mtime::mtime(p).map_err(|e| super::util::os_note(&crate::os_error(&e, p))) +} + +/// The annotation table's sources and its PASS rows, or a note. +fn mito_annotation(c: &Ctx<'_>, vcf: &Found) -> Result<(Obj, Vec), String> { + let Some(p) = c.find1(format!("mito/{}_chrM_annotated.tsv", c.s())) else { + return Ok(( + Obj::new().set("note", s("no annotation table (mito/_chrM_annotated.tsv): step 20 ran before it annotated, or the Nextflow MITO_VARIANTS_ANNOTATE task failed (see the Nextflow log)")), + Vec::new(), + )); + }; + if mtime(&p.path)? < mtime(&vcf.path)? { + return Ok(( + Obj::new().set( + "note", + s(&format!( + "{} is older than the chrM calls: rerun step 20", + p.rel + )), + ), + Vec::new(), + )); + } + let mut sources: IndexMap = IndexMap::new(); + let mut body = Vec::new(); + for line in lines(&p.path)? { + if let Some(rest) = line.strip_prefix("# ") { + let c: Vec<&str> = rest.split('\t').collect(); + if c.len() >= 3 { + let v = if c[1] == "not annotated" { + format!("not annotated: {}", c[2]) + } else if c[2] != "-" { + format!("{} ({})", c[1], c[2]) + } else { + c[1].to_owned() + }; + sources.insert(c[0].to_owned(), v); + } + } else if !pycompat::num::py_strip(&line).is_empty() { + body.push(Ok::<_, std::io::Error>(line + "\n")); + } + } + let rows = DictReader::new(body.into_iter(), '\t', Quoting::Minimal) + .collect::, _>>() + .map_err(|e| py_error(e.exception(), &e.to_string()))? + .into_iter() + .filter(|r| r.get("filter").flatten() == Some("PASS")) + .collect(); + let src = |k: &str| opt_s(sources.get(k).map(String::as_str)); + Ok(( + Obj::new() + .set("source", s(&p.rel)) + .set("clinvar", src("clinvar")) + .set("gnomad", src("gnomad")), + rows, + )) +} + +fn clinvar_terms(sig: Option<&str>) -> Vec<&str> { + sig.unwrap_or("") + .split(['|', ',']) + .filter(|t| !t.is_empty() && *t != ".") + .collect() +} + +pub fn mito(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("mito/{}_chrM_filtered.vcf.gz", c.s())) else { + return Ok(None); + }; + let mut records = Vec::new(); + vcf_lines(&p.path, |l| records.push(cells(l)))?; + let (mut passed, mut het) = (0usize, 0usize); + let (mut polyc, mut sites) = (Vec::new(), Vec::new()); + for r in &records { + if r.len() < 10 || r[6] != "PASS" { + continue; + } + passed += 1; + let fmt: Vec<&str> = r[8].split(':').collect(); + let val: Vec<&str> = r[9].split(':').collect(); + let Some(i) = fmt.iter().position(|f| *f == "AF").filter(|i| *i < val.len()) else { + continue; + }; + let afs: Result, _> = val[i] + .split(',') + .filter(|x| !matches!(*x, "" | ".")) + .map(pycompat::num::parse_float) + .collect(); + let Some(af) = afs.ok().and_then(|a| py_max(&a)) else { + continue; + }; + if !(HET_LOW..HET_HIGH).contains(&af) { + continue; + } + let pos = py_int(Some(&r[1]))?; + if r[4].split(',').any(|a| in_polyc(pos, &r[3], a)) { + polyc.push( + Obj::new() + .set("variant", s(&format!("{}:{} {}>{}", r[0], r[1], r[3], r[4]))) + .set("af", round(af, 3)) + .val(), + ); + } else { + het += 1; + sites.push( + Obj::new() + .set("pos", Value::Int(pos)) + .set("ref", s(&r[3])) + .set("alt", s(&r[4])) + .set("af", round(af, 3)) + .val(), + ); + } + } + let (mut ann, rows) = mito_annotation(c, &p)?; + let mut listed = Vec::new(); + let mut plp = 0i128; + for r in &rows { + let terms = clinvar_terms(r.get("clinvar_significance").flatten()); + let is_plp = terms.iter().any(|t| CLINVAR_PLP.contains(t)); + plp += i128::from(is_plp); + let het_row = r + .get("af") + .flatten() + .and_then(|x| pycompat::num::parse_float(x).ok()) + .is_some_and(|af| (HET_LOW..HET_HIGH).contains(&af)); + if is_plp + || het_row + || (!terms.is_empty() && !terms.iter().all(|t| CLINVAR_BENIGN.contains(t))) + { + let mut o = Obj::new(); + for k in MITO_ANNOT_COLS { + if k != "chrom" && k != "filter" { + o.put(k, opt_s(r.get(k).flatten())); + } + } + let pos = py_int(col(r, "pos")?)?; + let rf = col(r, "ref")?.unwrap_or(""); + let al = col(r, "alt")?.unwrap_or(""); + o.put("polyc", Value::Bool(in_polyc(pos, rf, al))); + listed.push(o.val()); + } + } + if ann.0.contains_key("source") { + ann.put("variants", Value::Array(listed)); + ann.put("clinvar_plp", Value::Int(plp)); + } + let mut out = mito_extended(c, &p)?; + out.put("pass", super::util::count(passed)); + out.put("heteroplasmic", super::util::count(het)); + out.put("heteroplasmic_sites", Value::Array(sites)); + out.put("heteroplasmy_floor", Value::Float(HET_LOW)); + out.put("polyc_heteroplasmic", Value::Array(polyc)); + out.put( + "polyc_tracts", + strs(POLYC_TRACTS.iter().map(|(a, b)| format!("m.{a}-{b}"))), + ); + out.put("annotation", ann.val()); + Ok(Some((p, out.val()))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn min_allele_trims_end_then_start() { + assert_eq!(min_allele(302, "AC", "ACC"), (302, "A".into(), "AC".into())); + assert_eq!(min_allele(10, "ACT", "AGT"), (11, "C".into(), "G".into())); + assert_eq!(min_allele(5, "A", "G"), (5, "A".into(), "G".into())); + } + + #[test] + fn polyc_span() { + assert!(in_polyc(302, "A", "AC")); + assert!(!in_polyc(300, "A", "G")); + assert!(in_polyc(16189, "T", "C")); + } + + #[test] + fn python_max_keeps_a_leading_nan() { + assert!(py_max(&[f64::NAN, 0.5]).is_some_and(f64::is_nan)); + assert_eq!(py_max(&[0.5, f64::NAN]), Some(0.5)); + } +} diff --git a/rust/crates/og-summary/src/collect/group_c/cn.rs b/rust/crates/og-summary/src/collect/group_c/cn.rs new file mode 100644 index 000000000..b5ec14d6a --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/cn.rs @@ -0,0 +1,743 @@ +//! Step 35 (Parascopy and the depth counts, short reads) and step 35b +//! (Paraphase, PacBio HiFi): SMN1 and SMN2 at exon 7 with the SMA reading, +//! the HBA, CYP21A2, interval and gene-dosage tables, every other catalog +//! locus, and the Paraphase genes with SMN1 from both tools. + +use super::util::{ + Obj, Table, asset_rows, at, base, columns, count, isdigit, lines, opt_s, or, or_dot, pairs, s, + strs, table, +}; +use super::{Ctx, Found, Out}; +use og_cn::markers::{MARKERS, Status}; +use og_cn::smn::SMN1_EXON7; +use pycompat::csv::DictRow; +use pycompat::json::Value; + +/// SMN2 exon 7, c.840 (the base that tells SMN2 from SMN1), GRCh38 1-based. +pub const SMN2_EXON7: (&str, i64) = ("chr5", 70_076_526); + +/// Tables in paralogs/ that other parts of steps 35 and 35b write, not +/// catalog rows. +const NOT_CATALOG_TABLES: [&str; 2] = ["hba", "paraphase"]; +/// The columns of a Parascopy copy-number table. +const PARASCOPY_COLUMNS: [&str; 9] = [ + "chrom", + "start", + "end", + "agCN", + "agCN_filter", + "agCN_qual", + "psCN", + "psCN_qual", + "psCN_filter", +]; +const NO_SMN_TABLE: &str = "no SMN1 table (step 35 wrote no SMN copy-number table)"; +const MARKER_FIELDS: [&str; 8] = [ + "marker", + "rsid", + "status", + "alt_reads", + "depth", + "fraction", + "copies", + "message", +]; +const DOSAGE_COLUMNS: [&str; 6] = ["locus", "status", "copies", "estimate", "reason", "reading"]; +const INTERVAL_COLUMNS: [&str; 9] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", +]; + +/// A copy number read at one position from a Parascopy table. +#[derive(Debug, Clone, PartialEq)] +pub struct CnAt { + pub cn: Option, + pub text: String, + pub agcn: String, + pub agcn_qual: String, + pub agcn_filter: String, +} + +impl CnAt { + fn none(text: &str) -> CnAt { + CnAt { + cn: None, + text: text.to_owned(), + agcn: ".".into(), + agcn_qual: ".".into(), + agcn_filter: ".".into(), + } + } + + fn json(&self) -> Value { + Obj::new() + .set("cn", cn(self.cn)) + .set("text", s(&self.text)) + .set("agcn", s(&self.agcn)) + .set("agcn_qual", s(&self.agcn_qual)) + .set("agcn_filter", s(&self.agcn_filter)) + .val() + } +} + +fn cn(x: Option) -> Value { + x.map_or(Value::Null, Value::Int) +} + +/// The region of a row and its homologous regions, 1-based closed; `None` +/// for a row without a whole start and end. +fn regions(get: &dyn Fn(&str) -> Option) -> Option> { + let chrom = get("chrom")?; + let start = crate::int(&get("start")?).ok()?; + let end = crate::int(&get("end")?).ok()?; + let mut out = vec![(chrom, start + 1, end)]; + let homologous = get("homologous_regions").filter(|h| !h.is_empty()); + for h in homologous.as_deref().unwrap_or("*").split(',') { + if let Some(r) = region(pycompat::num::py_strip(h)) { + out.push(r); + } + } + Some(out) +} + +/// `^([^:]+):(\d+)-(\d+)` at the start of a text. +fn region(h: &str) -> Option<(String, i128, i128)> { + let (c, rest) = h.split_once(':')?; + if c.is_empty() { + return None; + } + let digits = |x: &str| -> Option<(i128, usize)> { + let n = x.bytes().take_while(u8::is_ascii_digit).count(); + (n > 0).then(|| (x[..n].parse().ok(), n)).and_then(|(v, n)| Some((v?, n))) + }; + let (b, n) = digits(rest)?; + let rest = rest[n..].strip_prefix('-')?; + let (e, _) = digits(rest)?; + Some((c.to_owned(), b, e)) +} + +/// The paralog-specific copy number (psCN) of the row, or of its +/// homologous region, that holds a position, with the row's aggregate +/// value (agCN). `cn` is `None` unless the filter is PASS, the quality 20 +/// or more and the value a whole number (Parascopy's threshold). +fn cn_at_rows<'r>( + rows: impl Iterator Option>, + chrom: &str, + pos: i64, + nothing: &str, +) -> CnAt { + let pos = i128::from(pos); + for get in rows { + let Some(regs) = regions(get) else { continue }; + let Some(idx) = regs + .iter() + .position(|(c, b, e)| c == chrom && *b <= pos && pos <= *e) + else { + continue; + }; + let val = |k: &str| get(k).filter(|v| !v.is_empty()); + let cns = val("psCN").unwrap_or_default(); + let quals = val("psCN_qual").unwrap_or_default(); + let cns: Vec<&str> = cns.split(',').collect(); + let quals: Vec<&str> = quals.split(',').collect(); + let cn_txt = cns.get(idx).copied().unwrap_or("?"); + let q_txt = quals.get(idx).copied().unwrap_or("*"); + let filt = val("psCN_filter").unwrap_or_else(|| ".".into()); + let q = pycompat::num::parse_float(q_txt).ok(); + let ok = filt == "PASS" && q.is_some_and(|q| q >= 20.0) && isdigit(cn_txt); + return CnAt { + cn: if ok { crate::int(cn_txt).ok() } else { None }, + text: format!( + "{cn_txt} (quality {q_txt}, filter {filt}){}", + if ok { "" } else { ", not resolved" } + ), + agcn: val("agCN").unwrap_or_else(|| ".".into()), + agcn_qual: val("agCN_qual").unwrap_or_else(|| ".".into()), + agcn_filter: val("agCN_filter").unwrap_or_else(|| ".".into()), + }; + } + CnAt::none(nothing) +} + +/// `parascopy_cn_at`: over the rows of step 35's table. +/// A lookup of a row's cell by column. +type Getter<'a> = Box Option + 'a>; + +pub fn parascopy_cn_at(rows: &[DictRow], chrom: &str, pos: i64) -> CnAt { + let getters: Vec> = rows + .iter() + .map(|r| { + Box::new(move |k: &str| r.get(k).flatten().map(str::to_owned)) + as Box Option> + }) + .collect(); + cn_at_rows( + getters.iter().map(|g| g.as_ref() as &dyn Fn(&str) -> Option), + chrom, + pos, + "no row covers exon 7", + ) +} + +/// `paraphase_report.parascopy_smn1`: SMN1 at exon 7 from step 35's SMN +/// table read line by line, each line zipped with the header. +pub fn parascopy_smn1(path: &str) -> Result<(Option, String), String> { + let ls = lines(path)?; + let mut it = ls.iter(); + let header: Vec<&str> = it.next().map_or_else(Vec::new, |h| h.split('\t').collect()); + let rows: Vec> = it + .map(|l| header.iter().copied().zip(l.split('\t')).collect()) + .collect(); + let getters: Vec> = rows + .iter() + .map(|r| { + Box::new(move |k: &str| r.get(k).map(|v| (*v).to_owned())) + as Box Option> + }) + .collect(); + let r = cn_at_rows( + getters.iter().map(|g| g.as_ref() as &dyn Fn(&str) -> Option), + SMN1_EXON7.0, + SMN1_EXON7.1, + "no row covers SMN1 exon 7", + ); + Ok((r.cn, r.text)) +} + +/// What an SMN1 copy number means for SMA, in one line. +pub fn smn_reading(smn1: Option) -> String { + match smn1 { + None => { + "SMN1 not resolved (quality under 20 or filter not PASS): no carrier reading".into() + } + Some(0) => "no SMN1 copy: the pattern of spinal muscular atrophy (SMA) itself".into(), + Some(1) => "one SMN1 copy: the pattern of an SMA carrier".into(), + Some(2) => "two SMN1 copies: not a carrier by copy number; a 2+0 carrier (both copies on one chromosome) cannot be ruled out".into(), + Some(n) => format!("{n} SMN1 copies: not a carrier by copy number"), + } +} + +/// A copy-aware marker table (metric, value) beside a count; a table that +/// is not there or has no known status reads as not assessed, with the +/// reason in its message. +pub fn read_marker_table(path: &str, step: &str, key: &str) -> Value { + let m = MARKERS + .iter() + .find(|m| m.key == key) + .expect("a marker of og-cn"); + let statuses = [ + Status::Present, + Status::NotSeen, + Status::Uncertain, + Status::TooLittleDepth, + Status::NotAssessed, + ] + .map(Status::as_str); + let mut out: indexmap::IndexMap<&str, String> = + MARKER_FIELDS.iter().map(|k| (*k, ".".to_owned())).collect(); + out.insert("marker", m.name.to_owned()); + out.insert("rsid", m.dbsnp.0.to_owned()); + out.insert("status", "not assessed".to_owned()); + let name = base(path); + let done = |out: indexmap::IndexMap<&str, String>| { + Value::Object(out.into_iter().map(|(k, v)| (k.to_owned(), s(&v))).collect()) + }; + if !crate::is_file(path) { + out.insert( + "message", + format!( + "{}: not assessed, step {step} wrote no {name} (a run from before this reader, or the check did not run).", + m.name + ), + ); + return done(out); + } + let kv: std::collections::HashMap = + pairs(path).unwrap_or_default().into_iter().collect(); + if !kv.get("status").is_some_and(|v| statuses.contains(&v.as_str())) { + out.insert( + "message", + format!( + "{}: not assessed, {name} has no status row ({}).", + m.name, + statuses.join(", ") + ), + ); + return done(out); + } + for k in MARKER_FIELDS { + if let Some(v) = kv.get(k).filter(|v| !v.is_empty()) { + out.insert(k, v.clone()); + } + } + done(out) +} + +/// `{table: path}` of step 35's catalog tables (`_
_copy_number.tsv`). +fn parascopy_tables(c: &Ctx<'_>) -> Vec<(String, Found)> { + let pat = format!("paralogs/{}_*_copy_number.tsv", c.s()); + let skip = c.sample.chars().count() + 1; + c.index + .all(&pat) + .into_iter() + .filter_map(|rel| { + let b = base(rel); + let n = b.chars().count(); + let name: String = b + .chars() + .skip(skip) + .take(n.saturating_sub(skip + "_copy_number.tsv".len())) + .collect(); + (!NOT_CATALOG_TABLES.contains(&name.as_str())).then(|| { + ( + name, + Found { + path: c.path(rel), + rel: rel.to_owned(), + }, + ) + }) + }) + .collect() +} + +/// A table whose header must hold `need`. +fn has_all(cols: &[String], need: &[&str]) -> bool { + need.iter().all(|k| cols.iter().any(|c| c == k)) +} + +/// The first `agCN_filter` that starts with "not assessed". +fn not_assessed(rows: &[DictRow]) -> Result, String> { + for r in rows { + if r.get("agCN_filter") + .flatten() + .is_some_and(|f| f.starts_with("not assessed")) + { + return Ok(at(r, "agCN_filter")?.map(str::to_owned)); + } + } + Ok(None) +} + +/// Step 35 (Parascopy, short reads): SMN1 and SMN2 at exon 7, their +/// aggregate copy number and the SMA reading; the HBA, interval, CYP21A2 +/// and dosage tables; and every other catalog table, one locus each. +pub fn parascopy(c: &Ctx<'_>) -> Out { + let sm = c.sample; + let tables = parascopy_tables(c); + let direct = |name: &str| { + let rel = format!("paralogs/{sm}_{name}"); + Found { + path: c.path(&rel), + rel, + } + }; + let hp = direct("hba_copy_number.tsv"); + let ip = direct("cn_intervals.tsv"); + let cp = direct("cyp21a2_psv.tsv"); + let dp = direct("cn_dosage.tsv"); + let smn = tables.iter().find(|(n, _)| n == "smn").map(|x| x.1.clone()); + let Some(p) = smn + .clone() + .or_else(|| tables.first().map(|x| x.1.clone())) + .or_else(|| { + [&hp, &ip, &cp, &dp] + .into_iter() + .find(|f| crate::is_file(&f.path)) + .cloned() + }) + else { + return Ok(None); + }; + let (smn1, smn2, reading, nrows) = match &smn { + Some(f) => { + let t = table(&f.path, false)?; + if !has_all(&columns(&f.path)?, &PARASCOPY_COLUMNS) { + return Err(format!( + "{} lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or psCN_filter column", + base(&f.path) + )); + } + let smn1 = parascopy_cn_at(&t.rows, SMN1_EXON7.0, SMN1_EXON7.1); + let smn2 = parascopy_cn_at(&t.rows, SMN2_EXON7.0, SMN2_EXON7.1); + let reading = match not_assessed(&t.rows)? { + Some(na) => format!("SMN1 and SMN2 {na}: no carrier reading"), + None if smn1.text == "no row covers exon 7" => { + "no row of step 35's table covers SMN1 exon 7: no carrier reading".into() + } + None => smn_reading(smn1.cn), + }; + (smn1, smn2, reading, t.rows.len()) + } + None => ( + CnAt::none(NO_SMN_TABLE), + CnAt::none(NO_SMN_TABLE), + format!("{NO_SMN_TABLE}: no carrier reading"), + 0, + ), + }; + let others: Vec<&(String, Found)> = tables.iter().filter(|(n, _)| n != "smn").collect(); + let mut catalog: indexmap::IndexMap = indexmap::IndexMap::new(); + if !others.is_empty() { + for r in asset_rows(c.assets, "paralog_loci.tsv")? { + let k = at(&r, "table")?.unwrap_or("None").to_owned(); + catalog.insert(k, r); + } + } + let mut order: Vec<&(String, Found)> = others; + order.sort_by(|a, b| { + let key = |n: &str| catalog.get_index_of(n).unwrap_or(catalog.len()); + key(&a.0).cmp(&key(&b.0)).then_with(|| a.0.cmp(&b.0)) + }); + let mut loci = Vec::new(); + for (n, f) in order { + loci.push(parascopy_locus(n, &f.path, catalog.get(n))?); + } + let opt = |f: &Found, read: fn(&str) -> Result| -> Result { + if crate::is_file(&f.path) { + read(&f.path) + } else { + Ok(Value::Null) + } + }; + let data = Obj::new() + .set("smn1", smn1.json()) + .set( + "smn2", + Obj::new() + .set("cn", cn(smn2.cn)) + .set("text", s(&smn2.text)) + .val(), + ) + .set("agcn", s(&smn1.agcn)) + .set("agcn_qual", s(&smn1.agcn_qual)) + .set("agcn_filter", s(&smn1.agcn_filter)) + .set( + "smn1_marker", + read_marker_table( + &c.path(&format!("paralogs/{sm}_smn1_marker.tsv")), + "35", + "SMN1_27134", + ), + ) + .set("rows", count(nrows)) + .set("hba", opt(&hp, read_hba)?) + .set("cn_intervals", opt(&ip, read_cn_intervals)?) + .set("cyp21a2", opt(&cp, read_hba)?) + .set("dosage", opt(&dp, read_cn_dosage)?) + .set("reading", s(&reading)) + .set("loci", Value::Array(loci)); + Ok(Some((p, data.val()))) +} + +/// One catalog table that is not SMN1's: the locus's aggregate copy number +/// per region of its profile, the catalog's genes, tag and reading. +fn parascopy_locus(name: &str, path: &str, row: Option<&DictRow>) -> Result { + let cat = |k: &str, default: &str| row.map_or_else(|| default.to_owned(), |r| or(r, k, default)); + let out = Obj::new() + .set("table", s(name)) + .set("genes", s(&cat("genes", name))) + .set("tag", s(&cat("tag", "."))) + .set("reading", s(&cat("reading", "."))); + let t = table(path, false)?; + let with = |o: Obj, status: &str, cnv: Value, text: &str| { + o.set("status", s(status)).set("cn", cnv).set("text", s(text)).val() + }; + if !has_all(&t.head, &PARASCOPY_COLUMNS) { + let text = format!("unreadable: {} lacks the copy-number columns", base(path)); + return Ok(with(out, "unreadable", Value::Null, &text)); + } + if let Some(na) = not_assessed(&t.rows)? { + return Ok(with(out, "not assessed", Value::Null, &na)); + } + struct Reg { + chrom: Option, + start: Option, + end: Option, + cn: String, + qual: String, + filter: String, + ok: bool, + } + let regs: Vec = t + .rows + .iter() + .map(|r| { + let get = |k: &str| r.get(k).flatten().map(str::to_owned); + let q = pycompat::num::parse_float(&or(r, "agCN_qual", "")).ok(); + let cnv = or_dot(r, "agCN"); + let ok = isdigit(&cnv) + && r.get("agCN_filter").flatten() == Some("PASS") + && q.is_some_and(|q| q >= 20.0); + Reg { + chrom: get("chrom"), + start: get("start"), + end: get("end"), + cn: cnv, + qual: or_dot(r, "agCN_qual"), + filter: or_dot(r, "agCN_filter"), + ok, + } + }) + .collect(); + if regs.is_empty() { + return Ok(with( + out, + "not assessed", + Value::Null, + "not assessed: the table has no region row", + )); + } + let values: std::collections::HashSet<&str> = + regs.iter().filter(|r| r.ok).map(|r| r.cn.as_str()).collect(); + if values.len() == 1 && regs.iter().all(|r| r.ok) { + let n = crate::int(values.into_iter().next().unwrap_or_default()) + .map_err(|e| super::super::sections::py_error("ValueError", &e))?; + let mut min = f64::INFINITY; + for r in ®s { + min = min.min(crate::float(&r.qual).map_err(|e| { + super::super::sections::py_error("ValueError", &e) + })?); + } + let text = format!( + "{n} ({} region(s), quality {} or more, filter PASS)", + regs.len(), + pycompat::fmt::general(min, 6) + ); + return Ok(with(out, "called", Value::Int(n), &text)); + } + let none = |x: &Option| x.clone().unwrap_or_else(|| "None".into()); + let text = regs + .iter() + .map(|r| { + let b = match &r.start { + Some(b) if isdigit(b) => crate::int(b).map_or_else(|_| b.clone(), |v| (v + 1).to_string()), + b => none(b), + }; + format!( + "{}:{b}-{} {} (quality {}, filter {})", + none(&r.chrom), + none(&r.end), + r.cn, + r.qual, + r.filter + ) + }) + .collect::>() + .join("; "); + let called = regs.iter().any(|r| r.ok); + let text = format!( + "{text}{}", + if called { + "" + } else { + ": no region with a whole copy number, filter PASS and quality 20 or more" + } + ); + let out = out.set("reading", s(".")); + Ok(with( + out, + if called { "called" } else { "not resolved" }, + Value::Null, + &text, + )) +} + +/// Step 35's interval table: the rows with a deletion or a duplication, and +/// a count of the rows by status. +fn read_cn_intervals(path: &str) -> Result { + let t = table(path, false)?; + if !has_all(&t.head, &INTERVAL_COLUMNS) { + return Err(format!( + "{} lacks the locus, name, status, copies, expected or change column", + base(path) + )); + } + short_row(path, &t, &INTERVAL_COLUMNS)?; + let mut by: indexmap::IndexMap = indexmap::IndexMap::new(); + let mut changes = Vec::new(); + for r in &t.rows { + let cell = |k: &str| r.get(k).flatten().unwrap_or_default().to_owned(); + *by.entry(cell("status")).or_default() += 1; + if matches!(cell("change").as_str(), "deletion" | "duplication") { + let keep = [ + "locus", "name", "chrom", "start", "end", "copies", "expected", "change", + ]; + changes.push( + keep.iter() + .fold(Obj::new(), |o, k| o.set(k, s(&cell(k)))) + .val(), + ); + } + } + Ok(Obj::new() + .set("rows", count(t.rows.len())) + .set( + "by_status", + Value::Object(by.into_iter().map(|(k, v)| (k, Value::Int(v))).collect()), + ) + .set("changes", Value::Array(changes)) + .val()) +} + +/// Step 35's gene-dosage table: one entry per locus, in the table's order. +fn read_cn_dosage(path: &str) -> Result { + if !has_all(&columns(path)?, &DOSAGE_COLUMNS) { + return Err(format!( + "{} lacks the {} or reading column", + base(path), + DOSAGE_COLUMNS[..5].join(", ") + )); + } + let t = table(path, false)?; + short_row(path, &t, &DOSAGE_COLUMNS)?; + Ok(Value::Array( + t.rows + .iter() + .map(|r| super::util::dots(r, &DOSAGE_COLUMNS)) + .collect(), + )) +} + +/// A row cut short of a column makes the table unreadable. +fn short_row(path: &str, t: &Table, cols: &[&str]) -> Result<(), String> { + match t + .rows + .iter() + .position(|r| cols.iter().any(|k| !matches!(r.get(k), Some(Some(_))))) + { + Some(i) => Err(format!("{} line {} lacks a field", base(path), i + 2)), + None => Ok(()), + } +} + +/// Step 35's HBA table (metric, value), and its CYP21A2 table the same way; +/// a table without a status row reads as not assessed. +fn read_hba(path: &str) -> Result { + let mut out: indexmap::IndexMap = indexmap::IndexMap::new(); + for l in lines(path)? { + if let Some((k, v)) = l.split_once('\t') + && k != "metric" + { + out.insert(k.to_owned(), s(v)); + } + } + let ok = matches!(out.get("status"), Some(Value::Str(v)) if v == "called" || v == "not assessed"); + if !ok { + return Ok(Obj::new() + .set("status", s("not assessed")) + .set( + "message", + s(&format!("not assessed: {} has no status row", base(path))), + ) + .val()); + } + Ok(Value::Object(out)) +} + +/// The Paraphase regions shown gene by gene, in the order of +/// assets/paraphase_genes.tsv. +fn paralog_regions(assets: &str) -> Result, String> { + let mut out: Vec = Vec::new(); + for r in asset_rows(assets, "paraphase_genes.tsv")? { + let g = at(&r, "region")?.unwrap_or("None").to_owned(); + if !out.contains(&g) { + out.push(g); + } + } + Ok(out) +} + +/// Step 35b (Paraphase, PacBio HiFi): the genes of the catalog regions, a +/// count of the other regions, the variant rows per region, and SMN1 +/// beside step 35's Parascopy call. +pub fn paralogs(c: &Ctx<'_>) -> Out { + let s_ = c.sample; + let Some(p) = c.find1(format!("paralogs/{}_paraphase_copy_number.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + if !t.rows.is_empty() && !has_all(&t.head, &["region", "gene", "copy_number", "status"]) { + return Err(format!( + "{} lacks the region, gene, copy_number or status column", + base(&p.path) + )); + } + let regions = paralog_regions(c.assets)?; + let region_of = |r: &DictRow| r.get("region").flatten().map(str::to_owned); + let mut genes: Vec<(usize, Value, Option, Option)> = Vec::new(); + let mut others = 0; + let mut others_called = 0; + for r in &t.rows { + let reg = region_of(r); + match reg.as_ref().and_then(|g| regions.iter().position(|x| x == g)) { + Some(i) => { + let cell = |k: &str| opt_s(r.get(k).flatten()); + genes.push(( + i, + Obj::new() + .set("region", cell("region")) + .set("gene", cell("gene")) + .set("copy_number", cell("copy_number")) + .set("status", cell("status")) + .set("detail", s(&or_dot(r, "detail"))) + .val(), + r.get("gene").flatten().map(str::to_owned), + r.get("copy_number").flatten().map(str::to_owned), + )); + } + None => { + others += 1; + if r.get("status").flatten() == Some("called") { + others_called += 1; + } + } + } + } + genes.sort_by_key(|g| g.0); + let mut variants: indexmap::IndexMap = indexmap::IndexMap::new(); + if let Some(vp) = c.beside(&p, &format!("{s_}_paraphase_variants.tsv")) { + for r in table(&vp.path, false)?.rows { + let k = match r.get("region") { + None => String::new(), + Some(None) => "null".into(), + Some(Some(v)) => v.to_owned(), + }; + *variants.entry(k).or_default() += 1; + } + } + let smn1 = genes + .iter() + .find(|g| g.2.as_deref() == Some("SMN1")) + .and_then(|g| g.3.clone()) + .filter(|x| isdigit(x)) + .and_then(|x| crate::int(&x).ok()); + let parascopy = match c.beside(&p, &format!("{s_}_smn_copy_number.tsv")) { + Some(pp) => Some(parascopy_smn1(&pp.path)?), + None => None, + }; + let agree = match (smn1, ¶scopy) { + (Some(a), Some((Some(b), _))) => s(if a == *b { "agree" } else { "disagree" }), + _ => Value::Null, + }; + let parascopy_json = parascopy.map_or(Value::Null, |(n, text)| { + Obj::new().set("cn", cn(n)).set("text", s(&text)).val() + }); + let data = Obj::new() + .set("genes", Value::Array(genes.into_iter().map(|g| g.1).collect())) + .set("other_regions", count(others)) + .set("other_regions_called", count(others_called)) + .set( + "variants", + Value::Object(variants.into_iter().map(|(k, v)| (k, Value::Int(v))).collect()), + ) + .set( + "smn1", + Obj::new() + .set("paraphase", cn(smn1)) + .set("parascopy", parascopy_json) + .set("agree", agree) + .val(), + ); + let _ = strs::<[&str; 0], &str>; + Ok(Some((p, data.val()))) +} diff --git a/rust/crates/og-summary/src/collect/group_c/hla.rs b/rust/crates/og-summary/src/collect/group_c/hla.rs new file mode 100644 index 000000000..3d976a9fc --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/hla.rs @@ -0,0 +1,396 @@ +//! Step 08 (T1K): the HLA alleles per gene with their qualities, the +//! candidate cap, the HFE label, the coeliac DQ haplotypes and the table +//! alleles marked present or absent; OptiType's concordance and the KIR +//! pass when they ran. The rules are docs/08-hla-typing.md's "What the +//! report derives". + +use super::util::{Obj, at, bin_rows, bin_table, count, key_error, lines, s, strs, table}; +use super::{Ctx, Out}; +use og_steps::pgx_rules::HLA_GENES; +use pycompat::json::Value; + +/// A quality-0 allele with more candidates than this is "no confident call". +pub const MAX_CANDIDATES: usize = 3; +const HFE_LABEL: &str = "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)"; + +/// One gene's row of T1K's genotype table. +#[derive(Debug, Clone)] +pub struct Locus { + pub gene: String, + pub alleles: Vec, + pub quality: Vec, + pub low_confidence: bool, + pub candidates: Vec, + pub one_allele_type: bool, +} + +impl Locus { + fn json(&self) -> Value { + let mut o = Obj::new() + .set("gene", s(&self.gene)) + .set("alleles", strs(&self.alleles)) + .set("quality", strs(&self.quality)) + .set("low_confidence", Value::Bool(self.low_confidence)) + .set( + "withheld_from_pharmcat", + Value::Bool(self.low_confidence && HLA_GENES.contains(&self.gene.as_str())), + ) + .set( + "candidates", + Value::Array(self.candidates.iter().map(|n| count(*n)).collect()), + ) + .set("one_allele_type", Value::Bool(self.one_allele_type)); + if name(&self.gene) == "HFE" { + o.put("label", s(HFE_LABEL)); + } + o.val() + } +} + +/// T1K's quality as a number; one that is not a number counts as -1, as in +/// step 36. +pub fn quality(q: &str) -> f64 { + pycompat::num::parse_float(q).unwrap_or(-1.0) +} + +/// `HLA-DQB1*02:01:01` or `DQB1*02:01:01` is `DQB1*02:01:01`. +fn name(x: &str) -> &str { + let x = pycompat::num::py_strip(x); + x.strip_prefix("HLA-").unwrap_or(x) +} + +/// T1K's allele field: one allele, or equally likely candidates separated +/// by commas. +fn candidates(allele: &str) -> Vec<&str> { + allele.split(',').map(name).filter(|a| !a.is_empty()).collect() +} + +/// The alleles and their qualities of columns 3 and 6 (`.` is no allele). +fn alleles(c: &[&str], skip_empty: bool) -> (Vec, Vec) { + let (mut a, mut q) = (Vec::new(), Vec::new()); + for i in [2, 5] { + let x = c.get(i).copied().unwrap_or(""); + if (skip_empty && x.is_empty()) || x == "." || x.is_empty() { + continue; + } + a.push(x.to_owned()); + q.push(c.get(i + 2).copied().unwrap_or("").to_owned()); + } + (a, q) +} + +/// One locus with the candidate cap, the candidate counts and whether T1K +/// typed one allele. +fn locus(gene: &str, alleles: Vec, quals: Vec, n_diff: &str) -> Locus { + // Step 36's rule: an allele quality of 0 or below withholds the gene. + let low = quals.iter().any(|q| quality(q) <= 0.0); + let mut counts = Vec::new(); + let mut kept = Vec::new(); + for (i, a) in alleles.iter().enumerate() { + let q = quals.get(i).map_or("", String::as_str); + let cands = candidates(a); + counts.push(cands.len()); + kept.push(if quality(q) <= 0.0 && cands.len() > MAX_CANDIDATES { + cands[..MAX_CANDIDATES].join(",") + } else { + a.clone() + }); + } + let one = pycompat::num::py_strip(n_diff) == "1" && kept.len() == 1; + Locus { + gene: gene.to_owned(), + alleles: kept, + quality: quals, + low_confidence: low, + candidates: counts, + one_allele_type: one, + } +} + +/// `yes` when every candidate of the allele is in the group, `no` when +/// none is, `mixed` when only some are. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Match { + Yes, + No, + Mixed, +} + +fn matches(allele: &str, group: &str) -> Match { + let prefix = format!("{group}:"); + let hits: Vec = candidates(allele) + .iter() + .map(|c| *c == group || c.starts_with(&prefix)) + .collect(); + if !hits.is_empty() && hits.iter().all(|h| *h) { + Match::Yes + } else if hits.iter().any(|h| *h) { + Match::Mixed + } else { + Match::No + } +} + +/// Over both alleles of a locus. +fn carries(l: &Locus, group: &str) -> Match { + let m: Vec = l.alleles.iter().map(|a| matches(a, group)).collect(); + if m.contains(&Match::Yes) { + Match::Yes + } else if m.contains(&Match::Mixed) { + Match::Mixed + } else { + Match::No + } +} + +fn both(a: Match, b: Match) -> &'static str { + if a == Match::Yes && b == Match::Yes { + "present" + } else if a == Match::No || b == Match::No { + "absent" + } else { + "uncertain" + } +} + +/// The DQ haplotypes, the DQB1*02 dose and each table allele present or +/// absent. +fn derived(loci: &[Locus], table: &[indexmap::IndexMap]) -> Result { + let mut by: indexmap::IndexMap<&str, &Locus> = indexmap::IndexMap::new(); + for l in loci { + by.entry(name(&l.gene)).or_insert(l); + } + let usable = |gene: &str| -> (Option<&Locus>, String) { + match by.get(gene) { + Some(l) if !l.alleles.is_empty() => { + if l.low_confidence { + (None, format!("HLA-{gene} low confidence")) + } else { + (Some(*l), String::new()) + } + } + _ => (None, format!("HLA-{gene} not typed")), + } + }; + let (dqa, ra) = usable("DQA1"); + let (dqb, rb) = usable("DQB1"); + let reason = [ra, rb] + .into_iter() + .filter(|r| !r.is_empty()) + .collect::>() + .join("; "); + let mut co = Obj::new() + .set("status", s("not_determined")) + .set("reason", s(&reason)) + .set("dq2_5", Value::Null) + .set("dq2_5_phase", Value::Null) + .set("dq2_2", Value::Null) + .set("dq8", Value::Null) + .set("dqb1_02_dose", Value::Null); + if let Some(dqb) = dqb { + let m: Vec = dqb.alleles.iter().map(|a| matches(a, "DQB1*02")).collect(); + if !m.contains(&Match::Mixed) { + let n = m.iter().filter(|x| **x == Match::Yes).count(); + co.put( + "dqb1_02_dose", + count(if dqb.one_allele_type { 2 * n } else { n }), + ); + } + } + if let (Some(dqa), Some(dqb)) = (dqa, dqb) { + co.put("status", s("derived")); + co.put("reason", s("")); + let dq25 = both(carries(dqa, "DQA1*05"), carries(dqb, "DQB1*02")); + co.put("dq2_5", s(dq25)); + co.put( + "dq2_2", + s(both(carries(dqa, "DQA1*02:01"), carries(dqb, "DQB1*02:02"))), + ); + co.put( + "dq8", + s(both(carries(dqa, "DQA1*03"), carries(dqb, "DQB1*03:02"))), + ); + if dq25 == "present" { + let (drb1, why) = usable("DRB1"); + let phase = match drb1.map(|l| carries(l, "DRB1*03")) { + Some(Match::Yes) => "likely cis (DRB1*03 present)".to_owned(), + Some(Match::No) => "possible trans (no DRB1*03)".to_owned(), + _ => format!( + "phase unknown ({})", + if why.is_empty() { + "DRB1*03 uncertain" + } else { + &why + } + ), + }; + co.put("dq2_5_phase", s(&phase)); + } + } + let mut alleles = Vec::new(); + for r in table { + let cell = |k: &str| r.get(k).cloned().ok_or_else(|| key_error(k)); + let allele = cell("allele")?; + let gene = allele.split('*').next().unwrap_or_default(); + let (l, why) = usable(gene); + let state = match l.map(|l| carries(l, &allele)) { + Some(Match::Yes) => "present", + Some(Match::No) => "absent", + Some(Match::Mixed) => "uncertain", + None => "not_determined", + }; + alleles.push( + Obj::new() + .set("allele", s(&allele)) + .set("kind", s(&cell("kind")?)) + .set("association", s(&cell("association")?)) + .set("pmids", s(&cell("pmids")?)) + .set("citation", s(&cell("citation")?)) + .set("state", s(state)) + .set("reason", s(&why)) + .val(), + ); + } + Ok(Obj::new() + .set("coeliac", co.val()) + .set("alleles", Value::Array(alleles)) + .val()) +} + +/// The first `#` line of a file step 08 wrote instead of a result. +fn first_comment(path: &str) -> Result { + Ok(lines(path)? + .iter() + .find(|l| l.starts_with('#')) + .map(|l| pycompat::num::py_strip(l.trim_start_matches('#')).to_owned()) + .unwrap_or_default()) +} + +/// OptiType's concordance with T1K per class I gene, or why it did not run. +fn optitype(c: &Ctx<'_>) -> Result, String> { + let conc = c.find1(format!("hla_optitype/{}_hla_concordance.tsv", c.s())); + let res = c.find1(format!("hla_optitype/{}_optitype_result.tsv", c.s())); + let Some(conc) = conc else { + let Some(res) = res else { return Ok(None) }; + let why = first_comment(&res.path)?; + let why = if why.is_empty() { + "no concordance table: step 08 did not compare the callers".to_owned() + } else { + why + }; + return Ok(Some( + Obj::new() + .set("genes", Value::Array(Vec::new())) + .set("reason", s(&why)) + .val(), + )); + }; + let mut genes = Vec::new(); + for r in table(&conc.path, false)?.rows { + let cell = |k: &str| pycompat::num::py_strip(r.get(k).flatten().unwrap_or("")).to_owned(); + if !cell("Gene").is_empty() { + genes.push( + Obj::new() + .set("gene", s(&cell("Gene"))) + .set("t1k", s(&cell("T1K"))) + .set("optitype", s(&cell("OptiType"))) + .set("agree", s(&cell("Agree"))) + .set("note", s(&cell("Note"))) + .val(), + ); + } + } + let reason = if genes.is_empty() { + "the concordance table has no rows" + } else { + "" + }; + Ok(Some( + Obj::new() + .set("genes", Value::Array(genes)) + .set("reason", s(reason)) + .val(), + )) +} + +/// The KIR pass: the genes T1K gave an allele, with each quality, and the +/// genes it gave none. +fn kir(c: &Ctx<'_>) -> Result, String> { + let s_ = c.s(); + let Some(p) = c.find(&[ + format!("kir_t1k/{s_}_kir_genotype.tsv"), + format!("kir/{s_}_kir_genotype.tsv"), + ]) else { + return Ok(None); + }; + let (mut typed, mut not_called) = (Vec::new(), Vec::new()); + for l in lines(&p.path)? { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 3 || c[0].starts_with('#') { + continue; + } + let (a, q) = alleles(&c, true); + if a.is_empty() { + not_called.push(c[0].to_owned()); + } else { + typed.push( + Obj::new() + .set("gene", s(c[0])) + .set("alleles", strs(&a)) + .set("quality", strs(&q)) + .val(), + ); + } + } + let reason = if !typed.is_empty() || !not_called.is_empty() { + String::new() + } else { + let why = first_comment(&p.path)?; + if why.is_empty() { + "the KIR genotype file has no gene rows".to_owned() + } else { + why + } + }; + Ok(Some( + Obj::new() + .set("typed", Value::Array(typed)) + .set("not_called", strs(¬_called)) + .set("reason", s(&reason)) + .val(), + )) +} + +/// Step 08's T1K genotype table, the derived calls, and OptiType and KIR +/// when they ran. +pub fn hla(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find(&[ + format!("hla_t1k/{s_}_hla_genotype.tsv"), + format!("hla/{s_}_hla_genotype.tsv"), + "hla/*_hla_genotype.tsv".to_owned(), + ]) else { + return Ok(None); + }; + let mut loci = Vec::new(); + for l in lines(&p.path)? { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 3 || c[0].starts_with('#') { + continue; + } + let (a, q) = alleles(&c, false); + loci.push(locus(c[0], a, q, c.get(1).copied().unwrap_or(""))); + } + let table = bin_rows(&bin_table("hla_alleles.tsv"))?; + let mut out = Obj::new() + .set("loci", Value::Array(loci.iter().map(Locus::json).collect())) + .set("derived", derived(&loci, &table)?); + if let Some(o) = optitype(c)? { + out.put("optitype", o); + } + if let Some(k) = kir(c)? { + out.put("kir", k); + } + let _ = at; + Ok(Some((p, out.val()))) +} diff --git a/rust/crates/og-summary/src/collect/group_c/repeats.rs b/rust/crates/og-summary/src/collect/group_c/repeats.rs new file mode 100644 index 000000000..a0e75c123 --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/repeats.rs @@ -0,0 +1,902 @@ +//! Step 09 (ExpansionHunter) and 9b (Stranger, the RFC1 and other motif +//! counts): every locus with its counts, how well each allele was sized, +//! Stranger's grade, the intermediate zone, and what the counts of CFTR's +//! TG and poly-T tracts and UGT1A1's TA repeat mean. + +use super::super::model::State; +use super::util::{ + Obj, bin_rows, bin_table, cells, count, get_default, s, strs, table, vcf_lines, +}; +use super::{Ctx, Out}; +use indexmap::IndexMap; +use pycompat::json::Value; + +/// The five loci the report names first. +pub const EH_LOCI: [&str; 5] = ["HTT", "FMR1", "C9ORF72", "ATXN1", "DMPK"]; +/// Fewer spanning reads than this: a low-confidence size. +const MIN_SPANNING: i128 = 2; +const MOTIF_KEYS: [&str; 7] = [ + "verdict", + "note", + "dominant_motif", + "dominant_class", + "dominant_share", + "motif_shares", + "repeat_reads", +]; +/// The repeats STRchive leaves out, read when step 09 sized them. +const EXTRA_SETUP: &str = "STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data "; +const CFTR_R117H: &str = "rs78655421"; +const CFTR_SOURCE: &str = "Deignan et al. 2020, PMID 32404922"; +const UGT1A1_SOURCE: &str = "Gammal et al. 2016, PMID 26417955"; +const HTTLPR_READING: &str = "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation."; + +/// How well ExpansionHunter sized an allele, worst first; a locus takes +/// the worst of its alleles. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Sizing { + NoCall, + FlankingOnly, + InRepeat, + LowConfidence, + Sized, + Unknown, +} + +impl Sizing { + pub fn word(self) -> &'static str { + match self { + Sizing::NoCall => "no_call", + Sizing::FlankingOnly => "flanking_only", + Sizing::InRepeat => "in_repeat", + Sizing::LowConfidence => "low_confidence", + Sizing::Sized => "sized", + Sizing::Unknown => "unknown", + } + } + + fn of(w: &str) -> Option { + [ + Sizing::NoCall, + Sizing::FlankingOnly, + Sizing::InRepeat, + Sizing::LowConfidence, + Sizing::Sized, + Sizing::Unknown, + ] + .into_iter() + .find(|z| z.word() == w) + } + + /// Sized well enough to read a count. + fn read(self) -> bool { + matches!(self, Sizing::Sized | Sizing::Unknown) + } +} + +fn str_int(x: &str) -> Option { + crate::int(x).ok() +} + +/// One allele's sizing from ExpansionHunter's SO, ADSP and REPCI fields. +fn allele_sizing(so: &str, adsp: &str, ci: &str, cn: &str, path_min: Option) -> Sizing { + let Some(n) = str_int(cn) else { + return Sizing::NoCall; + }; + match so { + "" | "." => return Sizing::Unknown, + "FLANKING" => return Sizing::FlankingOnly, + "INREPEAT" => return Sizing::InRepeat, + _ => {} + } + let hi = if ci.is_empty() { + None + } else { + str_int(ci.rsplit('-').next().unwrap_or(ci)) + }; + let few = str_int(adsp).is_some_and(|a| a < MIN_SPANNING); + let reaches = matches!((path_min, hi), (Some(m), Some(h)) if n < m && m <= h); + if few || reaches { + Sizing::LowConfidence + } else { + Sizing::Sized + } +} + +/// A per-allele FORMAT value as `n` parts; one value serves every allele. +fn split_n(x: &str, n: usize) -> Vec { + let parts: Vec<&str> = if x.is_empty() { + Vec::new() + } else { + x.split('/').collect() + }; + if parts.len() == n { + parts.into_iter().map(str::to_owned).collect() + } else if parts.len() == 1 { + vec![parts[0].to_owned(); n] + } else { + vec![String::new(); n] + } +} + +/// `bin/str_intermediate_ranges.tsv` by locus. +fn ranges() -> Result>, String> { + let mut out = IndexMap::new(); + for r in bin_rows(&bin_table("str_intermediate_ranges.tsv"))? { + let k = r + .get("locus") + .cloned() + .ok_or_else(|| super::util::key_error("locus"))?; + out.insert(k, r); + } + Ok(out) +} + +fn mtime(p: &str) -> Result { + pycompat::mtime::mtime(p).map_err(|e| super::util::os_note(&crate::os_error(&e, p))) +} + +/// The motif table `name` unless it is older than Stranger's file. +fn motif_table(c: &Ctx<'_>, name: &str, after: &str) -> Result, String> { + let Some(p) = c.find1(format!("expansion_hunter/{}_{name}", c.s())) else { + return Ok(None); + }; + if mtime(&p.path)? < mtime(after)? { + return Ok(None); + } + table(&p.path, false).map(Some) +} + +fn motif_row(r: &pycompat::csv::DictRow) -> Value { + MOTIF_KEYS + .iter() + .fold(Obj::new(), |o, k| o.set(k, get_default(r, k, "."))) + .val() +} + +/// `{locus: reason}` of the loci the catalog step 09 ran on gives no +/// thresholds (its NoThresholds field), from the copy beside its VCF. +fn not_graded(c: &Ctx<'_>) -> IndexMap { + let mut out = IndexMap::new(); + let Some(p) = c.find1(format!("expansion_hunter/{}_eh_catalog.json", c.s())) else { + return out; + }; + let Ok(text) = std::fs::read(&p.path).map(String::from_utf8) else { + return out; + }; + let Ok(Ok(Value::Array(cat))) = text.map(|t| pycompat::json::loads(&t)) else { + return out; + }; + for e in cat { + let Value::Object(m) = e else { continue }; + let (Some(id), Some(why)) = (m.get("LocusId"), m.get("NoThresholds")) else { + continue; + }; + if pycompat::value::truthy(id) + && pycompat::value::truthy(why) + && let (Value::Str(id), Ok(why)) = (id, pycompat::value::py_str(why)) + { + out.insert(id.clone(), why); + } + } + out +} + +/// One locus as the reports read it. +struct Locus { + json: Obj, + locus: String, +} + +/// Step 09's ExpansionHunter VCF, or Stranger's copy of it with a grade per +/// locus when it is not older than the VCF. +pub fn expansions(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let eh = c.find(&[ + format!("expansion_hunter/{s_}_eh.vcf"), + format!("expansion_hunter/{s_}.vcf"), + "expansion_hunter/*_eh.vcf".to_owned(), + ]); + let mut st = c.find(&[ + format!("expansion_hunter/{s_}_eh_stranger.vcf"), + "expansion_hunter/*_eh_stranger.vcf".to_owned(), + ]); + let outdated = match (&st, &eh) { + (Some(a), Some(b)) => mtime(&a.path)? < mtime(&b.path)?, + _ => false, + }; + if outdated { + st = None; + } + let Some(p) = st.clone().or(eh) else { + return Ok(None); + }; + let ranges = ranges()?; + let stp = st.as_ref().map(|f| f.path.as_str()); + let rfc1 = match stp { + Some(after) => motif_table(c, "rfc1_motif.tsv", after)? + .and_then(|t| t.rows.first().map(motif_row)), + None => None, + }; + let mut motifs: IndexMap = IndexMap::new(); + if let Some(after) = stp + && let Some(t) = motif_table(c, "str_motifs.tsv", after)? + { + for r in &t.rows { + if let Some(l) = r.get("locus").flatten().filter(|l| !l.is_empty()) { + motifs.insert(l.to_owned(), motif_row(r)); + } + } + } + let ungraded = if st.is_some() { + not_graded(c) + } else { + IndexMap::new() + }; + let mut records = Vec::new(); + vcf_lines(&p.path, |l| records.push(cells(l)))?; + let (mut loci, mut flagged, mut no_status, mut graded_none) = + (Vec::::new(), Vec::new(), 0, Vec::::new()); + let mut tested = 0; + for r in &records { + if r.len() < 10 { + continue; + } + tested += 1; + let info: IndexMap<&str, &str> = r[7] + .split(';') + .map(|kv| kv.split_once('=').unwrap_or((kv, ""))) + .collect(); + let get = |k: &str| info.get(k).copied().filter(|v| !v.is_empty()); + let lid = get("REPID") + .or(get("VARID")) + .map_or_else(|| format!("{}:{}", r[0], r[1]), str::to_owned); + let rep = get("VARID").map_or_else(|| lid.clone(), str::to_owned); + let fmt: Vec<&str> = r[8].split(':').collect(); + let val: Vec<&str> = r[9].split(':').collect(); + let field = |k: &str| { + fmt.iter() + .position(|f| *f == k) + .and_then(|i| val.get(i)) + .copied() + .unwrap_or("") + }; + let repcn = field("REPCN"); + let counts: Vec<&str> = if repcn.is_empty() || repcn == "." { + Vec::new() + } else { + repcn.split('/').collect() + }; + let n = counts.len().max(1); + let path_min = info.get("STR_PATHOLOGIC_MIN").and_then(|v| str_int(v)); + let cn_list: Vec<&str> = if counts.is_empty() { vec!["."] } else { counts.clone() }; + let sizing: Vec = split_n(field("SO"), n) + .iter() + .zip(split_n(field("ADSP"), n)) + .zip(split_n(field("REPCI"), n)) + .zip(cn_list) + .map(|(((so, ad), ci), cn)| allele_sizing(so, &ad, &ci, cn, path_min)) + .collect(); + let status: Vec<&str> = if st.is_some() { + info.get("STR_STATUS") + .copied() + .unwrap_or("") + .split(',') + .filter(|x| !x.is_empty() && *x != ".") + .collect() + } else { + Vec::new() + }; + let worst = sizing.iter().copied().min().unwrap_or(Sizing::NoCall); + let dots = |k: &str| info.get(k).copied().unwrap_or("").trim_matches('.').to_owned(); + let mut item = Obj::new() + .set("locus", s(&rep)) + .set("repeat_count", s(if repcn.is_empty() { "." } else { repcn })) + .set("status", s(&status.join(","))) + .set("normal_max", s(&dots("STR_NORMAL_MAX"))) + .set("pathologic_min", s(&dots("STR_PATHOLOGIC_MIN"))) + .set("support", s(field("SO"))) + .set("adsp", s(field("ADSP"))) + .set("ci", s(field("REPCI"))) + .set("sizing", s(worst.word())) + .set( + "allele_sizing", + s(&sizing.iter().map(|z| z.word()).collect::>().join("/")), + ) + .set("zone", s("")) + .set("zone_label", s("")) + .set("zone_citation", s("")); + let sized: Vec = counts + .iter() + .zip(&sizing) + .filter(|(_, z)| z.read()) + .filter_map(|(c, _)| str_int(c)) + .collect(); + if let Some(rng) = ranges.get(&rep) { + let bound = |k: &str| -> Result { + let v = rng.get(k).ok_or_else(|| super::util::key_error(k))?; + crate::int(v).map_err(|e| super::super::sections::py_error("ValueError", &e)) + }; + if !sized.is_empty() { + let (lo, hi) = (bound("intermediate_min")?, bound("intermediate_max")?); + if sized.iter().any(|c| lo <= *c && *c <= hi) { + let cell = |k: &str| { + rng.get(k) + .cloned() + .ok_or_else(|| super::util::key_error(k)) + }; + item.put( + "zone", + s(&format!( + "{}-{}", + cell("intermediate_min")?, + cell("intermediate_max")? + )), + ); + item.put("zone_label", s(&cell("label")?)); + item.put("zone_citation", s(&cell("citation")?)); + } + } + } + let abnormal = status.iter().any(|x| *x != "normal"); + if rep == "RFC1" && st.is_some() && abnormal { + item.put( + "rfc1_motif", + rfc1.clone().unwrap_or_else(|| { + Obj::new() + .set("verdict", s("not_checked")) + .set("note", s("no motif count beside Stranger's output (step 9b runs it with the sample's BAM): confirm the flag with repeat-primed PCR")) + .val() + }), + ); + } + if rep != "RFC1" + && abnormal + && let Some(m) = motifs.get(&rep) + { + item.put("motif", m.clone()); + } + if st.is_some() + && status.is_empty() + && let Some(why) = ungraded.get(&lid) + { + item.put("not_graded", s(why)); + } + if st.is_some() { + if status.is_empty() && item.0.contains_key("not_graded") { + if !graded_none.contains(&lid) { + graded_none.push(lid.clone()); + } + } else if status.is_empty() { + no_status += 1; + } else if abnormal { + let mut f = Obj::new(); + for k in ["locus", "repeat_count", "status", "sizing"] { + f.put(k, item.0[k].clone()); + } + for k in ["rfc1_motif", "motif"] { + if let Some(v) = item.0.get(k) { + f.put(k, v.clone()); + } + } + flagged.push(f.val()); + } + } + loci.push(Locus { + json: item, + locus: rep, + }); + } + let by_unsorted = by_locus(loci.iter().map(|l| (&l.locus, &l.json))); + loci.sort_by(|a, b| { + let rank = |l: &str| EH_LOCI.iter().position(|k| *k == l).unwrap_or(EH_LOCI.len()); + rank(&a.locus) + .cmp(&rank(&b.locus)) + .then_with(|| a.locus.cmp(&b.locus)) + }); + let key_loci = EH_LOCI + .iter() + .map(|k| { + Obj::new() + .set("locus", s(k)) + .set( + "repeat_count", + by_unsorted.get(*k).map_or_else(|| s("not in output"), |l| l.repeat_count.clone()), + ) + .val() + }) + .collect(); + let word = |l: &Locus, k: &str| match l.json.0.get(k) { + Some(Value::Str(x)) => x.clone(), + _ => String::new(), + }; + let unsized_: Vec = loci + .iter() + .filter(|l| !Sizing::of(&word(l, "sizing")).is_some_and(Sizing::read)) + .map(|l| l.locus.clone()) + .collect(); + let intermediate: Vec = loci + .iter() + .filter(|l| !word(l, "zone").is_empty()) + .map(|l| l.locus.clone()) + .collect(); + let readings = readings(c, &by_unsorted, State::Ok, State::Ok)?; + let mut out = Obj::new() + .set("records", count(tested)) + .set("key_loci", Value::Array(key_loci)) + .set( + "loci", + Value::Array(loci.into_iter().map(|l| l.json.val()).collect()), + ) + .set("stranger", Value::Bool(st.is_some())) + .set("stranger_outdated", Value::Bool(outdated)) + .set("unsized", strs(&unsized_)) + .set("intermediate", strs(&intermediate)) + .set("readings", readings); + if st.is_some() { + out.put("flagged", Value::Array(flagged)); + out.put("no_status", count(no_status)); + out.put( + "not_graded", + Value::Object( + graded_none + .iter() + .map(|k| (k.clone(), s(&ungraded[k]))) + .collect(), + ), + ); + } + + Ok(Some((p, out.val()))) +} + +/// What a reading needs of a locus: its count and how well it was sized. +#[derive(Debug, Clone)] +struct Sized { + repeat_count: Value, + sizing: Option, +} + +/// `{locus: l}` over loci in this order, the last of a name kept. +fn by_locus<'a>(loci: impl Iterator) -> IndexMap { + loci.map(|(k, o)| { + let get = |f: &str| o.0.get(f).cloned().unwrap_or(Value::Null); + let sizing = match get("sizing") { + Value::Str(w) => Sizing::of(&w), + _ => None, + }; + ( + k.clone(), + Sized { + repeat_count: get("repeat_count"), + sizing, + }, + ) + }) + .collect() +} + +/// The two counts of a sized record, sorted. +fn counts(item: Option<&Sized>) -> Option<[i128; 2]> { + let item = item?; + if !item.sizing.is_some_and(Sizing::read) { + return None; + } + let Value::Str(rc) = &item.repeat_count else { + return None; + }; + let mut c: Vec> = rc.split('/').map(str_int).collect(); + if c.len() == 1 && c[0].is_some() { + c.push(c[0]); + } + match c[..] { + [Some(a), Some(b)] => Some(if a <= b { [a, b] } else { [b, a] }), + _ => None, + } +} + +fn text_of(v: &Value) -> String { + match v { + Value::Str(x) => x.clone(), + other => pycompat::value::py_str(other).unwrap_or_default(), + } +} + +fn sizing_text(item: &Sized) -> String { + item.sizing.map_or("None", Sizing::word).to_owned() +} + +/// What a section state other than ok says about another step's table. +fn other_state(st: State) -> &'static str { + match st { + State::Stale => "from an earlier run", + State::Failed => "its step failed in the last run", + _ => "unreadable", + } +} + +/// R117H's genotype in step 42's traits table: (genotype, why not read, +/// a note on a table of an earlier run). +fn r117h(c: &Ctx<'_>, state: State) -> (Option, String, String) { + if matches!(state, State::Failed | State::Unreadable) { + return ( + None, + format!("step 42's traits table is {}", other_state(state)), + String::new(), + ); + } + let row = c + .find1(format!("traits/{}_traits.tsv", c.s())) + .and_then(|p| table(&p.path, false).ok()) + .and_then(|t| { + t.rows + .into_iter() + .find(|r| r.get("rsid").flatten() == Some(CFTR_R117H)) + }); + let called = row.as_ref().map(|r| { + ( + super::util::or(r, "status", ""), + super::util::or(r, "genotype", ""), + ) + }); + match called { + Some((st, gt)) if st == "called" => ( + Some(gt), + String::new(), + if state == State::Stale { + " (R117H from step 42's traits table of an earlier run)".into() + } else { + String::new() + }, + ), + _ => ( + None, + "no step 42 traits row for rs78655421 was called".into(), + String::new(), + ), + } +} + +fn a_copies(gt: &str) -> usize { + gt.split('/').filter(|x| *x == "A").count() +} + +fn cftr(c: &Ctx<'_>, by: &IndexMap, traits: State) -> Value { + let out = Obj::new() + .set("locus", s("CFTR")) + .set("repeat", s("intron 9 (legacy IVS8) TG and poly-T tract")) + .set("source", s(CFTR_SOURCE)); + let not_sized = |o: Obj, reading: String| { + o.set("status", s("not_sized")) + .set("genotype", s(".")) + .set("phase", s(".")) + .set("reading", s(&reading)) + .val() + }; + let (Some(tg), Some(t)) = (by.get("CFTR_TG"), by.get("CFTR_T")) else { + return not_sized( + out, + format!( + "not in step 09's output: {EXTRA_SETUP} adds the tract to the catalog (docs/09-str-expansions.md)." + ), + ); + }; + let (Some(a), Some(b)) = (counts(Some(tg)), counts(Some(t))) else { + return not_sized( + out, + format!( + "ExpansionHunter did not size both tracts (TG {}, {}; poly-T {}, {}): not read.", + text_of(&tg.repeat_count), + sizing_text(tg), + text_of(&t.repeat_count), + sizing_text(t) + ), + ); + }; + let pairs: Vec = if a[0] == a[1] || b[0] == b[1] { + vec![format!("TG{}-{}T/TG{}-{}T", a[0], b[0], a[1], b[1])] + } else { + vec![ + format!("TG{}-{}T/TG{}-{}T", a[0], b[0], a[1], b[1]), + format!("TG{}-{}T/TG{}-{}T", a[1], b[0], a[0], b[1]), + ] + }; + let genotype = if pairs.len() == 1 { + pairs[0].clone() + } else { + format!("TG{}/TG{} {}T/{}T", a[0], a[1], b[0], b[1]) + }; + let ints = |x: [i128; 2]| Value::Array(x.iter().map(|v| Value::Int(*v)).collect()); + let out = out + .set("status", s("read")) + .set("genotype", s(&genotype)) + .set("pairings", strs(&pairs)) + .set("tg", ints(a)) + .set("poly_t", ints(b)); + let five = b.iter().filter(|n| **n == 5).count(); + let mut other: Vec = b + .iter() + .copied() + .filter(|n| !matches!(n, 5 | 7 | 9)) + .collect(); + other.sort_unstable(); + other.dedup(); + let odd = if other.is_empty() { + String::new() + } else { + format!( + " A poly-T length of {} is none of the usual 5T, 7T and 9T.", + other + .iter() + .map(|n| format!("{n}T")) + .collect::>() + .join(", ") + ) + }; + let (r117, why, note) = r117h(c, traits); + let r117_text = s(r117.as_deref().filter(|x| !x.is_empty()).unwrap_or(".")); + if five == 0 { + let mut line = format!( + "No 5T copy ({}T/{}T): the tract adds no 5T allele of its own. The poly-T length still matters beside R117H, which is milder and of variable penetrance on a 7T or 9T copy and CF-causing with 5T in cis.", + b[0], b[1] + ); + match &r117 { + None => line += &format!(" R117H was not read ({why})."), + Some(g) if a_copies(g) == 2 => { + let copies = if b[0] == b[1] { + format!("both {}T copies", b[0]) + } else { + format!("both copies ({}T and {}T)", b[0], b[1]) + }; + line += &format!(" R117H ({g}) is read: it sits on {copies}.{note}"); + } + Some(g) if a_copies(g) > 0 => { + let copies = if b[0] == b[1] { + format!("{}T", b[0]) + } else { + format!("{}T or {}T", b[0], b[1]) + }; + line += &format!(" R117H ({g}) is read: with no 5T copy it sits on a {copies} copy.{note}"); + } + Some(_) => line += &format!(" No R117H copy.{note}"), + } + return out + .set("five_t", Value::Bool(false)) + .set("phase", s("not_needed")) + .set("r117h", r117_text) + .set("reading", s(&(line + &odd))) + .val(); + } + let tg_text = if pairs.len() == 1 { + let tgs: Vec = if a[0] == a[1] { vec![a[0]] } else { a.to_vec() }; + format!( + "{}{}", + tgs.iter() + .map(|x| format!("TG{x}")) + .collect::>() + .join(" and "), + if five == 1 { + " on the 5T copy" + } else { + " on the 5T copies" + } + ) + } else { + format!( + "TG{} or TG{} on the 5T copy, not told by the counts ({})", + a[0], + a[1], + pairs.join(" or ") + ) + }; + let line = format!( + "5T: {five} copy{} with the 5T poly-T tract, {tg_text}. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant.", + if five > 1 { "s" } else { "" } + ); + let (phase, text) = match &r117 { + None => ( + "not_read", + format!(" R117H was not read ({why}), so whether it sits on the 5T copy is not known."), + ), + Some(g) => { + let n = a_copies(g); + let (p, t) = if n == 0 { + ("not_needed", " No R117H copy: the 5T is read on its own.".to_owned()) + } else if n == 2 || five == 2 { + ( + "in_cis", + format!(" R117H ({g}) and 5T share a copy: R117H with 5T in cis is a CF-causing allele of variable penetrance."), + ) + } else { + ( + "unknown", + " One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart.".to_owned(), + ) + }; + (p, t + ¬e) + } + }; + out.set("five_t", Value::Bool(true)) + .set("phase", s(phase)) + .set("r117h", r117_text) + .set("reading", s(&(line + &text + &odd))) + .val() +} + +fn ugt1a1_name(n: i128) -> Option<&'static str> { + match n { + 6 => Some("*36"), + 7 => Some("*1"), + 8 => Some("*28"), + 9 => Some("*37"), + _ => None, + } +} + +fn ugt1a1(c: &Ctx<'_>, by: &IndexMap, cpic: State) -> Value { + let out = Obj::new() + .set("locus", s("UGT1A1")) + .set("repeat", s("promoter TA repeat")) + .set("source", s(UGT1A1_SOURCE)); + let not_sized = |o: Obj, reading: String| { + o.set("status", s("not_sized")) + .set("genotype", s(".")) + .set("pharmcat", s(".")) + .set("agreement", s("not_compared")) + .set("reading", s(&reading)) + .val() + }; + let Some(item) = by.get("UGT1A1") else { + return not_sized( + out, + format!( + "not in step 09's output: {EXTRA_SETUP} adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + ), + ); + }; + let Some(cn) = counts(Some(item)) else { + return not_sized( + out, + format!( + "ExpansionHunter did not size the repeat ({}, {}): not read.", + text_of(&item.repeat_count), + sizing_text(item) + ), + ); + }; + let names: Vec = cn + .iter() + .map(|n| ugt1a1_name(*n).map_or_else(|| format!("TA{}", n - 1), str::to_owned)) + .collect(); + let genotype = names.join("/"); + let dec = names.iter().filter(|n| *n == "*28" || *n == "*37").count(); + let ta = cn + .iter() + .map(|n| format!("TA{}", n - 1)) + .collect::>() + .join("/"); + let out = out + .set("status", s("read")) + .set("genotype", s(&genotype)) + .set( + "ta_units", + Value::Array(cn.iter().map(|v| Value::Int(*v)).collect()), + ); + let mut reading = format!( + "{genotype} ({ta}, {}/{} TA units as ExpansionHunter counts them): ", + cn[0], cn[1] + ); + if !cn.iter().all(|n| ugt1a1_name(*n).is_some()) { + reading += "a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)."; + return out + .set("pharmcat", s(".")) + .set("agreement", s("not_compared")) + .set("reading", s(&reading)) + .val(); + } + reading += &if dec == 0 { + "no decreased-function TA allele.".to_owned() + } else { + format!( + "{dec} decreased-function allele{} (*28 and *37 are decreased function).", + if dec > 1 { "s" } else { "" } + ) + }; + let (mut pc, mut agreement) = (".".to_owned(), "not_compared"); + let readable = !matches!(cpic, State::Failed | State::Unreadable); + let row = c + .find1(format!("cpic/{}_phenotypes.tsv", c.s())) + .filter(|_| readable) + .and_then(|p| table(&p.path, false).ok()) + .and_then(|t| { + t.rows + .into_iter() + .find(|r| r.get("Gene").flatten() == Some("UGT1A1")) + }); + if let Some(row) = row { + let d = super::util::or(&row, "Diplotype", ""); + let d = pycompat::num::py_strip(&d); + pc = if d.is_empty() { ".".into() } else { d.to_owned() }; + let alleles: Vec<&str> = pc.split('/').collect(); + if alleles.len() == 2 + && !pc.contains("possible") + && super::util::or(&row, "Status", "") != "not called" + { + let n = alleles + .iter() + .filter(|a| { + a.split(['+', ' ', '[', ']']) + .any(|x| x == "*28" || x == "*37") + }) + .count(); + agreement = if n == dec { "agrees" } else { "differs" }; + } + } + let old = if cpic == State::Stale && agreement != "not_compared" { + " (step 27's table is from an earlier run)" + } else { + "" + }; + reading += &match agreement { + "agrees" => format!( + " PharmCAT (step 7) called {pc}: the same number of *28 and *37 copies{old}." + ), + "differs" => format!( + " PharmCAT (step 7) called {pc}, a different number of *28 and *37 copies{old}: read the reads at chr2:233760234 before using either." + ), + _ if !readable => format!( + " Not compared with PharmCAT (step 27's table is {}).", + other_state(cpic) + ), + _ => " Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27).".to_owned(), + }; + out.set("pharmcat", s(&pc)) + .set("agreement", s(agreement)) + .set("reading", s(&reading)) + .val() +} + +/// What the counts of the extra repeats mean, and the 5-HTTLPR line no +/// step can test. +fn readings( + c: &Ctx<'_>, + by: &IndexMap, + traits: State, + cpic: State, +) -> Result { + Ok(Value::Array(vec![ + cftr(c, by, traits), + ugt1a1(c, by, cpic), + Obj::new() + .set("locus", s("SLC6A4")) + .set("repeat", s("5-HTTLPR promoter length polymorphism")) + .set("status", s("not_tested")) + .set("genotype", s(".")) + .set("reading", s(HTTLPR_READING)) + .set("source", s("Bousman et al. 2023, PMID 37032427")) + .val(), + ])) +} + +/// The post pass: the readings again, with the traits and CPIC states, over +/// the loci as the section lists them. +pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { + let Value::Object(m) = v else { return }; + let loci: Vec<(String, Obj)> = match m.get("loci") { + Some(Value::Array(xs)) => xs + .iter() + .filter_map(|x| match x { + Value::Object(o) => match o.get("locus") { + Some(Value::Str(l)) => Some((l.clone(), Obj(o.clone()))), + _ => None, + }, + _ => None, + }) + .collect(), + _ => return, + }; + let by = by_locus(loci.iter().map(|(k, o)| (k, o))); + if let Ok(r) = readings(c, &by, traits, cpic) { + m.insert("readings".into(), r); + } +} diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs new file mode 100644 index 000000000..ca743eb77 --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -0,0 +1,419 @@ +//! Step 11 (bcftools roh): the runs of homozygosity of chr1 to chr22 +//! outside GRCh38's centromere and heterochromatin bands, F_ROH, the +//! relationship band it is nearest to and the one-chromosome (UPD) flag; +//! and `og roh summary`, the step's own table of the long runs. +//! +//! [`MASKED_BANDS`] is the one copy of the bands: the summary section, +//! CNVpytor's count and `og roh summary` (step 11 and the ROH module) all +//! read it here. + +use super::util::{Obj, count, lines, s}; +use super::{Ctx, Out}; +use pycompat::json::Value; +use std::io::Write; + +/// GRCh38's centromeres and heterochromatin: the acen, gvar and stalk rows +/// of UCSC's cytoBand.hg38.txt (the file setup.sh installs as +/// reference/cytoBand.hg38.txt), neighbouring rows merged, as (chrom, +/// 0-based start, exclusive end). Reads there map badly, so a run of +/// homozygosity or a copy number call over them is mostly an artefact. +/// Regenerate with +/// gzip -cd cytoBand.txt.gz | awk '$1 ~ /^chr([0-9]+|X|Y)$/ && $5 ~ /^(acen|gvar|stalk)$/' \ +/// | sort -k1,1V -k2,2n | awk '{ if ($1 == c && $2 == e) e = $3; else { if (c != "") printf "%s:%d-%d ", c, s, e; +/// c = $1; s = $2; e = $3 } } END { print c ":" s "-" e }' +pub const MASKED_BANDS: [(&str, i128, i128); 25] = [ + ("chr1", 121_700_000, 143_200_000), + ("chr2", 91_800_000, 96_000_000), + ("chr3", 87_800_000, 98_600_000), + ("chr4", 48_200_000, 51_800_000), + ("chr5", 46_100_000, 51_400_000), + ("chr6", 58_500_000, 62_600_000), + ("chr7", 58_100_000, 62_100_000), + ("chr8", 43_200_000, 47_200_000), + ("chr9", 42_200_000, 61_500_000), + ("chr10", 38_000_000, 41_600_000), + ("chr11", 51_000_000, 55_800_000), + ("chr12", 33_200_000, 37_800_000), + ("chr13", 0, 18_900_000), + ("chr14", 0, 18_200_000), + ("chr15", 0, 20_500_000), + ("chr16", 35_300_000, 47_000_000), + ("chr17", 22_700_000, 27_400_000), + ("chr18", 15_400_000, 21_500_000), + ("chr19", 19_900_000, 31_900_000), + ("chr20", 25_700_000, 30_400_000), + ("chr21", 0, 13_000_000), + ("chr22", 0, 17_400_000), + ("chrX", 58_100_000, 63_800_000), + ("chrY", 10_300_000, 10_600_000), + ("chrY", 26_600_000, 57_227_415), +]; + +/// GRCh38's chr1 to chr22 (UCSC hg38.chrom.sizes). +pub const AUTOSOME_BP: i128 = 2_875_001_522; +/// The runs F_ROH and the one-chromosome flag count (docs/11-roh-analysis.md). +pub const ROH_MIN_BP: f64 = 5e6; +/// One chromosome carrying most of the long runs: at least this much of it +/// in runs of 5 Mb or more ... +pub const UPD_MIN_BP: f64 = 20e6; +/// ... and more than this share of all such autosomal runs. +pub const UPD_SHARE: f64 = 0.5; + +/// Bases of the 1-based, inclusive span `start..=end` inside the masked +/// bands of `chrom`. +pub fn masked_overlap(chrom: &str, start: i128, end: i128) -> i128 { + MASKED_BANDS + .iter() + .filter(|(c, ..)| *c == chrom) + .map(|(_, a, b)| overlap(*a, *b, start, end)) + .sum() +} + +fn overlap(a: i128, b: i128, start: i128, end: i128) -> i128 { + (end.min(b) - (start - 1).max(a)).max(0) +} + +/// Whether a chromosome name is one of chr1 to chr22. +pub fn is_autosome(chrom: &str) -> bool { + chrom + .strip_prefix("chr") + .and_then(|n| n.parse::().ok().filter(|v| (1..=22).contains(v)).map(|v| (v, n))) + .is_some_and(|(v, n)| n == v.to_string()) +} + +/// [`AUTOSOME_BP`] without the autosomal bands: the length F_ROH is a share +/// of, since a run's part in a band is not counted either. +pub fn autosome_unmasked_bp() -> i128 { + AUTOSOME_BP + - MASKED_BANDS + .iter() + .filter(|(c, ..)| is_autosome(c)) + .map(|(_, a, b)| b - a) + .sum::() +} + +/// docs/11-roh-analysis.md's table: F of a child of the parents. A value +/// goes to the row whose F is nearest on a log scale (the boundaries are +/// the geometric means of neighbouring rows), and under 1/128 (half the +/// second-cousin F) to "no sign of related parents". +pub fn roh_band(f: f64) -> &'static str { + let gm = |a: f64, b: f64| (a * b).powf(0.5); + let bands = [ + (1.0 / 128.0, "no sign of related parents"), + (gm(1.0 / 64.0, 1.0 / 32.0), "second cousins (F 1/64)"), + ( + gm(1.0 / 32.0, 1.0 / 16.0), + "first cousins once removed (F 1/32)", + ), + (gm(1.0 / 16.0, 1.0 / 8.0), "first cousins (F 1/16)"), + ( + gm(1.0 / 8.0, 1.0 / 4.0), + "half siblings, uncle and niece, double first cousins (F 1/8)", + ), + (f64::INFINITY, "parent and child, full siblings (F 1/4)"), + ]; + bands + .iter() + .find(|(top, _)| f < *top) + .map_or("parent and child, full siblings (F 1/4)", |b| b.1) +} + +/// assets/imprinted_regions.tsv, the same rows (a test keeps the two +/// equal): chrom, the region's span (none for the whole chromosome), the +/// region, the maternal and paternal UPD syndromes and the PMIDs. +/// chrom, span, then four text fields. +type Imprinted = (&'static str, Option<(i128, i128)>, &'static str, &'static str, &'static str, &'static str); + +pub const IMPRINTED_REGIONS: [Imprinted; 6] = [ + ( + "chr6", + None, + "6q24 (PLAGL1, HYMAI)", + "-", + "Transient neonatal diabetes mellitus (6q24-related)", + "20301706", + ), + ( + "chr7", + None, + "7p12 (GRB10), 7q32 (MEST)", + "Silver-Russell syndrome", + "-", + "27585961", + ), + ( + "chr11", + Some((0, 22_000_000)), + "11p15 (H19/IGF2, KCNQ1OT1)", + "Silver-Russell syndrome", + "Beckwith-Wiedemann syndrome", + "29377879;27585961", + ), + ( + "chr14", + None, + "14q32.2 (DLK1-MEG3)", + "Temple syndrome", + "Kagami-Ogata syndrome", + "24891339;26377239", + ), + ( + "chr15", + Some((20_500_000, 33_400_000)), + "15q11-q13 (SNRPN, UBE3A)", + "Prader-Willi syndrome", + "Angelman syndrome", + "22237428;27615419", + ), + ( + "chr20", + None, + "20q13.32 (GNAS)", + "Mulchandani-Bhoj-Conlin syndrome (growth failure)", + "Pseudohypoparathyroidism type 1B", + "26248010;29959430", + ), +]; + +/// The imprinted regions a flagged chromosome joins: every row of the +/// whole chromosome, and a row with a region only when a run overlaps it. +fn imprinted(chrom: &str, runs: &[(i128, i128)]) -> Value { + Value::Array( + IMPRINTED_REGIONS + .iter() + .filter(|r| r.0 == chrom) + .filter(|r| { + r.1.is_none_or(|(a, b)| runs.iter().any(|&(s, e)| overlap(a, b, s, e) > 0)) + }) + .map(|r| { + Obj::new() + .set("region", s(r.2)) + .set("maternal_upd", s(r.3)) + .set("paternal_upd", s(r.4)) + .set("pmid", s(r.5)) + .val() + }) + .collect(), + ) +} + +fn r1(x: f64, n: i32) -> Value { + super::util::round(x, n) +} + +/// Step 11's bcftools roh output: runs on chrX and chrY counted apart, runs +/// on any contig other than chr1 to chr22 left out, and the part of a run +/// in a masked band not counted. +pub fn section(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find(&[ + format!("vcf/{s_}_roh.txt"), + format!("roh/{s_}_roh.txt"), + "roh/*_roh.txt".to_owned(), + ]) else { + return Ok(None); + }; + let (mut total, mut largest, mut masked) = (0.0_f64, 0.0_f64, 0.0_f64); + let (mut segs, mut sex_segs) = (0usize, 0usize); + let mut big = Vec::new(); + // chrom -> the bases of its long runs, and the runs (start, end) + let mut per_chrom: indexmap::IndexMap)> = + indexmap::IndexMap::new(); + for line in lines(&p.path)? { + let c = pycompat::text::split(&line, None, None); + if c.len() < 6 || c[0] != "RG" { + continue; + } + let (Ok(start), Ok(end), Ok(mut length)) = ( + crate::int(c[3]), + crate::int(c[4]), + pycompat::num::parse_float(c[5]), + ) else { + continue; + }; + if !length.is_finite() { + return Err(format!( + "{} has a run of length {}", + crate::basename(&p.path), + c[5] + )); + } + if matches!(c[2], "chrX" | "chrY" | "X" | "Y") { + sex_segs += 1; + continue; + } + if !is_autosome(c[2]) { + continue; + } + let inside = length.min(masked_overlap(c[2], start, end) as f64); + masked += inside; + length -= inside; + if length <= 0.0 { + continue; + } + segs += 1; + total += length; + largest = largest.max(length); + if length >= ROH_MIN_BP { + let pc = per_chrom.entry(c[2].to_owned()).or_default(); + pc.0 += length; + pc.1.push((start, end)); + let mut row = Obj::new() + .set("region", s(&format!("{}:{}-{}", c[2], c[3], c[4]))) + .set("mb", r1(length / 1e6, 1)); + if inside != 0.0 { + row.put("masked_mb", r1(inside / 1e6, 1)); + } + big.push(row.val()); + } + } + let long_bp: f64 = per_chrom.values().fold(0.0, |a, v| a + v.0); + let f_roh = long_bp / autosome_unmasked_bp() as f64; + let mut upd = Value::Null; + // The first chromosome of the most bases, as max() picks. + let top = per_chrom + .iter() + .fold(None::<(&String, &(f64, Vec<(i128, i128)>))>, |best, kv| match best { + Some(b) if b.1.0 >= kv.1.0 => Some(b), + _ => Some(kv), + }); + if let Some((chrom, (bp, runs))) = top + && *bp >= UPD_MIN_BP + && *bp > UPD_SHARE * long_bp + { + upd = Obj::new() + .set("chrom", s(chrom)) + .set("mb", r1(bp / 1e6, 1)) + .set("share", r1(bp / long_bp, 2)) + .set("imprinted", imprinted(chrom, runs)) + .val(); + } + let relationship = if upd == Value::Null { + s(roh_band(f_roh)) + } else { + Value::Null + }; + let data = Obj::new() + .set("segments", count(segs)) + .set("total_mb", r1(total / 1e6, 1)) + .set("largest_mb", r1(largest / 1e6, 1)) + .set("autosomal_over_5mb", Value::Array(big)) + .set("masked_mb", r1(masked / 1e6, 1)) + .set("sex_chromosome_segments", count(sex_segs)) + .set("long_mb", r1(long_bp / 1e6, 1)) + .set("f_roh", r1(f_roh, 4)) + .set("relationship", relationship) + .set("upd_flag", upd); + Ok(Some((p, data.val()))) +} + +// --- og roh summary ------------------------------------------------------------ + +const USAGE: &str = "usage: og roh summary --sample SAMPLE --roh ROH --out OUT\n"; + +/// A field as awk reads it as a number: the longest leading decimal +/// number, else 0. +fn awk_num(x: &str) -> f64 { + let b = x.as_bytes(); + let mut i = 0; + if matches!(b.first(), Some(b'+' | b'-')) { + i += 1; + } + let digits = |i: &mut usize| { + let s = *i; + while b.get(*i).is_some_and(u8::is_ascii_digit) { + *i += 1; + } + *i - s + }; + let mut n = digits(&mut i); + if b.get(i) == Some(&b'.') { + i += 1; + n += digits(&mut i); + } + if n == 0 { + return 0.0; + } + let mantissa_end = i; + if matches!(b.get(i), Some(b'e' | b'E')) { + let mut j = i + 1; + if matches!(b.get(j), Some(b'+' | b'-')) { + j += 1; + } + if digits(&mut j) > 0 { + i = j; + } else { + i = mantissa_end; + } + } + x[..i].parse().unwrap_or(0.0) +} + +/// The rows of step 11's summary: every run on chr1 to chr22 whose part +/// outside the masked bands is 5 Mb or more, with that part's length. +pub fn summary_rows(roh: &str) -> Result { + let mut out = String::new(); + for line in lines(roh)? { + // awk's fields: runs of blanks, leading ones dropped. + let f: Vec<&str> = line.split([' ', '\t']).filter(|x| !x.is_empty()).collect(); + if f.len() < 3 || f[0] != "RG" || !is_autosome(f[2]) { + continue; + } + let get = |i: usize| f.get(i).copied().unwrap_or(""); + let (start, end, len) = (awk_num(get(3)), awk_num(get(4)), awk_num(get(5))); + let mut m = 0.0; + for (c, a, b) in MASKED_BANDS { + if c == f[2] { + let o = end.min(b as f64) - (start - 1.0).max(a as f64); + if o > 0.0 { + m += o; + } + } + } + if m > len { + m = len; + } + let l = len - m; + if l >= 5_000_000.0 { + out.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\n", + f[2], + get(3), + get(4), + l.trunc() as i64, + pycompat::fmt::fixed(l / 1e6, 1) + )); + } + } + Ok(out) +} + +/// `og roh summary`: step 11's summary table of the long runs, for the bash +/// step and the ROH module alike. Exit 0 when written, 1 when the roh file +/// cannot be read or the table written, 2 for a command line it cannot +/// read. +pub fn cli(argv: &[String], err: &mut impl Write) -> u8 { + use crate::args::{Kind, Opt, parse}; + const OPTS: &[Opt] = &[ + Opt("--sample", Kind::Value, true), + Opt("--roh", Kind::Value, true), + Opt("--out", Kind::Value, true), + ]; + let a = match parse(argv, OPTS) { + Ok(a) => a, + Err(e) => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og roh summary: error: {e}"); + return 2; + } + }; + let run = || -> Result<(), String> { + let rows = summary_rows(a.text("--roh"))?; + let body = format!( + "# ROH Summary for {}\n# Segments >=5MB on autosomes (excludes chrX/chrY)\nchrom\tstart\tend\tlength_bp\tlength_mb\n{rows}", + a.text("--sample") + ); + crate::write_file(a.text("--out"), body.as_bytes()) + }; + crate::finish(run(), err) +} diff --git a/rust/crates/og-summary/src/collect/group_c/sv.rs b/rust/crates/og-summary/src/collect/group_c/sv.rs new file mode 100644 index 000000000..55195c58e --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/sv.rs @@ -0,0 +1,461 @@ +//! Structural variants: Manta's and Delly's record counts, CNVpytor's +//! calls with the significance and quality filters, step 22's consensus +//! count, and step 05's class 4 and 5 calls with their support from the +//! consensus. + +use super::roh::masked_overlap; +use super::util::{Obj, at, base, cells, count, int, lines, number, s, strs, table, vcf_lines}; +use super::{Ctx, Found, Out}; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use std::collections::HashMap; + +/// A record count and the records that PASS. +fn count_vcf(path: &str) -> Result<(usize, usize), String> { + let (mut total, mut pass) = (0, 0); + vcf_lines(path, |l| { + total += 1; + if l.split(|&c| c == b'\t').nth(6) == Some(b"PASS") { + pass += 1; + } + })?; + Ok((total, pass)) +} + +fn totals(p: Found) -> Out { + let (t, ps) = count_vcf(&p.path)?; + Ok(Some(( + p, + Obj::new().set("total", count(t)).set("pass", count(ps)).val(), + ))) +} + +/// Step 04 (Manta). +pub fn manta(c: &Ctx<'_>) -> Out { + match c.find(&[ + "manta/results/variants/diploidSV.vcf.gz".to_owned(), + "manta/*diploidSV.vcf.gz".to_owned(), + "manta/**/diploidSV.vcf.gz".to_owned(), + ]) { + Some(p) => totals(p), + None => Ok(None), + } +} + +/// Step 19 (Delly). +pub fn delly(c: &Ctx<'_>) -> Out { + match c.find1(format!("delly/{}_sv.vcf.gz", c.s())) { + Some(p) => totals(p), + None => Ok(None), + } +} + +/// Step 22's consensus count. +pub fn sv_consensus(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("sv_merged/{}_sv_consensus.vcf.gz", c.s())) else { + return Ok(None); + }; + let (t, _) = count_vcf(&p.path)?; + Ok(Some((p, Obj::new().set("consensus", count(t)).val()))) +} + +/// A CNVpytor call is kept in the filtered count when fewer than half of +/// its reads have mapping quality 0 and fewer than half of its bases are N, +/// and it touches no masked band. +const CNV_MAX_Q0: f64 = 0.5; +const CNV_MAX_PN: f64 = 0.5; + +/// `(\S+):(\d+)-(\d+)` over a whole cell. +fn span(x: &str) -> Option<(&str, i128, i128)> { + let (c, rest) = x.rsplit_once(':')?; + let (a, b) = rest.split_once('-')?; + let digits = |v: &str| !v.is_empty() && v.bytes().all(|d| d.is_ascii_digit()); + if c.is_empty() || c.chars().any(char::is_whitespace) || !digits(a) || !digits(b) { + return None; + } + Some((c, a.parse().ok()?, b.parse().ok()?)) +} + +/// Step 18 (CNVpytor): the calls, the significant ones (e-value under +/// 0.01), and those kept after the quality and band filters. +pub fn cnvpytor(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("cnvpytor/{}_cnvs.txt", c.s())) else { + return Ok(None); + }; + let (mut total, mut dels, mut dups, mut sig, mut kept, mut kept_y) = (0, 0, 0, 0, 0, 0); + let (mut q0s, mut gaps, mut bands) = (0, 0, 0); + for line in lines(&p.path)? { + let c = pycompat::text::split(&line, None, None); + if c.is_empty() { + continue; + } + total += 1; + dels += usize::from(c[0] == "deletion"); + dups += usize::from(c[0] == "duplication"); + let m = if c.len() > 4 { span(c[1]) } else { None }; + match c.get(4).map(|v| pycompat::num::parse_float(v)) { + Some(Ok(e)) if e.partial_cmp(&0.01).is_none_or(|o| o.is_lt()) => {} + _ => continue, + } + sig += 1; + let (q0, pn) = if c.len() > 9 { + (number(c[8]), number(c[9])) + } else { + (None, None) + }; + if q0.is_some_and(|q| q >= CNV_MAX_Q0) { + q0s += 1; + } else if pn.is_some_and(|q| q >= CNV_MAX_PN) { + gaps += 1; + } else if let Some((ch, a, b)) = m + && masked_overlap(ch, a, b) > 0 + { + bands += 1; + } else { + kept += 1; + kept_y += usize::from(m.is_some_and(|(ch, ..)| ch == "chrY" || ch == "Y")); + } + } + let data = Obj::new() + .set("total", count(total)) + .set("deletions", count(dels)) + .set("duplications", count(dups)) + .set("significant", count(sig)) + .set("significant_filtered", count(kept)) + .set("filtered_chry", count(kept_y)) + .set( + "dropped", + Obj::new() + .set("q0", count(q0s)) + .set("gap", count(gaps)) + .set("band", count(bands)) + .val(), + ); + Ok(Some((p, data.val()))) +} + +/// Takes the chrY calls out of the filtered count of a declared female; +/// with no declared sex they stay, and the data says so. +pub fn cnvpytor_sex_filter(v: &mut Value, declared_sex: Option<&str>) { + let Value::Object(m) = v else { return }; + let (Some(Value::Int(kept)), Some(Value::Int(y))) = + (m.get("significant_filtered").cloned(), m.get("filtered_chry").cloned()) + else { + return; + }; + let declared = declared_sex.filter(|x| !x.is_empty()); + m.insert( + "declared_sex".into(), + declared.map_or(Value::Null, s), + ); + if declared == Some("female") { + if let Some(Value::Object(d)) = m.get_mut("dropped") { + d.insert("chry_female".into(), Value::Int(y)); + } + m.insert("significant_filtered".into(), Value::Int(kept - y)); + } +} + +// --- step 05, AnnotSV class 4 and 5 --------------------------------------------- + +/// Two breakpoints within this many bases of a consensus record's are the +/// same SV. +const CONSENSUS_BP: i128 = 1000; + +/// AnnotSV writes chromosomes without `chr`; step 22 writes them with it. +fn chrom_key(c: &str) -> &str { + if c.get(..3).is_some_and(|p| p.eq_ignore_ascii_case("chr")) { + &c[3..] + } else { + c + } +} + +fn info_value<'a>(info: &'a str, key: &str) -> &'a str { + info.split(';') + .find_map(|kv| { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + (k == key).then_some(v) + }) + .unwrap_or("") +} + +fn kind(t: &str) -> String { + let t = pycompat::text::upper(t); + if t.starts_with("DUP") { "DUP".into() } else { t } +} + +struct Rec { + pos: i128, + end: i128, + supp: i128, + vec: String, +} + +/// Step 22's consensus: caller names, and the records by (chrom, type). +/// The header and the records of the consensus VCF by (chrom, id). +type Consensus = (Vec, HashMap<(String, String), Vec>); + +fn read_consensus(path: &str) -> Result { + let mut names = Vec::new(); + let mut by: HashMap<(String, String), Vec> = HashMap::new(); + for line in lines(path)? { + if line.starts_with("##") { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if line.starts_with("#CHROM") { + names = c.iter().skip(9).map(|x| (*x).to_owned()).collect(); + continue; + } + if c.len() < 8 { + continue; + } + let Ok(pos) = crate::int(c[1]) else { continue }; + let end = match info_value(c[7], "END") { + "" => Ok(pos), + v => crate::int(v), + }; + let supp = match info_value(c[7], "SUPP") { + "" => Ok(0), + v => crate::int(v), + }; + let (Ok(end), Ok(supp)) = (end, supp) else { + continue; + }; + by.entry(( + chrom_key(c[0]).to_owned(), + kind(info_value(c[7], "SVTYPE")), + )) + .or_default() + .push(Rec { + pos, + end, + supp, + vec: info_value(c[7], "SUPP_VEC").to_owned(), + }); + } + Ok((names, by)) +} + +/// Support and the supporting callers of each row from the consensus: the +/// record of the same type whose two breakpoints lie within 1,000 bp (the +/// nearest, the first of a tie), else 1 and the row's own caller; `None` +/// for both without a consensus file. +fn consensus_support(rows: &[DictRow], path: Option<&str>) -> Result, String> { + let Some(path) = path.filter(|p| crate::is_file(p)) else { + return Ok(rows.iter().map(|_| (Value::Null, Value::Null)).collect()); + }; + let (names, by) = read_consensus(path)?; + let mut out = Vec::new(); + for r in rows { + let cell = |k: &str| r.get(k).flatten().unwrap_or_default(); + let mut best: Option<(i128, &Rec)> = None; + if let (Ok(start), Ok(end)) = (crate::int(cell("SV_start")), crate::int(cell("SV_end"))) { + let key = (chrom_key(cell("SV_chrom")).to_owned(), kind(cell("SV_type"))); + for rec in by.get(&key).map_or(&[][..], Vec::as_slice) { + let (dp, de) = ((rec.pos - start).abs(), (rec.end - end).abs()); + if dp <= CONSENSUS_BP && de <= CONSENSUS_BP && best.is_none_or(|(d, _)| dp + de < d) + { + best = Some((dp + de, rec)); + } + } + } + out.push(match best { + Some((_, rec)) => { + let callers: Vec = rec + .vec + .chars() + .enumerate() + .filter(|&(_, x)| x == '1') + .map(|(i, _)| { + names + .get(i) + .cloned() + .unwrap_or_else(|| format!("caller {}", i + 1)) + }) + .collect(); + (int(rec.supp), s(&callers.join(","))) + } + None => ( + int(1), + s(&super::util::or(r, "Caller", ".")), + ), + }); + } + Ok(out) +} + +/// `{FORMAT key: value}` of the sample column (the column after FORMAT, +/// unless it is `Annotation_mode`). +fn sample_format(r: &DictRow, head: &[String]) -> HashMap { + let keys: Vec<&str> = r.iter().map(|(k, _)| k).collect(); + let _ = head; + let Some(i) = keys.iter().position(|k| *k == "FORMAT").map(|i| i + 1) else { + return HashMap::new(); + }; + let Some(col) = keys.get(i).filter(|k| **k != "Annotation_mode") else { + return HashMap::new(); + }; + let fmt = r.get("FORMAT").flatten().unwrap_or_default(); + let val = r.get(col).flatten().unwrap_or_default(); + fmt.split(':') + .zip(val.split(':')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect() +} + +/// Step 05's class 4 and 5 SVs, each with the callers that support it from +/// step 22's consensus; a call step 05's rules moved to class 3 listed +/// apart. +pub fn annotsv(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("annotsv/{}_sv_class4_5_checked.tsv", c.s())) else { + return Ok(None); + }; + let size = std::fs::metadata(&p.path).map_or(0, |m| m.len()); + let t = if size > 0 { + table(&p.path, false)? + } else { + super::util::Table { + head: Vec::new(), + rows: Vec::new(), + } + }; + let need = [ + "SV_chrom", + "SV_start", + "SV_end", + "SV_type", + "ACMG_class", + "Caller", + "Exon_edge", + "Population_AF", + "Common", + ]; + if !t.rows.is_empty() && !need.iter().all(|k| t.has(k)) { + let mut sorted = need.to_vec(); + sorted.sort_unstable(); + return Err(format!( + "{} lacks a column of [{}]", + base(&p.path), + sorted + .iter() + .map(|k| crate::repr(k)) + .collect::>() + .join(", ") + )); + } + let mut cons = c.find1(format!("sv_merged/{}_sv_consensus.vcf.gz", c.s())); + let mut cons_note = Value::Null; + let support = match consensus_support(&t.rows, cons.as_ref().map(|f| f.path.as_str())) { + Ok(x) => x, + Err(e) => { + let f = cons.take().map(|f| f.path).unwrap_or_default(); + let why = e.split_once(": ").map_or(e.as_str(), |x| x.1); + cons_note = s(&format!("{} could not be read: {why}", base(&f))); + consensus_support(&t.rows, None)? + } + }; + let mut keep = Vec::new(); + let mut moved = Vec::new(); + for (r, (sp, callers)) in t.rows.iter().zip(support) { + // A row cut short reads None there, as Python's DictReader gives it. + let cell = |k: &str| -> Result { + Ok(at(r, k)?.unwrap_or("None").to_owned()) + }; + let cellv = |k: &str| -> Result { Ok(super::util::opt_s(at(r, k)?)) }; + let af = match at(r, "Population_AF")? { + None | Some("") => Value::Null, + Some(v) => pycompat::num::parse_float(v).map_or(Value::Null, Value::Float), + }; + let genes: Vec = super::util::or(r, "Gene_name", "") + .split(';') + .filter(|g| !g.is_empty()) + .map(str::to_owned) + .collect(); + let fmt = sample_format(r, &t.head); + let fmt_get = |k: &str| fmt.get(k).filter(|v| !v.is_empty()).map_or(Value::Null, |v| s(v)); + let acmg = cell("ACMG_class")?; + let support_callers = match &callers { + Value::Str(x) if !x.is_empty() => strs(x.split(',')), + _ => Value::Array(Vec::new()), + }; + let row = Obj::new() + .set( + "sv", + s(&format!( + "{}:{}-{}", + cell("SV_chrom")?, + cell("SV_start")?, + cell("SV_end")? + )), + ) + .set("type", cellv("SV_type")?) + .set("gt", fmt_get("GT")) + .set("ps", fmt_get("PS")) + .set("length", s(&super::util::or(r, "SV_length", ""))) + .set("acmg_class", cellv("ACMG_class")?) + .set("genes", strs(&genes)) + .set("caller", cellv("Caller")?) + .set("support", sp) + .set("support_callers", support_callers) + .set("exon_edge", cellv("Exon_edge")?) + .set( + "exon_edge_detail", + s(&super::util::or(r, "Exon_edge_detail", "")), + ) + .set("population_af", af) + .set( + "population_source", + s(&super::util::or(r, "Population_source", "")), + ) + .set("common", Value::Bool(at(r, "Common")? == Some("yes"))) + .set( + "annotsv_class", + match r.get("AnnotSV_class").flatten().filter(|v| !v.is_empty()) { + Some(v) => s(v), + None => cellv("ACMG_class")?, + }, + ) + .set("rule", s(&super::util::or(r, "Rule", ""))) + .set("rule_detail", s(&super::util::or(r, "Rule_detail", ""))); + if matches!(acmg.as_str(), "4" | "5") { + keep.push(row.val()); + } else { + moved.push(row.val()); + } + } + let field = |v: &Value, k: &str| -> Value { + match v { + Value::Object(m) => m.get(k).cloned().unwrap_or(Value::Null), + _ => Value::Null, + } + }; + let single = keep + .iter() + .filter(|x| field(x, "support") == Value::Int(1)) + .count(); + let edge = keep + .iter() + .filter(|x| field(x, "exon_edge") == s("yes")) + .count(); + let edge_checked = !keep + .iter() + .any(|x| field(x, "exon_edge") == s("not checked")); + let common = keep + .iter() + .filter(|x| field(x, "common") == Value::Bool(true)) + .count(); + let data = Obj::new() + .set("count", count(keep.len())) + .set("rows", Value::Array(keep)) + .set("downgraded", Value::Array(moved)) + .set("consensus", cons.map_or(Value::Null, |f| s(&f.rel))) + .set("consensus_note", cons_note) + .set("single_caller", count(single)) + .set("exon_edge", count(edge)) + .set("exon_edge_checked", Value::Bool(edge_checked)) + .set("common", count(common)); + let _ = cells; + Ok(Some((p, data.val()))) +} diff --git a/rust/crates/og-summary/src/collect/group_c/tables.rs b/rust/crates/og-summary/src/collect/group_c/tables.rs new file mode 100644 index 000000000..7151ee4dd --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/tables.rs @@ -0,0 +1,1536 @@ +//! The sections that read one step table each: the couple screen, the +//! polygenic scores, fine ancestry, telomeres, the haplogroups, the Y-STR +//! panel, the raw-data export, blood groups, archaic and ancient ancestry, +//! Lp(a), VNtyper, Locityper, viral DNA, CHIP, Exomiser, mobile elements +//! and MoChA. + +use super::super::model::State; +use super::util::{ + Obj, Table, at, base, count, dots, dots_strict, get_default, isdigit, kv_dot, kv_get, + metric_map, opt_s, or, or_dot, ragged, round, s, strs, table, +}; +use super::{Ctx, Found, Others, Out}; +use crate::collect::sections::{num, py_error}; +use indexmap::IndexMap; +use pycompat::json::Value; + +fn unreadable(msg: String) -> Result { + Err(msg) +} + +/// Python's `sorted(need)` of a set, as its list repr. +fn sorted_repr(need: &[&str]) -> String { + let mut v: Vec<&str> = need.to_vec(); + v.sort_unstable(); + format!( + "[{}]", + v.iter() + .map(|k| crate::repr(k)) + .collect::>() + .join(", ") + ) +} + +fn float_err(x: &str) -> String { + py_error("ValueError", &format!("could not convert string to float: {}", crate::repr(x))) +} + +fn int_of(x: &str) -> Result { + crate::int(x).map_err(|e| py_error("ValueError", &e)) +} + +fn opt_num(x: Option<&str>) -> Value { + num(x).map_or(Value::Null, Value::Float) +} + +/// `{k: r[k] or "."}` over the header's own first line, read as text. +fn first_line(path: &str) -> Result, String> { + let l = super::util::lines(path)?; + let h = l.first().map_or("", String::as_str); + Ok(h.trim_end_matches('\r').split('\t').map(str::to_owned).collect()) +} + +/// Base64 of a file's bytes (`base64.b64encode`). +fn b64(bytes: &[u8]) -> String { + const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for ch in bytes.chunks(3) { + let b = [ch[0], *ch.get(1).unwrap_or(&0), *ch.get(2).unwrap_or(&0)]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for i in 0..4 { + if i <= ch.len() { + out.push(T[((n >> (18 - 6 * i)) & 63) as usize] as char); + } else { + out.push('='); + } + } + } + out +} + +/// The base64 of a picture under `limit` bytes, else `null`. +fn picture(f: Option, limit: u64) -> Result { + let Some(f) = f else { return Ok(Value::Null) }; + let size = std::fs::metadata(&f.path) + .map_err(|e| super::util::os_note(&crate::os_error(&e, &f.path)))? + .len(); + if size >= limit { + return Ok(Value::Null); + } + let bytes = + std::fs::read(&f.path).map_err(|e| super::util::os_note(&crate::os_error(&e, &f.path)))?; + Ok(s(&b64(&bytes))) +} + +// --- step 39 -------------------------------------------------------------------- + +pub fn couple(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("couple/{}_couple_carrier.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, true)?; + let need = [ + "gene", + "condition", + "result", + "offspring_risk", + "partner_a", + "findings_a", + "partner_b", + "findings_b", + "explanation", + ]; + if need.iter().any(|k| !t.has(k)) { + return unreadable(format!( + "{} lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + let sample = c.sample; + let cell = |r: &pycompat::csv::DictRow, k: &str| r.get(k).flatten().unwrap_or("").to_owned(); + for (i, r) in t.rows.iter().enumerate() { + let bad = ragged(r, &need) + || (cell(r, "partner_a") != sample && cell(r, "partner_b") != sample) + || !matches!( + cell(r, "result").as_str(), + "shared" | "x_linked" | "one_partner" + ); + if bad { + return unreadable(format!( + "{} line {} is not a row of {sample}'s couple table", + base(&p.path), + i + 2 + )); + } + } + let side = |r: &pycompat::csv::DictRow| { + let (own, other) = if cell(r, "partner_a") == sample { + ("a", "b") + } else { + ("b", "a") + }; + Obj::new() + .set("gene", s(&cell(r, "gene"))) + .set("condition", s(&cell(r, "condition"))) + .set("risk", s(&cell(r, "offspring_risk"))) + .set("own", s(&cell(r, &format!("findings_{own}")))) + .set("partner_findings", s(&cell(r, &format!("findings_{other}")))) + .set("explanation", s(&cell(r, "explanation"))) + .val() + }; + let partner = t.rows.first().map(|r| { + if cell(r, "partner_a") == sample { + cell(r, "partner_b") + } else { + cell(r, "partner_a") + } + }); + let of = |kind: &str| { + Value::Array( + t.rows + .iter() + .filter(|r| cell(r, "result") == kind) + .map(side) + .collect(), + ) + }; + let v = Obj::new() + .set("partner", opt_s(partner.as_deref())) + .set("shared", of("shared")) + .set("x_linked", of("x_linked")) + .set( + "one_partner", + count(t.rows.iter().filter(|r| cell(r, "result") == "one_partner").count()), + ) + .val(); + Ok(Some((p, v))) +} + +// --- step 25 -------------------------------------------------------------------- + +pub fn prs(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("prs/{s_}_prs_summary.tsv")) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let mut rows = Vec::new(); + let mut adjusted = false; + for r in &t.rows { + let pct = or(r, "Percentile", ""); + let pct = if pct == "NA" { String::new() } else { pct }; + adjusted |= !pct.is_empty(); + let group = match r.get("Ancestry_Group") { + None => s(""), + Some(Some("NA")) => s(""), + Some(v) => opt_s(v), + }; + let evaluated = match r.get("Evaluated_In") { + Some(Some(x)) => strs(x.split(';').filter(|x| !x.is_empty())), + _ => Value::Null, + }; + rows.push( + Obj::new() + .set("condition", get_default(r, "Condition", "")) + .set("pgs_id", get_default(r, "PGS_ID", "")) + .set("score", get_default(r, "Score_SUM", "")) + .set("matched", get_default(r, "Variants_Matched", "")) + .set("total", get_default(r, "Variants_Total", "")) + .set("chrx", get_default(r, "ChrX", "")) + .set("ambiguous", get_default(r, "Ambiguous", "")) + .set("applies_to", get_default(r, "Applies_To", "")) + .set("sex_check", get_default(r, "Sex_Check", "")) + .set("percentile", s(&pct)) + .set("group", group) + .set("evaluated_in", evaluated) + .set("effect", s(&or(r, "Effect", ""))) + .val(), + ); + } + let mut out = Obj::new() + .set("scores", Value::Array(rows)) + .set("adjusted", Value::Bool(adjusted)); + if let Some(a) = c.find(&[ + format!("ancestry/{s_}_ancestry.tsv"), + format!("prs/{s_}_ancestry.tsv"), + ]) { + let mut kv: IndexMap = IndexMap::new(); + for r in table(&a.path, false)?.rows { + if let Some(Some(k)) = r.get("key") + && !k.is_empty() + { + kv.insert(k.to_owned(), get_default(&r, "value", "")); + } + } + let g = |k: &str| kv.get(k).cloned().unwrap_or_else(|| s("")); + out.put( + "ancestry", + Obj::new() + .set("population", g("population")) + .set("panel", g("reference_panel")) + .set("low_confidence", g("population_low_confidence")) + .val(), + ); + } + Ok(Some((p, out.val()))) +} + +// --- step 26b ------------------------------------------------------------------- + +pub fn ancestry_fine(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("ancestry_fine/{s_}_ancestry_fine.tsv")) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let need = ["level", "code", "group", "fraction", "se", "low", "high"]; + if !t.first_has(&need) { + return unreadable(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + sorted_repr(&need) + )); + } + let levels = ["continent", "region", "painting", "left_out", "not_painted"]; + let mut by: IndexMap<&str, Vec> = levels.iter().map(|l| (*l, Vec::new())).collect(); + for r in &t.rows { + let Some(Some(level)) = r.get("level") else { + continue; + }; + if let Some(list) = by.get_mut(level) { + list.push( + Obj::new() + .set("code", opt_s(at(r, "code")?)) + .set("group", opt_s(at(r, "group")?)) + .set("fraction", opt_num(at(r, "fraction")?)) + .set("low", opt_num(at(r, "low")?)) + .set("high", opt_num(at(r, "high")?)) + .val(), + ); + } + } + let mut out = Obj::new(); + for (k, v) in by { + out.put(k, Value::Array(v)); + } + out.put("min_references", count(og_ancestry::MIN_REFERENCES)); + let png = c.find1(format!("ancestry_fine/{s_}_painting.png")); + out.put("painting_png", Value::Null); + out.put( + "legend", + Value::Array( + og_ancestry::COLOURS + .iter() + .map(|(code, rgb)| { + Obj::new() + .set("code", s(code)) + .set( + "group", + s(og_ancestry::label(og_ancestry::Level::Continent, code)), + ) + .set( + "colour", + s(&format!("#{:02x}{:02x}{:02x}", rgb[0], rgb[1], rgb[2])), + ) + .val() + }) + .collect(), + ), + ); + out.put("painting_png", picture(png, 2_000_000)?); + Ok(Some((p, out.val()))) +} + +// --- step 44 -------------------------------------------------------------------- + +pub fn ancient(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("ancient/{s_}_ancient.tsv")) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let need = ["rank", "group", "n", "distance", "dates", "country", "locality"]; + if !t.first_has(&need) { + return unreadable(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + sorted_repr(&need) + )); + } + let info = match c.find1(format!("ancient/{s_}_ancient_info.tsv")) { + Some(ip) => metric_map(&table(&ip.path, false)?, "key", "value"), + None => IndexMap::new(), + }; + let nearest = t + .rows + .iter() + .map(|r| { + need.iter() + .fold(Obj::new(), |o, k| o.set(k, opt_s(r.get(k).flatten()))) + .val() + }) + .collect(); + let mut out = Obj::new().set("nearest", Value::Array(nearest)); + for k in [ + "release", + "sites_used", + "panel_sites", + "ancient_people", + "ancient_groups", + "present_day_people", + ] { + out.put(k, s(&kv_dot(&info, k))); + } + let svg = c.find1(format!("ancient/{s_}_ancient_pca.svg")); + out.put("plot_svg", picture(svg, 3_000_000)?); + Ok(Some((p, out.val()))) +} + +// --- step 10 -------------------------------------------------------------------- + +const GENOME_BASES: f64 = 3.1e9; +const TELOMERE_FLOOR_PER_30X: f64 = 2000.0; + +pub fn telomere(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find(&[ + format!("telomere/{s_}/{s_}/{s_}_summary.tsv"), + format!("telomere/{s_}/{s_}_summary.tsv"), + "telomere/**/*_summary.tsv".to_owned(), + ]) else { + return Ok(None); + }; + let lines: Vec> = super::util::lines(&p.path)? + .into_iter() + .filter(|l| !pycompat::num::py_strip(l).is_empty()) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + if lines.len() < 2 { + return Ok(Some((p, Obj::new().set("tel_content", s(".")).val()))); + } + let (head, row) = (&lines[0], &lines[1]); + let i = head.iter().position(|h| h == "tel_content").unwrap_or(10); + let mut out = Obj::new().set("tel_content", s(row.get(i).map_or(".", String::as_str))); + let col = |names: &[&str]| -> Option<&str> { + names.iter().find_map(|n| { + head.iter() + .position(|h| h == n) + .and_then(|i| row.get(i)) + .map(String::as_str) + }) + }; + let intratel = num(col(&["intratel_reads", "intratelomeric_reads"])); + let total = num(col(&["total_reads"])); + let lengths: Vec = col(&["read_lengths", "read_length"]) + .unwrap_or("") + .split([',', ';', '|', ' ']) + .filter_map(|x| num(Some(x))) + .filter(|x| *x != 0.0) + .collect(); + let Some(intratel) = intratel else { + out.put( + "read_check", + s("no intratelomeric read count in the summary"), + ); + return Ok(Some((p, out.val()))); + }; + out.put( + "intratelomeric_reads", + Value::Int(crate::trunc(intratel).map_err(|e| py_error("OverflowError", &e))?), + ); + let total = total.filter(|t| *t != 0.0); + let (Some(total), false) = (total, lengths.is_empty()) else { + out.put( + "read_check", + s("depth unknown (no total_reads or read_lengths in the summary): not checked"), + ); + return Ok(Some((p, out.val()))); + }; + let longest = lengths.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let depth = total * longest / GENOME_BASES; + let floor = TELOMERE_FLOOR_PER_30X * depth / 30.0; + out.put("depth", round(depth, 1)); + let fl = pycompat::fmt::round_int(floor) + .map_err(|e| py_error(e.exception(), "cannot round")) + .and_then(|x| crate::trunc(x).map_err(|e| py_error("OverflowError", &e)))?; + out.put("read_floor", Value::Int(fl)); + out.put("low_reads", Value::Bool(intratel < floor)); + Ok(Some((p, out.val()))) +} + +// --- steps 12 and 37 ------------------------------------------------------------ + +const EXPLORE_DOCS: &str = "https://github.com/GeiserX/OwnGenome/blob/main/docs/"; + +/// `urllib.parse.quote(x, safe="")`. +fn quote(x: &str) -> String { + let mut out = String::new(); + for b in x.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-' | b'~') { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +fn explore(kind: &str, hg: &str) -> Value { + let mut hg = pycompat::num::py_strip(hg); + if kind == "y" { + hg = hg.split('*').next().unwrap_or(""); + } + if matches!(hg, "" | "." | "NA" | "insufficient markers") { + return Value::Array(Vec::new()); + } + let q = quote(hg); + let link = |name: &str, url: String| Obj::new().set("name", s(name)).set("url", s(&url)).val(); + let (ftdna, yfull, doc) = if kind == "y" { + ( + format!("https://discover.familytreedna.com/y-dna/{q}"), + ("YFull YTree", format!("https://www.yfull.com/tree/{q}/")), + "37-y-haplogroup.md#explore-your-branch-further", + ) + } else { + ( + format!("https://discover.familytreedna.com/mtdna/{q}"), + ("YFull MTree", format!("https://www.yfull.com/mtree/{q}/")), + "12-mito-haplogroup.md#explore-your-branch-further", + ) + }; + Value::Array(vec![ + link("FamilyTreeDNA Discover (free, no upload)", ftdna), + link(yfull.0, yfull.1), + link( + "More routes, and the files to share", + format!("{EXPLORE_DOCS}{doc}"), + ), + ]) +} + +pub fn haplogroup(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("mito/{s_}_haplogroup.txt")) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let hg = match t.rows.first() { + None => String::new(), + Some(r) => { + let named = r.get("Haplogroup").flatten().filter(|x| !x.is_empty()); + let second = || { + r.iter() + .nth(1) + .and_then(|(_, v)| v) + .filter(|x| !x.is_empty()) + }; + named.or_else(second).unwrap_or("").trim_matches('"').to_owned() + } + }; + let mut out = Obj::new() + .set("haplogroup", s(if hg.is_empty() { "." } else { &hg })) + .set("explore", explore("mt", &hg)); + if let Some(hc) = c.find1(format!("mito/{s_}_haplocheck.txt")) { + let t = table(&hc.path, false)?; + let first: Option> = t.rows.first().map(|r| { + r.iter() + .filter(|(k, _)| !k.is_empty()) + .map(|(k, v)| { + ( + k.trim_matches('"').to_owned(), + v.unwrap_or("").trim_matches('"').to_owned(), + ) + }) + .collect() + }); + let Some(first) = first.filter(|m| m.contains_key("Contamination Status")) else { + return unreadable(format!( + "{} has no Contamination Status column", + base(&hc.path) + )); + }; + let dot = |k: &str| { + first + .get(k) + .filter(|v| !v.is_empty()) + .map_or(".", String::as_str) + .to_owned() + }; + out.put("contamination_status", s(&dot("Contamination Status"))); + out.put("contamination_level", s(&dot("Contamination Level"))); + } + Ok(Some((p, out.val()))) +} + +pub fn y_haplogroup(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("y_haplogroup/{}_y_haplogroup.txt", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let Some(r) = t.rows.first().filter(|_| t.has("Hg")) else { + return unreadable(format!( + "{} has no Hg column or no sample row", + base(&p.path) + )); + }; + let hg = pycompat::num::py_strip(r.get("Hg").flatten().unwrap_or("")).to_owned(); + let v = Obj::new() + .set( + "haplogroup", + s(if hg.is_empty() || hg == "NA" { + "insufficient markers" + } else { + &hg + }), + ) + .set("valid_markers", s(&or_dot(r, "Valid_markers"))) + .set("qc_score", s(&or_dot(r, "QC-score"))) + .set("explore", explore("y", &hg)) + .val(); + Ok(Some((p, v))) +} + +pub fn y_tree(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("y_haplogroup/{s_}_y_tree.txt")) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let need = ["Hg", "Level", "Tree"]; + let Some(r) = t.rows.first().filter(|_| need.iter().all(|k| t.has(k))) else { + return unreadable(format!( + "{} lacks a column of {}", + base(&p.path), + need.join(", ") + )); + }; + let hg = pycompat::num::py_strip(r.get("Hg").flatten().unwrap_or("")).to_owned(); + let mut out = Obj::new().set( + "haplogroup", + s(if hg.is_empty() || hg == "NA" { + "no branch has a derived call" + } else { + &hg + }), + ); + for (k, col) in [ + ("level", "Level"), + ("tree", "Tree"), + ("path_derived", "Path_derived"), + ("path_ancestral", "Path_ancestral"), + ("path_no_call", "Path_no_call"), + ("ancestral_in_terminal", "Ancestral_in_terminal"), + ("outvoted_ancestral", "Outvoted_ancestral"), + ("snps_called", "SNPs_called"), + ("min_depth", "Min_depth"), + ] { + out.put(k, s(&or_dot(r, col))); + } + out.put("explore", explore("y", &hg)); + if let Some(lvl) = c.find1(format!("y_haplogroup/{s_}_y_tree_path.tsv")) { + let pt = table(&lvl.path, false)?; + if !pt.has("Branch") { + return unreadable(format!("{} has no Branch column", base(&lvl.path))); + } + let keys = [ + "Level", + "Branch", + "SNPs", + "On_tables", + "Derived", + "Ancestral", + "No_call", + "Ancestral_SNPs", + ]; + out.put( + "path", + Value::Array( + pt.rows + .iter() + .map(|row| { + keys.iter() + .fold(Obj::new(), |o, k| o.set(k, s(&or(row, k, "")))) + .val() + }) + .collect(), + ), + ); + } + Ok(Some((p, out.val()))) +} + +// --- step 45 -------------------------------------------------------------------- + +pub fn y_str(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("y_str/{}_y_str.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let need = ["order", "panel", "marker", "count", "confidence"]; + if !t.first_has(&need) { + return unreadable(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + let keep: Vec> = t + .rows + .iter() + .map(|r| need.iter().map(|k| or(r, k, "")).collect()) + .collect(); + let kinds = ["high", "low", "no call", "not callable"]; + let counts: Vec = kinds + .iter() + .map(|c| keep.iter().filter(|r| r[4] == *c).count()) + .collect(); + if counts.iter().sum::() != keep.len() { + return unreadable(format!( + "{} has a confidence other than high, low, no call or not callable", + base(&p.path) + )); + } + let mut out = Obj::new().set( + "markers", + Value::Array( + keep.iter() + .map(|r| { + need.iter() + .zip(r) + .fold(Obj::new(), |o, (k, v)| o.set(k, s(v))) + .val() + }) + .collect(), + ), + ); + for (k, n) in kinds.iter().zip(counts) { + out.put(&k.replace(' ', "_"), count(n)); + } + Ok(Some((p, out.val()))) +} + +// --- step 40 -------------------------------------------------------------------- + +pub fn consumer_export(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("consumer_export/{}_consumer_export.tsv", c.s())) else { + return Ok(None); + }; + let mut data: IndexMap> = IndexMap::new(); + for r in table(&p.path, false)?.rows { + if let Some(Some(k)) = r.get("key") + && !k.is_empty() + { + let v = match r.get("value") { + None => Some(String::new()), + Some(v) => v.map(str::to_owned), + }; + data.insert(k.to_owned(), v); + } + } + let text = |k: &str| data.get(k).cloned().flatten().unwrap_or_default(); + for k in ["sites", "called", "no_call"] { + if !isdigit(&text(k)) { + return unreadable(format!( + "{} has no whole-number '{k}' row", + base(&p.path) + )); + } + } + let mut reasons = Obj::new(); + for (k, v) in &data { + if let Some(why) = k.strip_prefix("no_call: ") { + let v = v.clone().unwrap_or_default(); + if !isdigit(&v) { + return unreadable(format!( + "{} has a '{k}' row without a whole-number count", + base(&p.path) + )); + } + reasons.put(why, Value::Int(int_of(&v)?)); + } + } + let blocks = text("from_reference_blocks"); + let blocks = if blocks.is_empty() { 0 } else { int_of(&blocks)? }; + let dot = |k: &str| { + let v = text(k); + s(if v.is_empty() { "." } else { &v }) + }; + let v = Obj::new() + .set("sites", Value::Int(int_of(&text("sites"))?)) + .set("called", Value::Int(int_of(&text("called"))?)) + .set("no_call", Value::Int(int_of(&text("no_call"))?)) + .set("from_reference_blocks", Value::Int(blocks)) + .set("template_sites", dot("template_sites")) + .set("file_23andme", dot("file_23andme")) + .set("file_ancestrydna", dot("file_ancestrydna")) + .set("no_call_reasons", reasons.val()) + .val(); + Ok(Some((p, v))) +} + +// --- step 41 -------------------------------------------------------------------- + +pub fn bloodtype(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("bloodtype/{}_bloodtype.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + if !t.first_has(&["system", "phenotype", "confidence"]) { + return unreadable(format!( + "{} has no system, phenotype and confidence columns, or no row", + base(&p.path) + )); + } + let mut by: IndexMap, String> = IndexMap::new(); + let mut systems = Vec::new(); + for r in &t.rows { + let system = r.get("system").flatten(); + let pheno = or_dot(r, "phenotype"); + by.insert(system.map(str::to_owned), pheno.clone()); + systems.push( + Obj::new() + .set("system", opt_s(system)) + .set("phenotype", s(&pheno)) + .set("genotype", s(&or_dot(r, "genotype"))) + .set("confidence", s(&or_dot(r, "confidence"))) + .val(), + ); + } + let get = |k: &str| s(by.get(&Some(k.to_owned())).map_or(".", String::as_str)); + let v = Obj::new() + .set("systems", Value::Array(systems)) + .set("abo", get("ABO")) + .set("rhd", get("RhD")) + .val(); + Ok(Some((p, v))) +} + +// --- step 43 -------------------------------------------------------------------- + +fn all_digits(x: &str) -> bool { + !x.is_empty() && x.bytes().all(|b| b.is_ascii_digit()) +} + +pub fn archaic(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("archaic/{}_archaic.tsv", c.s())) else { + return Ok(None); + }; + let mut data: IndexMap = IndexMap::new(); + for r in table(&p.path, false)?.rows { + if let Some(Some(k)) = r.get("key") + && !k.is_empty() + { + data.insert(k.to_owned(), or(&r, "value", "")); + } + } + let mut longest: Vec<(i128, &String)> = data + .iter() + .filter_map(|(k, v)| { + let n = k.strip_prefix("longest_").filter(|n| all_digits(n))?; + Some((crate::int(n).ok()?, v)) + }) + .collect(); + longest.sort_by_key(|x| x.0); + let mut out = Obj::new() + .set( + "population", + s(data.get("population").map_or(".", String::as_str)), + ) + .set("longest", strs(longest.iter().map(|x| x.1.as_str()))); + for k in [ + "neanderthal_segments", + "neanderthal_bp", + "denisovan_segments", + "denisovan_bp", + "analysed_bp", + ] { + let v = data.get(k).map_or("", String::as_str); + if !all_digits(v) { + return unreadable(format!( + "{} has no whole number for {k}", + base(&p.path) + )); + } + out.put(k, Value::Int(int_of(v)?)); + } + for k in ["neanderthal_percent", "denisovan_percent"] { + let v = data.get(k).map_or("", String::as_str); + let ok = v + .split_once('.') + .is_some_and(|(a, b)| all_digits(a) && all_digits(b)); + if !ok { + return unreadable(format!( + "{} has no percentage for {k}", + base(&p.path) + )); + } + out.put(k, s(v)); + } + Ok(Some((p, out.val()))) +} + +// --- step 46 -------------------------------------------------------------------- + +const LPA_RISK_ALLELES: [(&str, &str); 2] = [("rs10455872", "G"), ("rs3798220", "C")]; + +pub fn lpa(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("lpa/{s_}_lpa_kiv2.tsv")) else { + return Ok(None); + }; + let kv = metric_map(&table(&p.path, false)?, "metric", "value"); + if !matches!( + kv_get(&kv, "status"), + Some("estimated" | "too little depth" | "not assessed") + ) { + return unreadable(format!( + "{} has no status row (estimated, too little depth or not assessed)", + base(&p.path) + )); + } + let mut data = Obj::new(); + for k in [ + "status", + "kiv2_copies", + "ci95_low", + "ci95_high", + "class", + "kiv2_depth", + "flank_depth", + "mapq0_share", + "message", + "advice", + ] { + data.put(k, s(&kv_dot(&kv, k))); + } + let dir = p.path.rsplit_once('/').map_or("", |x| x.0); + data.put( + "null_allele", + super::cn::read_marker_table( + &crate::join(dir, &format!("{}_lpa_kiv2_4925.tsv", c.sample)), + "46", + "LPA_KIV2_4925", + ), + ); + let mut variants = Vec::new(); + let mut risk = 0i128; + let mut any_read = false; + if let Some(t) = c.find1(format!("traits/{s_}_traits.tsv")) { + for r in table(&t.path, false)?.rows { + let rsid = r.get("rsid").flatten(); + let Some(alt) = LPA_RISK_ALLELES + .iter() + .find(|(id, _)| Some(*id) == rsid) + .map(|x| x.1) + else { + continue; + }; + let called = r.get("status").flatten() == Some("called"); + let gt = r.get("genotype").flatten().unwrap_or(""); + let n = called.then(|| gt.split(['/', '|']).filter(|a| *a == alt).count() as i128); + if let Some(n) = n { + risk += n; + any_read = true; + } + variants.push( + Obj::new() + .set("rsid", opt_s(rsid)) + .set("genotype", s(&or_dot(&r, "genotype"))) + .set("status", s(&or_dot(&r, "status"))) + .set("risk_alleles", n.map_or(Value::Null, Value::Int)) + .val(), + ); + } + } + let class = kv_dot(&kv, "class"); + let mut reasons = Vec::new(); + if risk != 0 { + reasons.push(format!( + "{risk} risk allele{} at rs10455872 and rs3798220", + if risk != 1 { "s" } else { "" } + )); + } + if class == "low" { + reasons.push("a low KIV-2 repeat count".to_owned()); + } + let read = kv_dot(&kv, "status") == "estimated" || any_read; + let flag_text = if !reasons.is_empty() { + format!( + "Genetic signs of a high Lp(a): {}. Measure Lp(a) in blood.", + reasons.join(" and ") + ) + } else if read { + "No genetic sign of a high Lp(a) from these markers. They explain only part of it: the KIV-2 count explained 21% to 27% of the variation in a Danish population study.".to_owned() + } else { + "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood.".to_owned() + }; + data.put("variants", Value::Array(variants)); + data.put("risk_alleles", Value::Int(risk)); + data.put("flag", s(if reasons.is_empty() { "." } else { "high" })); + data.put("flag_text", s(&flag_text)); + Ok(Some((p, data.val()))) +} + +// --- step 52 -------------------------------------------------------------------- + +const VNTYPER_STATUSES: [&str; 4] = ["positive", "possible", "negative", "not assessed"]; + +pub fn vntyper(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("vntyper/{}_vntyper.tsv", c.s())) else { + return Ok(None); + }; + let kv = metric_map(&table(&p.path, false)?, "metric", "value"); + if !kv_get(&kv, "status").is_some_and(|st| VNTYPER_STATUSES.contains(&st)) { + return unreadable(format!( + "{} has no status row ({})", + base(&p.path), + VNTYPER_STATUSES.join(", ") + )); + } + let v = [ + "status", + "confidence", + "variant", + "flag", + "alt_depth", + "vntr_mean_depth", + "coverage_qc", + "sensitivity", + "message", + "advice", + ] + .iter() + .fold(Obj::new(), |o, k| o.set(k, s(&kv_dot(&kv, k)))) + .val(); + Ok(Some((p, v))) +} + +// --- step 53 -------------------------------------------------------------------- + +const LOCITYPER_COLUMNS: [&str; 13] = [ + "locus", + "region", + "status", + "genotype", + "haplotype_1", + "haplotype_2", + "quality", + "total_reads", + "unexplained_reads", + "weighted_distance", + "warnings", + "filter", + "reason", +]; + +pub fn locityper(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("locityper/{}_locityper.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + if !t.first_has(&LOCITYPER_COLUMNS) { + return unreadable(format!( + "{} lacks a row or one of the columns {}", + base(&p.path), + LOCITYPER_COLUMNS.join(", ") + )); + } + let loci: Vec = t.rows.iter().map(|r| dots(r, &LOCITYPER_COLUMNS)).collect(); + let n = |k: &str, want: &str| { + t.rows + .iter() + .filter(|r| or_dot(r, k) == want) + .count() + }; + let v = Obj::new() + .set("called", count(n("status", "called"))) + .set("passed", count(n("filter", "pass"))) + .val(); + let mut out = Obj::new().set("loci", Value::Array(loci)); + if let Value::Object(m) = v { + out.0.extend(m); + } + Ok(Some((p, out.val()))) +} + +fn text_at<'a>(v: &'a Value, k: &str) -> Option<&'a Value> { + match v { + Value::Object(m) => m.get(k), + _ => None, + } +} + +/// Each Locityper locus's primary call (`locityper_primary`), as text. +pub fn locityper_post(v: &mut Value, hla_state: State, hla: Option<&Value>, others: &Others) { + let old = |st: State| { + if st == State::Stale { + " (stale: from an earlier run)" + } else { + "" + } + }; + let hla = hla.filter(|_| matches!(hla_state, State::Ok | State::Stale)); + let mut by_gene: IndexMap = IndexMap::new(); + if let Some(Value::Array(loci)) = hla.and_then(|h| text_at(h, "loci")) { + for x in loci { + if let Some(Value::Str(g)) = text_at(x, "gene") { + by_gene.insert(g.clone(), x); + } + } + } + let mut primary: IndexMap<&str, String> = IndexMap::new(); + for g in ["HLA-DRB1", "HLA-DQB1"] { + let x = by_gene + .get(g) + .or_else(|| by_gene.get(&g.replace("HLA-", ""))); + primary.insert( + g, + match x { + None => "T1K (step 08) did not run".to_owned(), + Some(x) => { + let alleles = match text_at(x, "alleles") { + Some(Value::Array(a)) => a + .iter() + .map(|a| pycompat::value::py_str(a).unwrap_or_default()) + .collect::>() + .join("/"), + _ => String::new(), + }; + let low = text_at(x, "low_confidence").is_some_and(pycompat::value::truthy); + format!( + "T1K (step 08): {}{}{}", + if alleles.is_empty() { + "no allele" + } else { + &alleles + }, + if low { " (low confidence)" } else { "" }, + old(hla_state) + ) + } + }, + ); + } + let typed = hla + .and_then(|h| text_at(h, "kir")) + .and_then(|k| text_at(k, "typed")) + .filter(|t| pycompat::value::truthy(t)); + primary.insert( + "KIR", + match typed { + Some(Value::Array(t)) => format!( + "T1K KIR pass (step 08): {} genes typed{}", + t.len(), + old(hla_state) + ), + Some(Value::Object(t)) => format!( + "T1K KIR pass (step 08): {} genes typed{}", + t.len(), + old(hla_state) + ), + _ => "T1K's KIR pass (step 08, KIR=true) gave no call".to_owned(), + }, + ); + let mut cyp = Vec::new(); + if let Some((st, call)) = &others.pypgx { + cyp.push(format!("pypgx (step 32): {call}{}", old(*st))); + } + if let Some((st, genotype, filter)) = &others.cyrius { + let usable = if filter.as_deref() == Some("PASS") { + String::new() + } else { + format!( + " (not usable: Filter {})", + filter.as_deref().filter(|f| !f.is_empty()).unwrap_or("missing") + ) + }; + cyp.push(format!( + "Cyrius (step 21): {}{usable}{}", + genotype.as_deref().unwrap_or("none"), + old(*st) + )); + } + primary.insert( + "CYP2D6", + if cyp.is_empty() { + "pypgx (step 32) and Cyrius (step 21) did not run".to_owned() + } else { + cyp.join("; ") + }, + ); + primary.insert("FCGR", "no other step calls FCGR".to_owned()); + let Value::Object(m) = v else { return }; + let Some(Value::Array(loci)) = m.get_mut("loci") else { + return; + }; + for x in loci { + let Value::Object(x) = x else { continue }; + let locus = match x.get("locus") { + Some(Value::Str(l)) => l.clone(), + _ => String::new(), + }; + let p = primary + .get(locus.as_str()) + .cloned() + .unwrap_or_else(|| "no other step calls this locus".to_owned()); + x.insert("primary".into(), s(&p)); + } +} + +// --- step 47 -------------------------------------------------------------------- + +pub fn viral(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("viral/{}_viral.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, false)?; + let need = [ + "virus", + "accession", + "reads", + "breadth", + "relative_depth", + "copies_per_cell", + "call", + ]; + if !t.first_has(&need) { + return unreadable(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + let all: Vec<&str> = need.iter().copied().chain(["human_depth", "note"]).collect(); + let keep = t + .rows + .iter() + .map(|r| dots_strict(r, &all)) + .collect::, _>>()?; + let get = |v: &Value, k: &str| match text_at(v, k) { + Some(Value::Str(x)) => x.clone(), + _ => String::new(), + }; + let ici = keep + .iter() + .find(|r| get(r, "call") == "inherited integrated HHV-6 likely") + .map(|r| get(r, "virus")); + let human = get(&keep[0], "human_depth"); + let hhv6 = match ici { + Some(v) => format!("inherited integrated {v} likely"), + None if human == "." => "not assessed (no human depth)".to_owned(), + None => "no integrated HHV-6".to_owned(), + }; + let found: Vec = keep + .iter() + .filter(|r| { + let n = get(r, "reads"); + isdigit(&n) && crate::int(&n).is_ok_and(|n| n > 0) + }) + .cloned() + .collect(); + let v = Obj::new() + .set("hhv6", s(&hhv6)) + .set("human_depth", s(&human)) + .set("found", Value::Array(found)) + .set("genomes", count(keep.len())) + .val(); + Ok(Some((p, v))) +} + +// --- step 49 -------------------------------------------------------------------- + +pub fn chip(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let (Some(p), Some(dp)) = ( + c.find1(format!("chip/{s_}_chip.tsv")), + c.find1(format!("chip/{s_}_chip_depth.tsv")), + ) else { + return Ok(None); + }; + let calls = table(&p.path, false)?; + let depth = table(&dp.path, false)?; + let need = [ + "gene", + "protein_change", + "vaf", + "alt_reads", + "depth", + "status", + "note", + ]; + let head = first_line(&p.path)?; + if need.iter().any(|k| !head.iter().any(|h| h == k)) { + return unreadable(format!( + "{} lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + let dneed = ["gene", "mean_depth", "detection_limit_vaf", "status"]; + if !depth.first_has(&dneed) { + return unreadable(format!( + "{} has no gene, mean_depth, detection_limit_vaf and status columns, or no row", + base(&dp.path) + )); + } + let chip: Vec = calls + .rows + .iter() + .filter(|r| r.get("status").flatten() == Some("chip")) + .map(|r| dots(r, &need)) + .collect(); + let mut covered = Vec::new(); + for r in &depth.rows { + if r.get("status").flatten() == Some("covered") { + let x = r.get("mean_depth").flatten().ok_or_else(|| { + py_error( + "TypeError", + "float() argument must be a string or a real number, not 'NoneType'", + ) + })?; + covered.push(pycompat::num::parse_float(x).map_err(|_| float_err(x))?); + } + } + covered.sort_by(f64::total_cmp); + let med = match covered.len() { + 0 => None, + n if n % 2 == 1 => Some(covered[n / 2]), + n => Some((covered[n / 2 - 1] + covered[n / 2]) / 2.0), + }; + let limit = match med { + None => Value::Null, + Some(m) => og_chip::binom::detection_limit(m) + .map_err(|e| py_error("ValueError", &e.to_string()))? + .map_or(Value::Null, Value::Float), + }; + let v = Obj::new() + .set("calls", Value::Array(chip.clone())) + .set("other", count(calls.rows.len() - chip.len())) + .set( + "genes", + Value::Array(depth.rows.iter().map(|r| dots(r, &dneed)).collect()), + ) + .set("median_depth", med.map_or(Value::Null, Value::Float)) + .set("median_limit", limit) + .set( + "not_covered", + Value::Array( + depth + .rows + .iter() + .filter(|r| r.get("status").flatten() != Some("covered")) + .map(|r| opt_s(r.get("gene").flatten())) + .collect(), + ), + ) + .val(); + Ok(Some((p, v))) +} + +// --- step 50 -------------------------------------------------------------------- + +pub fn mei(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("mei/{}_mei_sites.tsv", c.s())) else { + return Ok(None); + }; + let sites = table(&p.path, false)?; + let need = [ + "gene", + "element", + "condition", + "inheritance", + "status", + "mean_depth", + "call", + ]; + if !sites.first_has(&need) { + return unreadable(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + let dir = p.path.rsplit_once('/').map_or("", |x| x.0); + let cp = crate::join(dir, &format!("{}_mei.tsv", c.sample)); + if !crate::is_file(&cp) { + return unreadable(format!( + "{} is missing beside {}", + base(&cp), + base(&p.path) + )); + } + let calls = table(&cp, false)?; + if ["chrom", "pos", "element", "copies"] + .iter() + .any(|k| !calls.has(k)) + { + return unreadable(format!( + "{} lacks a column of chrom, pos, element, copies", + base(&cp) + )); + } + let mut counts: std::collections::BTreeMap = Default::default(); + for r in &calls.rows { + let e = r.get("element").flatten().unwrap_or("null").to_owned(); + *counts.entry(e).or_default() += 1; + } + let v = Obj::new() + .set("calls", count(calls.rows.len())) + .set( + "by_element", + Value::Object(counts.into_iter().map(|(k, n)| (k, count(n))).collect()), + ) + .set( + "sites", + Value::Array(sites.rows.iter().map(|r| dots(r, &need)).collect()), + ) + .val(); + Ok(Some((p, v))) +} + +// --- step 51 -------------------------------------------------------------------- + +const MOCHA_SUMMARY_KEYS: [&str; 12] = [ + "status", + "message", + "depth", + "het_sites", + "het_per_10mb", + "chromosomes", + "floor_read", + "floor_min", + "floor", + "calls_mocha", + "calls_reported", + "phasing", +]; +const MOCHA_CALL_KEYS: [&str; 11] = [ + "chrom", + "start", + "end", + "length_mb", + "arms", + "type", + "cell_fraction", + "n_hets", + "lod_baf_phase", + "reported", + "reason", +]; + +pub fn mosaic(c: &Ctx<'_>) -> Out { + let s_ = c.s(); + let Some(p) = c.find1(format!("mosaic/{s_}_mocha_summary.tsv")) else { + return Ok(None); + }; + let kv = metric_map(&table(&p.path, false)?, "metric", "value"); + if !matches!(kv_get(&kv, "status"), Some("assessed" | "not assessed")) { + return unreadable(format!( + "{} has no status row (assessed or not assessed)", + base(&p.path) + )); + } + let mut data = MOCHA_SUMMARY_KEYS + .iter() + .fold(Obj::new(), |o, k| o.set(k, s(&kv_dot(&kv, k)))); + let Some(cf) = c.find1(format!("mosaic/{s_}_mocha.tsv")) else { + return unreadable(format!( + "{} has no {}_mocha.tsv beside it", + base(&p.path), + c.sample + )); + }; + let head = first_line(&cf.path)?; + if MOCHA_CALL_KEYS.iter().any(|k| !head.iter().any(|h| h == k)) { + return unreadable(format!( + "{} lacks a column of {}", + base(&cf.path), + MOCHA_CALL_KEYS.join(", ") + )); + } + let rows: Table = table(&cf.path, false)?; + let reported = |r: &pycompat::csv::DictRow| r.get("reported").flatten() == Some("yes"); + let calls: Vec = rows + .rows + .iter() + .filter(|r| reported(r)) + .map(|r| dots(r, &MOCHA_CALL_KEYS)) + .collect(); + let n = calls.len(); + data.put("calls", Value::Array(calls)); + data.put( + "not_reported", + count(rows.rows.iter().filter(|r| !reported(r)).count()), + ); + let said = kv_dot(&kv, "calls_reported"); + if said != "." && said != n.to_string() { + return unreadable(format!( + "{} has {n} reported calls, {} says {said}", + base(&cf.path), + base(&p.path) + )); + } + Ok(Some((p, data.val()))) +} + +// --- step 48 -------------------------------------------------------------------- + +pub fn exomiser(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("exomiser/{}_exomiser_top.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, true)?; + let header = first_line(&p.path)?; + let need = [ + "rank", + "gene", + "moi", + "exomiser_score", + "phenotype_score", + "variant", + "genotype", + "consequence", + "hgvs", + "acmg", + "disease", + ]; + if need.iter().any(|k| !header.iter().any(|h| h == k)) { + return unreadable(format!( + "{} lacks a column of {}", + base(&p.path), + need.join(", ") + )); + } + if t.rows.iter().any(|r| ragged(r, &need)) { + return unreadable(format!( + "{} has a line that does not have one field per column", + base(&p.path) + )); + } + let dir = p.path.rsplit_once('/').map_or("", |x| x.0); + let tp = crate::join(dir, &format!("{}_exomiser_terms.tsv", c.sample)); + let mut terms = Vec::new(); + if crate::is_file(&tp) { + for r in table(&tp, true)?.rows { + terms.push( + Obj::new() + .set("id", get_default(&r, "hpo_id", "")) + .set("label", get_default(&r, "label", "")) + .val(), + ); + } + } + let cell = |r: &pycompat::csv::DictRow, k: &str| s(r.get(k).flatten().unwrap_or("")); + let mut genes: Vec<(String, Obj, Vec)> = Vec::new(); + for r in &t.rows { + let rank = r.get("rank").flatten().unwrap_or("").to_owned(); + if genes.last().is_none_or(|g| g.0 != rank) { + let o = ["rank", "gene", "moi", "exomiser_score", "phenotype_score"] + .iter() + .fold(Obj::new(), |o, k| o.set(k, cell(r, k))); + genes.push((rank, o, Vec::new())); + } + let v = ["variant", "genotype", "consequence", "hgvs", "acmg", "disease"] + .iter() + .fold(Obj::new(), |o, k| o.set(k, cell(r, k))) + .val(); + if let Some(g) = genes.last_mut() { + g.2.push(v); + } + } + let v = Obj::new() + .set("terms", Value::Array(terms)) + .set( + "genes", + Value::Array( + genes + .into_iter() + .map(|(_, o, vs)| o.set("variants", Value::Array(vs)).val()) + .collect(), + ), + ) + .val(); + Ok(Some((p, v))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn base64_matches_python() { + assert_eq!(b64(b""), ""); + assert_eq!(b64(b"f"), "Zg=="); + assert_eq!(b64(b"fo"), "Zm8="); + assert_eq!(b64(b"foo"), "Zm9v"); + assert_eq!(b64(b"foob"), "Zm9vYg=="); + } + + #[test] + fn quote_keeps_unreserved_only() { + assert_eq!(quote("R-M269"), "R-M269"); + assert_eq!(quote("H5a7 x/y"), "H5a7%20x%2Fy"); + assert_eq!(quote("é"), "%C3%A9"); + } +} diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs new file mode 100644 index 000000000..e69de29bb diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs new file mode 100644 index 000000000..56c4c856d --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -0,0 +1,570 @@ +//! Step 42's traits section: the traits table, its genotype-rules lines, +//! and the APOL1 G2 deletion read from the sample's VCF beside the APOL1 +//! rule, whose single-base sites read G1 only. + +use super::util::{Obj, base, count, ragged, s, table, vcf_lines}; +use super::{Ctx, Out}; +use pycompat::json::Value; + +/// The columns of step 42's genotype-rules table. +pub const RULE_OUT_COLUMNS: [&str; 15] = [ + "rule", + "gene", + "trait", + "rsids", + "chrom", + "pos", + "genotypes", + "gq", + "phase", + "result", + "status", + "reading", + "ancestry", + "note", + "pmids", +]; +/// The rules whose line the traits table holds already. +const TRAITS_TABLE_RULES: [&str; 1] = ["APOE"]; +const TRAIT_COLUMNS: [&str; 9] = [ + "rsid", "gene", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids", +]; + +pub fn traits(c: &Ctx<'_>) -> Out { + let Some(p) = c.find1(format!("traits/{}_traits.tsv", c.s())) else { + return Ok(None); + }; + let t = table(&p.path, true)?; + if !t.first_has(&TRAIT_COLUMNS) { + return Err(format!( + "{} has no rows or lacks a column of {}", + base(&p.path), + TRAIT_COLUMNS.join(", ") + )); + } + if let Some(i) = t.rows.iter().position(|r| ragged(r, &TRAIT_COLUMNS)) { + return Err(format!( + "{} line {} does not have one field per column", + base(&p.path), + i + 2 + )); + } + let cell = |r: &pycompat::csv::DictRow, k: &str| r.get(k).flatten().unwrap_or("").to_owned(); + let row = |r: &pycompat::csv::DictRow| { + TRAIT_COLUMNS + .iter() + .fold(Obj::new(), |o, k| o.set(k, s(&cell(r, k)))) + .val() + }; + let apoe = |r: &&pycompat::csv::DictRow| cell(r, "gene") == "APOE"; + let mut out = Obj::new() + .set( + "rows", + Value::Array(t.rows.iter().filter(|r| !apoe(r)).map(row).collect()), + ) + .set( + "apoe", + Value::Array(t.rows.iter().filter(apoe).map(row).collect()), + ); + for st in ["called", "not_covered", "other_allele"] { + out.put( + st, + count( + t.rows + .iter() + .filter(|r| !apoe(r) && cell(r, "status") == st) + .count(), + ), + ); + } + out.put("rules", genotype_rules(c)?); + Ok(Some((p, out.val()))) +} + +/// Step 42's genotype-rules lines, `null` without the table; the APOL1 line +/// gets the G2 reading. +fn genotype_rules(c: &Ctx<'_>) -> Result { + let Some(p) = c.find1(format!("traits/{}_genotype_rules.tsv", c.s())) else { + return Ok(Value::Null); + }; + let t = table(&p.path, true)?; + if !t.rows.is_empty() && RULE_OUT_COLUMNS.iter().any(|k| !t.has(k)) { + return Err(format!( + "{} lacks a column of {}", + base(&p.path), + RULE_OUT_COLUMNS.join(", ") + )); + } + if let Some(i) = t.rows.iter().position(|r| ragged(r, &RULE_OUT_COLUMNS)) { + return Err(format!( + "{} line {} does not have one field per column", + base(&p.path), + i + 2 + )); + } + let mut out = Vec::new(); + for r in &t.rows { + let cell = |k: &str| r.get(k).flatten().unwrap_or("").to_owned(); + if TRAITS_TABLE_RULES.contains(&cell("rule").as_str()) { + continue; + } + let mut o = RULE_OUT_COLUMNS + .iter() + .fold(Obj::new(), |o, k| o.set(k, s(&cell(k)))); + if cell("rule") == apol1::G2_RULE { + let g2 = apol1::read(c); + let both = apol1::combine(&cell("result"), &cell("status"), &g2); + let mut v = g2.json(); + v.put("result", s(&both.0)); + v.put("reading", s(&both.1)); + o.put("g2", v.val()); + } + out.push(o.val()); + } + Ok(Value::Array(out)) +} + +/// APOL1 G2 (rs71785313), a 6-base deletion: read from the VCF records over +/// its span, and put together with the G1 result of step 42's APOL1 rule. +pub mod apol1 { + use super::*; + + pub const G2_RULE: &str = "APOL1"; + const CHROM: &str = "chr22"; + const POS: i64 = 36_265_995; + const REF: &str = "AATAATT"; + const ALT: &str = "A"; + /// GRCh38 chr22:36265975-36266025, the plus strand. + const CONTEXT_START: i64 = 36_265_975; + const CONTEXT: &str = "AGAAGCTAAACATTCTCAACAATAATTATAAGATTCTGCAGGCGGACCAAG"; + const CONTEXT_END: i64 = CONTEXT_START + CONTEXT.len() as i64 - 1; + /// The anchor base of the left-aligned record to the end of the span a + /// 6-base deletion there can be written over. + pub const SPAN: (i64, i64) = (36_265_995, 36_266_005); + const TWO_COPIES: &str = "the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis."; + + /// What `read_g2` gave, with the file it read. + #[derive(Debug, Clone, PartialEq)] + pub struct G2 { + pub status: &'static str, + pub copies: Option, + pub gq: String, + pub text: String, + /// The VCF read, relative to the folder; `None` with no VCF. A + /// read error writes it before the text. + pub source: Option, + pub source_first: bool, + } + + impl G2 { + pub fn json(&self) -> Obj { + let o = Obj::new() + .set("status", s(self.status)) + .set("copies", self.copies.map_or(Value::Null, |c| Value::Int(c.into()))) + .set("gq", s(&self.gq)); + let src = super::super::util::opt_s(self.source.as_deref()); + if self.source_first { + o.set("source", src).set("text", s(&self.text)) + } else { + o.set("text", s(&self.text)).set("source", src) + } + } + } + + /// The context with `r` at `pos` replaced by `a`, or `None` when `r` is + /// not the GRCh38 bases there. + pub fn apply(pos: i64, r: &str, a: &str) -> Option { + let i = pos - CONTEXT_START; + if i < 0 || pos + r.len() as i64 - 1 > CONTEXT_END { + return None; + } + let i = i as usize; + if CONTEXT.get(i..i + r.len()) != Some(r) { + return None; + } + Some(format!("{}{a}{}", &CONTEXT[..i], &CONTEXT[i + r.len()..])) + } + + fn haplotype() -> Option { + apply(POS, REF, ALT) + } + + fn digits(x: &str) -> bool { + pycompat::text::isdigit(x) + } + + fn record_end(r: &[String]) -> Option { + if let Some(info) = r.get(7) { + for (i, _) in info.match_indices("END=") { + if i == 0 || info.as_bytes()[i - 1] == b';' { + let d: String = info[i + 4..] + .chars() + .take_while(char::is_ascii_digit) + .collect(); + if !d.is_empty() { + return d.parse().ok(); + } + } + } + } + Some(r[1].parse::().ok()? + r[3].chars().count() as i64 - 1) + } + + fn overlaps(r: &[String]) -> bool { + r.len() > 4 + && r[0] == CHROM + && digits(&r[1]) + && r[1].parse::().is_ok_and(|p| p <= SPAN.1) + && record_end(r).is_some_and(|e| e >= SPAN.0) + } + + fn symbolic(a: &str) -> bool { + a == "*" || a == "." || (a.len() >= 2 && a.starts_with('<') && a.ends_with('>')) + } + + /// G2 from the records over [`SPAN`]; `gvcf` says a base with no + /// record has no reads. + pub fn read_g2(records: &[Vec], gvcf: bool) -> G2 { + let g2_hap = haplotype(); + let (mut copies, mut covered, mut gqs, mut problems) = + (0i64, Vec::<(i64, i64)>::new(), Vec::::new(), Vec::::new()); + let (mut seen, mut g2_records) = (false, 0); + 'records: for r in records { + if !overlaps(r) { + continue; + } + if r.len() < 10 { + problems.push("a record over G2 has no sample column".into()); + continue; + } + seen = true; + let pos: i64 = r[1].parse().unwrap_or(0); + let end = record_end(r).unwrap_or(pos); + let rf = r[3].as_str(); + let alleles: Vec<&str> = std::iter::once(rf).chain(r[4].split(',')).collect(); + let mut fmt = indexmap::IndexMap::new(); + for (k, v) in r[8].split(':').zip(r[9].split(':')) { + fmt.insert(k, v); + } + let field = |k: &str| fmt.get(k).copied().unwrap_or(".").to_owned(); + let gt = field("GT"); + let gt: Vec<&str> = gt.split(['/', '|']).collect(); + if gt.iter().any(|g| !digits(g)) { + problems.push(format!("a no-call at {}:{pos}", r[0])); + continue; + } + let idx: Vec = gt.iter().map(|g| g.parse().unwrap_or(usize::MAX)).collect(); + if idx.iter().any(|i| *i >= alleles.len()) { + problems.push(format!( + "a genotype at {}:{pos} names an allele the record lacks", + r[0] + )); + continue; + } + if idx.iter().all(|i| *i == 0) { + covered.push((pos, end)); + gqs.push(field("GQ")); + continue; + } + if r[6] != "PASS" && r[6] != "." { + problems.push(format!( + "a call at {}:{pos} that failed the caller's filter ({})", + r[0], r[6] + )); + continue; + } + let mut n = 0; + let mut stopped = false; + for &i in &idx { + if i == 0 { + continue; + } + let a = alleles[i]; + let inside = pos >= CONTEXT_START && end <= CONTEXT_END; + if inside && apply(pos, rf, rf).is_none() { + problems.push(format!( + "REF {rf} at {}:{pos} is not the GRCh38 base (is the VCF on GRCh38?)", + r[0] + )); + stopped = true; + break; + } + let hap = if symbolic(a) || !inside { + None + } else { + apply(pos, rf, a) + }; + if hap.is_some() && hap == g2_hap { + n += 1; + } else { + problems.push(format!( + "an allele other than G2 inside its span ({}:{pos} {rf}>{a})", + r[0] + )); + stopped = true; + break; + } + } + if stopped { + continue; + } + if n > 0 { + g2_records += 1; + if g2_records > 1 { + problems.push("more than one record calls the G2 deletion".into()); + break 'records; + } + } + copies += n; + covered.push((pos, end)); + gqs.push(field("GQ")); + } + let gq = if gqs.is_empty() { + ".".to_owned() + } else { + gqs.join(",") + }; + let g2 = |status, copies, gq: String, text: String| G2 { + status, + copies, + gq, + text, + source: None, + source_first: false, + }; + if let Some(p) = problems.first() { + return g2( + "not_read", + None, + gq, + format!("G2 (rs71785313) not read: {p}."), + ); + } + if copies > 0 { + return g2( + "called", + Some(copies), + gq, + format!( + "G2 (rs71785313): {copies} cop{} of the deletion called.", + if copies > 1 { "ies" } else { "y" } + ), + ); + } + if (SPAN.0..=SPAN.1).all(|p| covered.iter().any(|(a, b)| *a <= p && p <= *b)) { + return g2( + "called", + Some(0), + gq, + "G2 (rs71785313): the reference over the whole deletion span, no G2 copy.".into(), + ); + } + if !gvcf && !seen { + return g2( + "no_call_record", + Some(0), + ".".into(), + "G2 (rs71785313): no G2 deletion among the VCF's calls.".into(), + ); + } + g2( + "not_covered", + None, + gq, + "G2 (rs71785313) not read: no genotype over part of the deletion span.".into(), + ) + } + + /// The records of `chr22` that can cover the span: by the tabix index + /// when the `.tbi` is there, else the whole file. + fn window(path: &str) -> Result>, String> { + let tbi = format!("{path}.tbi"); + let mut out = Vec::new(); + if std::path::Path::new(&tbi).exists() { + let mut rd = og_io::tabix::Reader::with_index(path, &tbi).map_err(|e| e.to_string())?; + for line in rd.query(CHROM, (SPAN.0 - 1) as u64, SPAN.1 as u64) { + let line = line.map_err(|e| e.to_string())?; + out.push(super::super::util::cells(&line)); + } + return Ok(out); + } + vcf_lines(path, |l| { + let c = super::super::util::cells(l); + if c.len() >= 2 && c[0] == CHROM && digits(&c[1]) { + out.push(c); + } + })?; + Ok(out) + } + + /// G2 from the sample's gVCF when it is there, else its VCF. + pub fn read(c: &Ctx<'_>) -> G2 { + for (name, gvcf) in [ + (format!("vcf/{}.g.vcf.gz", c.sample), true), + (format!("vcf/{}.vcf.gz", c.sample), false), + ] { + let p = c.path(&name); + if !crate::is_file(&p) { + continue; + } + return match window(&p) { + Err(e) => G2 { + status: "not_read", + copies: None, + gq: ".".into(), + text: format!("G2 (rs71785313) not read: {name} could not be read ({e})."), + source: Some(name), + source_first: true, + }, + Ok(recs) => G2 { + source: Some(name), + ..read_g2(&recs, gvcf) + }, + }; + } + G2 { + status: "not_read", + copies: None, + gq: ".".into(), + text: "G2 (rs71785313) not read: no VCF of the sample in this folder.".into(), + source: None, + source_first: true, + } + } + + /// The G1 + G2 result and its reading, from step 42's APOL1 line + /// (`result`, `status`) and the G2 reading. + pub fn combine(result: &str, status: &str, g2: &G2) -> (String, String) { + let g1: Option = if status == "called" { + match result { + "G0/G0" => Some(0), + "G0/G1" => Some(1), + "G1/G1" => Some(2), + _ => None, + } + } else { + None + }; + let two = |label: &str| format!("Two APOL1 risk copies ({label}): {TWO_COPIES}"); + let Some(g1) = g1 else { + if g2.status == "called" && g2.copies == Some(2) { + return ( + "G2/G2".into(), + two("G2/G2") + + " The G1 sites were not read, but two G2 copies leave no copy for G1.", + ); + } + return ( + ".".into(), + "Not read: the G1 sites of the APOL1 rule were not read.".into(), + ); + }; + let Some(c) = g2.copies else { + return (".".into(), "The G1 result above stands on its own.".into()); + }; + if g1 + c > 2 { + return ( + ".".into(), + format!( + "{g1} G1 and {c} G2 copies do not fit two copies of the gene (G1 and G2 never share a copy): not read. Check the calls." + ), + ); + } + let mut label = std::iter::repeat_n("G0", (2 - g1 - c) as usize) + .chain(std::iter::repeat_n("G1", g1 as usize)) + .chain(std::iter::repeat_n("G2", c as usize)) + .collect::>() + .join("/"); + let mut reading = match g1 + c { + 2 => two(&label), + 1 => format!( + "One APOL1 risk copy ({label}): one copy did not raise the risk in Genovese 2010 (a recessive pattern)." + ), + _ => "No G1 and no G2 copy: no APOL1 kidney risk copy.".into(), + }; + if g2.status == "no_call_record" { + label += " (no G2 call)"; + reading += " No G2 deletion was called, but this VCF has no reference blocks, so a G2 copy at a position with no reads would not show."; + } + (label, reading) + } + + #[cfg(test)] + mod tests { + use super::*; + + fn rec(pos: i64, r: &str, a: &str, gt: &str, info: &str) -> Vec { + [ + CHROM, + &pos.to_string(), + ".", + r, + a, + "50", + "PASS", + info, + "GT:GQ", + &format!("{gt}:40"), + ] + .iter() + .map(|x| (*x).to_owned()) + .collect() + } + + #[test] + fn the_marker_is_a_six_base_deletion_of_the_context() { + let h = haplotype().expect("REF is the GRCh38 bases"); + assert_eq!(CONTEXT.len() - h.len(), 6); + } + + #[test] + fn g2_left_aligned_is_one_copy() { + let g = read_g2(&[rec(POS, REF, ALT, "0/1", ".")], false); + assert_eq!((g.status, g.copies), ("called", Some(1))); + } + + #[test] + fn g2_shifted_is_matched() { + // The same deletion written three bases to the right: TAATTA>T + // after AATAATTATAA's repeat gives the same sequence. + let g = read_g2(&[rec(36_265_998, "ATTATA", "A", "1/1", ".")], false); + assert_eq!(g.status, "not_read", "{g:?}"); + let shifted = rec(36_265_996, "ATAATTA", "A", "1/1", "."); + let g = read_g2(&[shifted], false); + assert_eq!((g.status, g.copies), ("called", Some(2)), "{g:?}"); + } + + #[test] + fn reference_block_over_span_reads_zero() { + let g = read_g2(&[rec(36_265_990, "A", "", "0/0", "END=36266010")], true); + assert_eq!((g.status, g.copies), ("called", Some(0))); + } + + #[test] + fn vcf_without_record_is_no_call_record() { + let g = read_g2(&[], false); + assert_eq!(g.status, "no_call_record"); + let g = read_g2(&[], true); + assert_eq!(g.status, "not_covered"); + } + + #[test] + fn two_g2_records_are_not_read() { + let a = rec(POS, REF, ALT, "0/1", "."); + let b = rec(36_265_996, "ATAATTA", "A", "0/1", "."); + let g = read_g2(&[a, b], false); + assert_eq!(g.status, "not_read"); + assert!(g.text.contains("more than one record")); + } + + #[test] + fn combine_counts_both() { + let g2 = read_g2(&[rec(POS, REF, ALT, "0/1", ".")], false); + assert_eq!(combine("G0/G1", "called", &g2).0, "G1/G2"); + let none = read_g2(&[], false); + let (label, reading) = combine("G0/G0", "called", &none); + assert_eq!(label, "G0/G0 (no G2 call)"); + assert!(reading.contains("no reference blocks")); + } + } +} diff --git a/rust/crates/og-summary/src/collect/group_c/util.rs b/rust/crates/og-summary/src/collect/group_c/util.rs new file mode 100644 index 000000000..a89d331b6 --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_c/util.rs @@ -0,0 +1,387 @@ +//! What every reader of group c shares: the table reader with its header, +//! the cell lookups with the Python collector's three answers (no such +//! column, a row cut short, a cell), and an ordered JSON object builder. + +use crate::collect::sections::{num, py_error}; +use crate::{basename, repr, text_lines}; +use indexmap::IndexMap; +use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::json::Value; + +/// A table read as `csv.DictReader` reads it, with its header cells. +pub struct Table { + /// The header cells as written; empty for an empty file. + pub head: Vec, + pub rows: Vec, +} + +impl Table { + /// Whether the header names `col` (`col in reader.fieldnames`). + pub fn has(&self, col: &str) -> bool { + self.head.iter().any(|h| h == col) + } + + /// Whether the first row has every column (`need <= set(rows[0])`): + /// false for a table without rows. + pub fn first_has(&self, need: &[&str]) -> bool { + !self.rows.is_empty() && need.iter().all(|k| self.has(k)) + } +} + +/// Reads a tab-separated table with a header. `quote_none` reads `"` as a +/// plain character (`quoting=csv.QUOTE_NONE`). +pub fn table(path: &str, quote_none: bool) -> Result { + let mut lines = text_lines(path).map_err(|e| os_note(&e))?; + let iter = std::iter::from_fn(move || { + let mut s = String::new(); + match lines.read_line(&mut s) { + Ok(true) => Some(Ok(s)), + Ok(false) => None, + Err(e) => Some(Err(e)), + } + }); + let q = if quote_none { + Quoting::None + } else { + Quoting::Minimal + }; + let mut rd = DictReader::new(iter, '\t', q); + let csv_err = |e: pycompat::csv::Error| py_error(e.exception(), &e.to_string()); + let head = rd + .header() + .map_err(csv_err)? + .map(|h| h.names().to_vec()) + .unwrap_or_default(); + let rows = rd.collect::, _>>().map_err(csv_err)?; + Ok(Table { head, rows }) +} + +/// The lines of a text file, line ends removed (`open(p, errors="replace")`). +pub fn lines(path: &str) -> Result, String> { + let mut rd = text_lines(path).map_err(|e| os_note(&e))?; + let mut out = Vec::new(); + let mut s = String::new(); + while rd.read_line(&mut s).map_err(|e| { + py_error( + "OSError", + &crate::collect::sections::gzip_message(&e), + ) + })? { + out.push(s.strip_suffix('\n').unwrap_or(&s).to_owned()); + s.clear(); + } + Ok(out) +} + +/// An `OSError` from opening a file, as the collector's catch-all wrote it. +pub fn os_note(e: &str) -> String { + let kind = if e.contains("No such file") { + "FileNotFoundError" + } else if e.contains("Permission denied") { + "PermissionError" + } else if e.contains("Is a directory") { + "IsADirectoryError" + } else { + "OSError" + }; + py_error(kind, e) +} + +/// `KeyError: 'k'`, a column a row lacks. +pub fn key_error(k: &str) -> String { + py_error("KeyError", &repr(k)) +} + +/// `row[k]`: an error when the header has no such column, `None` for a +/// row cut short. +pub fn at<'a>(r: &'a DictRow, k: &str) -> Result, String> { + r.get(k).ok_or_else(|| key_error(k)) +} + +/// `row[k]` that must be a text: a short row is an error too. +pub fn text<'a>(r: &'a DictRow, k: &str) -> Result<&'a str, String> { + at(r, k)?.ok_or_else(|| { + py_error( + "TypeError", + &format!("the cell {} of a short row is None", repr(k)), + ) + }) +} + +/// `row.get(k) or "."`. +pub fn or_dot(r: &DictRow, k: &str) -> String { + or(r, k, ".") +} + +/// `row.get(k) or default`. +pub fn or(r: &DictRow, k: &str, default: &str) -> String { + match r.get(k).flatten() { + Some(v) if !v.is_empty() => v.to_owned(), + _ => default.to_owned(), + } +} + +/// `row.get(k, default)`: the default without the column, `null` for a +/// row cut short. +pub fn get_default(r: &DictRow, k: &str, default: &str) -> Value { + match r.get(k) { + None => s(default), + Some(None) => Value::Null, + Some(Some(v)) => s(v), + } +} + +/// Whether a row has a cell past the header (Python's `None` key) or is +/// cut short of a column of `need`. +pub fn ragged(r: &DictRow, need: &[&str]) -> bool { + r.extra().is_some() || need.iter().any(|k| !matches!(r.get(k), Some(Some(_)))) +} + +/// `str.isdigit()` of an ASCII cell. +pub fn isdigit(x: &str) -> bool { + pycompat::text::isdigit(x) +} + +/// `float(x)` that is a number and not NaN. +pub fn number(x: &str) -> Option { + num(Some(x)) +} + +/// `os.path.basename`. +pub fn base(p: &str) -> &str { + basename(p) +} + +pub fn s(x: &str) -> Value { + Value::Str(x.to_owned()) +} + +pub fn opt_s(x: Option<&str>) -> Value { + x.map_or(Value::Null, s) +} + +pub fn strs(xs: I) -> Value +where + I: IntoIterator, + S: AsRef, +{ + Value::Array(xs.into_iter().map(|x| s(x.as_ref())).collect()) +} + +pub fn int(n: impl Into) -> Value { + Value::Int(n.into()) +} + +pub fn count(n: usize) -> Value { + Value::Int(n as i128) +} + +/// An object built in insertion order. +#[derive(Debug, Default, Clone)] +pub struct Obj(pub IndexMap); + +impl Obj { + pub fn new() -> Obj { + Obj::default() + } + + pub fn set(mut self, k: &str, v: Value) -> Obj { + self.0.insert(k.to_owned(), v); + self + } + + pub fn put(&mut self, k: &str, v: Value) { + self.0.insert(k.to_owned(), v); + } + + pub fn val(self) -> Value { + Value::Object(self.0) + } +} + +/// `{k: row.get(k) or "." for k in keys}`. +pub fn dots(r: &DictRow, keys: &[&str]) -> Value { + keys.iter() + .fold(Obj::new(), |o, k| o.set(k, s(&or_dot(r, k)))) + .val() +} + +/// `{k: row[k] or "." for k in keys}`: a column the row lacks is an error. +pub fn dots_strict(r: &DictRow, keys: &[&str]) -> Result { + let mut o = Obj::new(); + for k in keys { + let v = at(r, k)?.filter(|v| !v.is_empty()).unwrap_or("."); + o.put(k, s(v)); + } + Ok(o.val()) +} + +/// `{r[key]: r.get(value) for r in rows}` of a `metric\tvalue` table: the +/// last row of a key wins, in the place of its first. +pub fn metric_map(t: &Table, key: &str, value: &str) -> IndexMap, Option> { + let mut out = IndexMap::new(); + for r in &t.rows { + let k = r.get(key).flatten().map(str::to_owned); + let v = r.get(value).flatten().map(str::to_owned); + out.insert(k, v); + } + out +} + +/// `kv.get(k) or "."` over a [`metric_map`]. +pub fn kv_dot(kv: &IndexMap, Option>, k: &str) -> String { + match kv.get(&Some(k.to_owned())) { + Some(Some(v)) if !v.is_empty() => v.clone(), + _ => ".".to_owned(), + } +} + +/// `kv.get(k)` over a [`metric_map`]. +pub fn kv_get<'a>(kv: &'a IndexMap, Option>, k: &str) -> Option<&'a str> { + kv.get(&Some(k.to_owned())).and_then(|v| v.as_deref()) +} + +/// `[(metric, value)]` of a two-column table with a header line +/// (`read_tsv_pairs`): blank lines skipped, short lines left out. +pub fn pairs(path: &str) -> Result, String> { + let ls = lines(path)?; + let ls: Vec<&String> = ls + .iter() + .filter(|l| !pycompat::num::py_strip(l).is_empty()) + .collect(); + Ok(ls + .iter() + .skip(1) + .filter_map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (c.len() >= 2).then(|| (c[0].to_owned(), c[1].to_owned())) + }) + .collect()) +} + +/// `round(x, n)` as a JSON float. +pub fn round(x: f64, n: i32) -> Value { + Value::Float(pycompat::fmt::round(x, n).unwrap_or(x)) +} + +/// The rows of `assets/`, comment lines left out (`asset_rows`). +pub fn asset_rows(assets: &str, name: &str) -> Result, String> { + let p = crate::join(assets, name); + if !crate::is_file(&p) { + return Err(format!( + "{name} is not in {assets}: the report needs the repository's assets/ folder (render_report.py --assets)" + )); + } + let text = std::fs::read(&p).map_err(|e| py_error("OSError", &e.to_string()))?; + let text = + String::from_utf8(text).map_err(|e| py_error("UnicodeDecodeError", &e.to_string()))?; + let lines = text + .split_inclusive('\n') + .filter(|l| !l.starts_with('#')) + .map(|l| Ok::<_, std::io::Error>(l.to_owned())); + DictReader::new(lines, '\t', Quoting::None) + .collect::, _>>() + .map_err(|e| py_error(e.exception(), &e.to_string())) +} + +/// The header of a table as `csv.reader` reads its first row (empty for an +/// empty file). +pub fn columns(path: &str) -> Result, String> { + Ok(table(path, false)?.head) +} + +/// The path of a table the Python collector read beside itself in `bin/` +/// (`hla_alleles.tsv`, `str_intermediate_ranges.tsv`): the `bin/` of the +/// checkout the binary sits in (`bin/.og/og-*` and `rust/target/*/og` both +/// do). `OG_BIN_TABLES` names another folder. +pub fn bin_table(name: &str) -> String { + if let Some(d) = std::env::var_os("OG_BIN_TABLES") { + return std::path::Path::new(&d).join(name).display().to_string(); + } + let exe = std::env::current_exe().unwrap_or_default(); + exe.ancestors() + .skip(1) + .flat_map(|a| [a.join(name), a.join("bin").join(name)]) + .find(|p| p.is_file()) + .unwrap_or_else(|| std::path::Path::new("bin").join(name)) + .display() + .to_string() +} + +/// The data rows of a `bin/` table: `#` and blank lines left out, the first +/// row the header, each row zipped with it (a short row lacks the keys). +pub fn bin_rows(path: &str) -> Result>, String> { + let text = std::fs::read(path).map_err(|e| { + py_error( + if e.kind() == std::io::ErrorKind::NotFound { + "FileNotFoundError" + } else { + "OSError" + }, + &crate::os_error(&e, path), + ) + })?; + let text = + String::from_utf8(text).map_err(|e| py_error("UnicodeDecodeError", &e.to_string()))?; + let mut head: Option> = None; + let mut out = Vec::new(); + for line in text.lines() { + if line.starts_with('#') || pycompat::num::py_strip(line).is_empty() { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + match &head { + None => head = Some(c), + Some(h) => out.push( + h.iter() + .zip(c) + .map(|(k, v)| ((*k).to_owned(), v.to_owned())) + .collect(), + ), + } + } + Ok(out) +} + +/// Each data line of a VCF, plain or gzip (BGZF inflated on many cores), +/// without its line end; `#` lines are left out. Universal newlines, as +/// Python reads text. An error is `: `. +pub fn vcf_lines(path: &str, mut each: impl FnMut(&[u8])) -> Result<(), String> { + use std::io::BufRead; + let fail = |e: std::io::Error| { + format!( + "{}: {}", + base(path), + crate::collect::sections::gzip_message(&e) + ) + }; + let mut r = og_io::bgzf::open(path, og_io::bgzf::workers()).map_err(fail)?; + let mut buf = Vec::with_capacity(1 << 16); + let mut one = |l: &[u8]| { + if l.first() != Some(&b'#') { + each(l); + } + }; + loop { + buf.clear(); + if r.read_until(b'\n', &mut buf).map_err(fail)? == 0 { + break; + } + let line = buf.strip_suffix(b"\n").unwrap_or(&buf); + if memchr::memchr(b'\r', line).is_none() { + one(line); + } else { + let line = line.strip_suffix(b"\r").unwrap_or(line); + line.split(|&c| c == b'\r').for_each(&mut one); + } + } + Ok(()) +} + +/// A VCF line's cells, as text (`errors="replace"`). +pub fn cells(line: &[u8]) -> Vec { + String::from_utf8_lossy(line) + .split('\t') + .map(str::to_owned) + .collect() +} diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 26d7c0087..698316d08 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -87,6 +87,23 @@ impl FolderIndex { } None } + + /// Every file of one pattern, sorted (`sorted(glob(pattern))`, files + /// only). + pub fn all(&self, pattern: &str) -> Vec<&str> { + if !has_magic(pattern) { + return self.files.get(pattern).map(|f| vec![f.as_str()]).unwrap_or_default(); + } + let pat: Vec<&str> = pattern.split('/').collect(); + self.sorted + .iter() + .filter(|f| { + let parts: Vec<&str> = f.split('/').collect(); + matches(&parts, &pat) + }) + .map(String::as_str) + .collect() + } } fn has_magic(s: &str) -> bool { diff --git a/rust/crates/og-summary/src/collect/sections.rs b/rust/crates/og-summary/src/collect/sections.rs index 8aa04d534..d64f0ac5a 100644 --- a/rust/crates/og-summary/src/collect/sections.rs +++ b/rust/crates/og-summary/src/collect/sections.rs @@ -20,6 +20,8 @@ pub enum Data { MosaicSexLoss(Mosaic), ReadCount(ReadCount), Variants(Variants), + /// A section of group c, kept as the ordered JSON it writes. + Json(Value), } impl Data { @@ -32,6 +34,7 @@ impl Data { Data::MosaicSexLoss(d) => d.to_json(), Data::ReadCount(d) => d.to_json(), Data::Variants(d) => d.to_json(), + Data::Json(v) => v.clone(), } } } @@ -750,7 +753,7 @@ fn chars(b: &[u8]) -> usize { /// What Python's gzip says for the same fault, where the two are the same /// fault; otherwise the reader's own words. -fn gzip_message(e: &std::io::Error) -> String { +pub(crate) fn gzip_message(e: &std::io::Error) -> String { if e.kind() == std::io::ErrorKind::UnexpectedEof { "Compressed file ended before the end-of-stream marker was reached".to_owned() } else { diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 36e41683e..65922dd61 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -21,6 +21,7 @@ fn summary(d: &Dir, sample_dir: &str) -> Value { sample_dir: d.path(sample_dir), declared_sex: None, assets: d.path("assets"), + group: super::Group::A, }; collect(&input).unwrap().to_json() } diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 234ea7cac..4a9ff4b55 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,8 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only a [args] + og summary collect --only a|c [args] + og roh summary --sample SAMPLE --roh ROH --out OUT og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] og bloodtype report|check-markers|systems [args] @@ -303,6 +304,13 @@ fn main() -> ExitCode { )), _ => usage_error("og summary takes collect"), }, + "roh" => match rest.first().map(String::as_str) { + Some("summary") => ExitCode::from(og_summary::collect::group_c::roh::cli( + &rest[1..], + &mut std::io::stderr().lock(), + )), + _ => usage_error("og roh takes summary"), + }, "mito" => ExitCode::from(og_summary::mito::cli( rest.first().map(String::as_str), rest.get(1..).unwrap_or_default(), diff --git a/scripts/11-roh-analysis.sh b/scripts/11-roh-analysis.sh index af2dfa241..a0f6b29ce 100755 --- a/scripts/11-roh-analysis.sh +++ b/scripts/11-roh-analysis.sh @@ -2,7 +2,7 @@ # ROH Analysis — Runs of homozygosity (consanguinity screening) # Detects long stretches of homozygous DNA that indicate shared ancestry. # The summary keeps chr1 to chr22 only and leaves out the part of each segment in a centromere -# or heterochromatin band (MASKED_BANDS below), where few markers map. +# or heterochromatin band (og roh summary's MASKED_BANDS), where few markers map. set -euo pipefail SAMPLE=${1:?Usage: $0 } @@ -44,31 +44,18 @@ run_in \ "${BCFTOOLS_IMAGE}" \ bash -euo pipefail -c "bcftools view -f PASS,. -Ou /data/${SAMPLE}.vcf.gz | bcftools roh ${ROH_FLAGS[*]} -o /data/${SAMPLE}_roh.txt -" -# GRCh38's centromere and heterochromatin bands: the acen, gvar and stalk -# rows of UCSC's cytoBand.hg38.txt, neighbouring rows merged (0-based start, -# exclusive end). bin/collect_summary.py (MASKED_BANDS, which says how to -# regenerate it) and modules/local/roh/main.nf hold the same line. -MASKED_BANDS="chr1:121700000-143200000 chr2:91800000-96000000 chr3:87800000-98600000 chr4:48200000-51800000 chr5:46100000-51400000 chr6:58500000-62600000 chr7:58100000-62100000 chr8:43200000-47200000 chr9:42200000-61500000 chr10:38000000-41600000 chr11:51000000-55800000 chr12:33200000-37800000 chr13:0-18900000 chr14:0-18200000 chr15:0-20500000 chr16:35300000-47000000 chr17:22700000-27400000 chr18:15400000-21500000 chr19:19900000-31900000 chr20:25700000-30400000 chr21:0-13000000 chr22:0-17400000 chrX:58100000-63800000 chrY:10300000-10600000 chrY:26600000-57227415" - -# Summary: autosomal segments with 5 Mb or more outside those bands, the -# threshold of docs/11-roh-analysis.md and of the reports; length_bp and -# length_mb are of that part. The Nextflow module writes the same file. +# Summary: autosomal segments with 5 Mb or more outside GRCh38's centromere +# and heterochromatin bands (the acen, gvar and stalk rows of UCSC's +# cytoBand.hg38.txt, merged; MASKED_BANDS in og's step 11 code, the one copy), +# the threshold of docs/11-roh-analysis.md and of the reports; length_bp and +# length_mb are of that part. The Nextflow module runs the same command. SUMMARY="${GENOME_DIR}/${SAMPLE}/vcf/${SAMPLE}_roh_summary.txt" -{ - echo "# ROH Summary for ${SAMPLE}" - echo "# Segments >=5MB on autosomes (excludes chrX/chrY)" - printf 'chrom\tstart\tend\tlength_bp\tlength_mb\n' - awk -v bands="$MASKED_BANDS" ' - BEGIN { n = split(bands, b, " "); for (i = 1; i <= n; i++) { split(b[i], x, /[:-]/); c[i] = x[1]; s[i] = x[2]; e[i] = x[3] } } - $1 == "RG" && $3 ~ /^chr([1-9]|1[0-9]|2[0-2])$/ { - m = 0 - for (i = 1; i <= n; i++) if (c[i] == $3) { o = ($5 < e[i] ? $5 : e[i]) - ($4 - 1 > s[i] ? $4 - 1 : s[i]); if (o > 0) m += o } - if (m > $6) m = $6 - l = $6 - m - if (l >= 5000000) printf "%s\t%s\t%s\t%d\t%.1f\n", $3, $4, $5, l, l / 1e6 - }' "$OUTPUT" -} > "${SUMMARY}.tmp" -mv -f "${SUMMARY}.tmp" "$SUMMARY" +run_in \ + --cpus 1 --memory 1g \ + -v "${GENOME_DIR}/${SAMPLE}/vcf:/data" \ + -v "${PGP_ROOT}/bin:/pgp-bin:ro" \ + "${BCFTOOLS_IMAGE}" \ + /pgp-bin/og roh summary --sample "$SAMPLE" --roh "/data/${SAMPLE}_roh.txt" --out "/data/${SAMPLE}_roh_summary.txt" echo "=== ROH complete ===" echo "Results: ${OUTPUT}" diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 62d7d1b93..150e5ee17 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1905,11 +1905,19 @@ def report_sections_mito_roh_telomere_cnv(work, schema): """ROH on autosomes outside the bands, the telomere read floor, CNVpytor's filtered count, the D-loop poly-C tracts and the chrM annotation.""" import re as _re - # The masked bands: one line in the summary, step 11 and the ROH module. + # The masked bands: og's step 11 code holds the one copy that step 11 + # and the ROH module run (og roh summary); the summary's line agrees. + rs = open(os.path.join(REPO, "rust/crates/og-summary/src/collect/group_c/roh.rs")).read() + body = rs[rs.index("pub const MASKED_BANDS"):] + body = body[:body.index("];")] + rust_bands = " ".join(f"{c}:{int(a.replace('_', ''))}-{int(b.replace('_', ''))}" + for c, a, b in _re.findall(r'\("(chr\w+)", ([0-9_]+), ([0-9_]+)\)', body)) + check("bands: og's MASKED_BANDS is the line of bin/collect_summary.py", + rust_bands == collect_summary.MASKED_BANDS, rust_bands[:80]) for f in ("scripts/11-roh-analysis.sh", "modules/local/roh/main.nf"): - m = _re.search(r'^\s*MASKED_BANDS="([^"]*)"', open(os.path.join(REPO, f)).read(), _re.M) - check(f"bands: {f} holds the line of bin/collect_summary.py", - m is not None and m.group(1) == collect_summary.MASKED_BANDS, m and m.group(1)[:80]) + text = open(os.path.join(REPO, f)).read() + check(f"bands: {f} holds no copy of the bands and runs og roh summary", + "MASKED_BANDS=" not in text and "og roh summary" in text, f) # ROH: a 45 Mb chrY segment, an 18 Mb segment in chr1's 1q12 gvar block, # one over chr9's centromere, a 25 Mb chr1 segment with 21.5 Mb in the @@ -1943,13 +1951,17 @@ def report_sections_mito_roh_telomere_cnv(work, schema): collect_summary.masked_bands = orig check("ROH red control: without the band mask the gvar segment is the largest again", r0["largest_mb"] == 25.0 and len(r0["autosomal_over_5mb"]) == 4, r0) - # Step 11's awk on the same file lists the same segments. - script = open(os.path.join(REPO, "scripts/11-roh-analysis.sh")).read() - awk = script[script.index(" awk -v bands="):script.index('"$OUTPUT"', script.index(" awk -v bands=")) + len('"$OUTPUT"')] - out = subprocess.run(["bash", "-c", f'MASKED_BANDS="{collect_summary.MASKED_BANDS}"; OUTPUT="$1"; {awk}', "_", - f"{d}/vcf/S_roh.txt"], capture_output=True, text=True) - check("ROH: step 11's summary awk lists the segments the report lists", - out.returncode == 0 and out.stdout == "chr2\t10000001\t17000000\t7000000\t7.0\n", (out.stdout, out.stderr)) + # Step 11's summary (og roh summary) on the same file lists the same segments. + if OG: + out = subprocess.run([OG, "roh", "summary", "--sample", "S", "--roh", f"{d}/vcf/S_roh.txt", + "--out", f"{d}/vcf/S_roh_summary.txt"], capture_output=True, text=True) + rows = open(f"{d}/vcf/S_roh_summary.txt").read().splitlines()[3:] if out.returncode == 0 else [] + check("ROH: step 11's summary (og roh summary) lists the segments the report lists", + rows == ["chr2\t10000001\t17000000\t7000000\t7.0"], (rows, out.stderr)) + elif os.environ.get("OG_REQUIRED"): + sys.exit("FAIL: OG_REQUIRED is set but OG names no og binary") + else: + print("[SKIP] og roh summary: OG is not set (the rust job of guard.yml runs it)") # Telomere: 40 and 4,000 intratelomeric reads at 30x (620 M reads of 150 bp). head = ("PID\tsample\ttotal_reads\tread_lengths\trepeat_threshold_set\trepeat_threshold_used\tintratel_reads\t" From 075b43becf59b7d339ad1eaef2afa3f9704ff232 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 06/24] perf(summary): read the clinical VCF and the ClinVar hits line by line The clinical VCF, slivar's VCF and the hits were read whole before being split; now one pass streams each, the hits are no longer copied for the ClinVar section, and the summary's own data is freed before the JSON text is made. A table cut short reads as Python's EOFError. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect.rs | 12 +- rust/crates/og-summary/src/collect/acmg.rs | 144 ++++++++++++------ .../crates/og-summary/src/collect/clinical.rs | 4 +- rust/crates/og-summary/src/collect/clinvar.rs | 35 ++--- rust/crates/og-summary/src/collect/text.rs | 49 ++++-- 5 files changed, 169 insertions(+), 75 deletions(-) diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index cc5e827f3..15a29828f 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -7,7 +7,9 @@ //! switches to it. Group `a` is the model and the first seven sections: //! coverage, callability, sex_check, sample_qc, mosaic_sex_loss, //! read_count and variants, with the run status, the manifest and each -//! section's state. +//! section's state. Group `b` is the clinical sections ([`group_b`]): +//! clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical (with +//! the ACMG SF tier and the two-hit rule) and slivar, and the CYP2D6 block. //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -93,8 +95,13 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { Ok(s) => s, Err(e) => return crate::finish(Err(e), err), }; + // The sections' own data is freed before the text is made, and the + // tree before the file is written: a large ClinVar list is in memory in + // one form at a time. + let value = summary.to_json(); + drop(summary); let body = match dumps( - &summary.to_json(), + &value, &Dump { indent: Some(1), ..Dump::default() @@ -103,6 +110,7 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { Ok(b) => b + "\n", Err(e) => return crate::finish(Err(e.to_string()), err), }; + drop(value); let r = crate::write_file(&target, body.as_bytes()); if r.is_ok() { let _ = writeln!(out, "Summary written: {target}"); diff --git a/rust/crates/og-summary/src/collect/acmg.rs b/rust/crates/og-summary/src/collect/acmg.rs index 70439bbd8..db3077299 100644 --- a/rust/crates/og-summary/src/collect/acmg.rs +++ b/rust/crates/og-summary/src/collect/acmg.rs @@ -262,72 +262,124 @@ pub struct ClinicalVcf { } impl ClinicalVcf { + /// One pass over the file, line by line: a whole-genome clinical VCF + /// is never held in memory. pub fn read(path: &str) -> Result { - let lines = text::lines(path).map_err(|e| e.unreadable(path))?; - let records = lines.iter().filter(|l| !l.starts_with('#')).count(); + let mut records = 0; + let mut scan = HighScan::default(); + text::each_line(path, |l| { + if !l.starts_with('#') { + records += 1; + } + scan.line(l); + }) + .map_err(|e| e.unreadable(path))?; Ok(ClinicalVcf { records, - high: high_impact(&lines), + high: scan.finish(), }) } } -/// `csq_high_impact_genes`. -fn high_impact(lines: &[String]) -> Result, String> { - let floor = ClinicalFloor::read(lines)?; - let mut fmt: Option> = None; - let mut out = IndexMap::new(); - for line in lines { +/// `csq_high_impact_genes`, fed one line at a time. The floors come from +/// the header (`clinical_call_floor`), which ends at the first line that +/// does not start with `##`. +#[derive(Default)] +struct HighScan { + header: Vec, + floor: Option>, + fmt: Option>, + out: IndexMap, + err: Option, +} + +impl HighScan { + fn line(&mut self, line: &str) { + if self.floor.is_none() { + if line.starts_with("##") { + self.header.push(line.to_owned()); + } else { + self.floor = Some(ClinicalFloor::read(&std::mem::take(&mut self.header))); + } + } + if self.err.is_some() { + return; + } if line.starts_with("##INFO= = line.split('\t').map(str::to_owned).collect(); - if r.len() < 8 { - continue; + return; } - let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); - let mut genes: IndexMap = IndexMap::new(); - for e in entries.into_iter().filter(|e| e.impact == "HIGH") { - genes.entry(e.gene).or_insert(e.consequence); + if let Err(e) = high_record(fm, line, floor, &mut self.out) { + self.err = Some(e); } - if genes.is_empty() { - continue; + } + + fn finish(mut self) -> Result, String> { + let floor = self + .floor + .take() + .unwrap_or_else(|| ClinicalFloor::read(&self.header)); + floor?; + match self.err { + Some(e) => Err(e), + None => Ok(self.out), } - let gt = text::gt_of(&r).to_owned(); - let (weak, why, gq, dp, ad) = if r.len() > 9 { - call_check(&r[8], &r[9], &floor) - } else { - (false, Vec::new(), None, None, None) - }; - let pos = text::pos(&r)?; - out.insert( - (r[0].clone(), pos, r[3].clone(), r[4].clone()), - HighRecord { - genotype: gt, - ps: gt_field(&r, "PS"), - genes, - transcript: cls, - frequency: freq, - weak, - why, - gq, - dp, - ad, - }, - ); } - Ok(out) +} + +/// One data line of `csq_high_impact_genes`. +fn high_record( + fm: &[String], + line: &str, + floor: &ClinicalFloor, + out: &mut IndexMap, +) -> Result<(), String> { + let r: Vec = line.split('\t').map(str::to_owned).collect(); + if r.len() < 8 { + return Ok(()); + } + let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); + let mut genes: IndexMap = IndexMap::new(); + for e in entries.into_iter().filter(|e| e.impact == "HIGH") { + genes.entry(e.gene).or_insert(e.consequence); + } + if genes.is_empty() { + return Ok(()); + } + let gt = text::gt_of(&r).to_owned(); + let (weak, why, gq, dp, ad) = if r.len() > 9 { + call_check(&r[8], &r[9], floor) + } else { + (false, Vec::new(), None, None, None) + }; + let pos = text::pos(&r)?; + out.insert( + (r[0].clone(), pos, r[3].clone(), r[4].clone()), + HighRecord { + genotype: gt, + ps: gt_field(&r, "PS"), + genes, + transcript: cls, + frequency: freq, + weak, + why, + gq, + dp, + ad, + }, + ); + Ok(()) } /// A source of a candidate variant. diff --git a/rust/crates/og-summary/src/collect/clinical.rs b/rust/crates/og-summary/src/collect/clinical.rs index 6dc161381..5936475e6 100644 --- a/rust/crates/og-summary/src/collect/clinical.rs +++ b/rust/crates/og-summary/src/collect/clinical.rs @@ -107,7 +107,9 @@ impl Slivar { let Some(v) = vcf else { return Ok(Slivar(Value::Null)); }; - let n = text::vcf_records(v).map_err(|e| e.unreadable(v))?.len(); + let mut n = 0; + text::each_line(v, |l| n += usize::from(!l.starts_with('#'))) + .map_err(|e| e.unreadable(v))?; m.insert("prioritized".into(), int(n)); comphet_phase( &crate::join(text::dirname(v), &format!("{s}_compound_het_pairs.tsv")), diff --git a/rust/crates/og-summary/src/collect/clinvar.rs b/rust/crates/og-summary/src/collect/clinvar.rs index 7350c5a6b..f930c25e7 100644 --- a/rust/crates/og-summary/src/collect/clinvar.rs +++ b/rust/crates/og-summary/src/collect/clinvar.rs @@ -79,22 +79,20 @@ pub struct HitsVcf { impl HitsVcf { pub fn read(path: String, rel: String) -> HitsVcf { - let read = text::lines(&path) - .map(|lines| { - let mut floor = Floor::default(); - for l in lines.iter().take_while(|l| l.starts_with("##")) { - if l.starts_with("##PGP_CALL_FLOOR=") { - floor_values(l, false, &mut floor); - } - } - let recs = lines - .iter() - .filter(|l| !l.starts_with('#')) - .map(|l| l.split('\t').map(str::to_owned).collect()) - .collect(); - (floor, recs) - }) - .map_err(|e: ReadError| e.unreadable(&path)); + let mut floor = Floor::default(); + let mut header = true; + let mut recs: Vec> = Vec::new(); + let read = text::each_line(&path, |l| { + header &= l.starts_with("##"); + if header && l.starts_with("##PGP_CALL_FLOOR=") { + floor_values(l, false, &mut floor); + } + if !l.starts_with('#') { + recs.push(l.split('\t').map(str::to_owned).collect()); + } + }) + .map(|()| (floor, recs)) + .map_err(|e: ReadError| e.unreadable(&path)); HitsVcf { path, rel, read } } } @@ -622,7 +620,10 @@ impl ClinVar { side: &dyn Fn(&str) -> Option<(String, String)>, s: &str, ) -> Result { - let (floor, recs) = hits.read.clone()?; + let (floor, recs) = match &hits.read { + Ok((f, r)) => (*f, r), + Err(e) => return Err(e.clone()), + }; let mut rows = Vec::new(); for r in recs.iter().filter(|r| r.len() >= 8) { rows.push(Hit::parse(r, floor)?); diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index a215514b0..d9f554d65 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -42,7 +42,9 @@ impl ReadError { } fn os_kind(e: &str) -> &'static str { - if e.contains("No such file") { + if e == GZ_EOF { + "EOFError" + } else if e.contains("No such file") { "FileNotFoundError" } else if e.contains("Permission denied") { "PermissionError" @@ -64,22 +66,29 @@ pub fn lines(path: &str) -> Result, ReadError> { .collect() } +/// Python's gzip on a stream cut short (an `EOFError`). +const GZ_EOF: &str = "Compressed file ended before the end-of-stream marker was reached"; + /// gzip's own words for a stream cut short; the reader's otherwise. fn gz_text(e: &std::io::Error) -> String { if e.kind() == std::io::ErrorKind::UnexpectedEof { - "Compressed file ended before the end-of-stream marker was reached".to_owned() + GZ_EOF.to_owned() } else { e.to_string() } } -/// The data lines of a VCF split at tabs (`vcf_records`). -pub fn vcf_records(path: &str) -> Result>, ReadError> { - Ok(lines(path)? - .into_iter() - .filter(|l| !l.starts_with('#')) - .map(|l| l.split('\t').map(str::to_owned).collect()) - .collect()) +/// Calls `f` with each line of a text file in turn (as [`lines`] reads +/// them), never holding the whole file. +pub fn each_line(path: &str, mut f: impl FnMut(&str)) -> Result<(), ReadError> { + let it = og_io::Lines::new( + og_io::open(path).map_err(|e| ReadError::Os(os_error(&e, path)))?, + og_io::Utf8::Replace, + ); + for l in it { + f(&l.map_err(|e| ReadError::Os(gz_text(&e)))?); + } + Ok(()) } /// The rows of a tab-separated table with a header (`csv.DictReader`). @@ -90,6 +99,9 @@ pub fn rows(path: &str) -> Result, ReadError> { match lines.read_line(&mut s) { Ok(true) => Some(Ok(s)), Ok(false) => None, + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + Some(Err(std::io::Error::new(e.kind(), gz_text(&e)))) + } Err(e) => Some(Err(e)), } }); @@ -193,3 +205,22 @@ pub fn relpath(base: &str, path: &str) -> String { pub fn dirname(p: &str) -> &str { p.rfind('/').map_or("", |i| &p[..i]) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + #[test] + fn a_table_cut_short_is_python_s_eof_error() { + let d = Dir::new("text-cut"); + let body = "A\tB\n".to_owned() + &"1\t2\n".repeat(2000); + let full = std::fs::read(d.put_gz("t.tsv.gz", &body)).unwrap(); + for n in [30, full.len() / 2, full.len() - 4] { + let p = d.put("cut.tsv.gz", &full[..n]); + let e = rows(&p).unwrap_err(); + assert_eq!(e.unreadable(&p), format!("cut.tsv.gz: {GZ_EOF}"), "{n}"); + assert_eq!(e.note(), format!("EOFError: {GZ_EOF}"), "{n}"); + } + } +} From 391581c8be60770e4d0a42fd6a7a2d4dab8459b4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 07/24] fix(summary): word a broken PharmCAT report as Python's json does The note now says "line L column C (char N)" in code points after universal newlines, and names a UTF-8 BOM, as json.load does. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/pgx.rs | 50 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs index 458660493..d59db94c2 100644 --- a/rust/crates/og-summary/src/collect/pgx.rs +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -50,14 +50,37 @@ pub struct PharmCat { pub html_report: bool, } +/// `json.load` of a file opened as text: universal newlines, and Python's +/// `JSONDecodeError` text (`MSG: line L column C (char N)`, in code points). +fn json_load(raw: &str) -> Result { + let text = raw.replace("\r\n", "\n").replace('\r', "\n"); + if text.starts_with('\u{feff}') { + return Err( + "Unexpected UTF-8 BOM (decode using utf-8-sig): line 1 column 1 (char 0)".into(), + ); + } + pycompat::json::loads(&text).map_err(|e| match e { + pycompat::json::JsonError::Decode(what, at) => { + let before = &text[..at.min(text.len())]; + let pos = before.chars().count(); + let line = before.matches('\n').count() + 1; + let col = match before.rfind('\n') { + Some(i) => before[i + 1..].chars().count() + 1, + None => pos + 1, + }; + format!("{what}: line {line} column {col} (char {pos})") + } + other => other.to_string(), + }) +} + impl PharmCat { pub fn read(path: &str) -> Result { let raw = std::fs::read(path) .map_err(|e| format!("{}: {}", crate::basename(path), crate::os_error(&e, path)))?; let body = String::from_utf8(raw) .map_err(|e| format!("{}: 'utf-8' codec can't decode: {e}", crate::basename(path)))?; - let data = - pycompat::json::loads(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; + let data = json_load(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; let version = match &data { Value::Object(m) => match m.get("pharmcatVersion") { None => String::new(), @@ -616,3 +639,26 @@ impl Cyp2d6 { ]) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_errors_read_as_python_words_them() { + assert_eq!( + json_load("{\"pharmcatVersion\": ").unwrap_err(), + "Expecting value: line 1 column 21 (char 20)" + ); + assert_eq!( + json_load("{\r\n\"é\": x").unwrap_err(), + "Expecting value: line 2 column 6 (char 7)" + ); + assert!( + json_load("\u{feff}{}") + .unwrap_err() + .starts_with("Unexpected UTF-8 BOM") + ); + assert!(json_load("[1]").is_ok()); + } +} From 4c06a83e5bd339e2c41e14589954ba715f261415 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 08/24] test(parity): the clinical sections of summary collect against the Python Ten cases of og summary collect --only b: every input in both layouts, failed and stale steps, unreadable inputs, a VEP output with and without its index, phased deletions in trans and in cis, and no assets. The oracle stages the helper modules the collector imports from the pinned commit instead of stubs. cb-notgz holds the one deliberate fix. Bead: pgp-bs9u.42 --- tests/parity/cases.tsv | 10 + tests/parity/gen/summary-clinical.sh | 247 ++++++++++++++++++ .../summary-collect/cb-notgz/cb-notgz.json | 180 +++++++++++++ .../inputs/summary-collect/cb_vep.vcf.gz | Bin 0 -> 727 bytes .../inputs/summary-collect/cb_vep.vcf.gz.tbi | Bin 0 -> 253 bytes tests/parity/inputs/summary-collect/oracle.sh | 11 +- 6 files changed, 444 insertions(+), 4 deletions(-) create mode 100644 tests/parity/gen/summary-clinical.sh create mode 100644 tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json create mode 100644 tests/parity/inputs/summary-collect/cb_vep.vcf.gz create mode 100644 tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 0a9afd6bf..03bf5e088 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -838,6 +838,16 @@ summary-collect empty gen:summary-collect.sh,summary-collect/oracle.sh --sample summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir declared-status --assets assets --declared-sex female --out declared-arg.json --only a declared-arg.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE +summary-collect cb-full gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets assets --out cb-full.json --only b cb-full.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-nf gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nf --assets assets --out cb-nf.json --only b cb-nf.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-probable gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-probable --assets assets --out cb-probable.json --only b cb-probable.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-failed gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-failed --assets assets --out cb-failed.json --only b cb-failed.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-bad gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-bad --assets assets --out cb-bad.json --only b cb-bad.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-nfbad gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nfbad --assets assets --out cb-nfbad.json --only b cb-nfbad.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-notbi gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notbi --assets assets --out cb-notbi.json --only b cb-notbi.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-notgz gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notgz --assets assets --out cb-notgz.json --only b cb-notgz.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-empty gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-empty --assets assets --out cb-empty.json --only b cb-empty.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-no-assets gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets nowhere --out cb-no-assets.json --only b cb-no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh new file mode 100644 index 000000000..aa1191f44 --- /dev/null +++ b/tests/parity/gen/summary-clinical.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# summary-clinical.sh: writes the sample folders of the summary-collect +# parity cases of group b (og summary collect --only b) in the current +# directory, the same every time. The clinical sections: ClinVar hits with +# each review status, a call floor, combined alleles, half calls and +# phased calls, the cDNA check, scan and deep intronic tables, gnomAD AF +# from an indexed VEP output; the carrier list; PharmCAT, CPIC, pypgx and +# Cyrius and step 36's CYP2D6 row; CPSR with its own class beside +# ClinVar's; the clinical filter's table and VCF with the ACMG SF tier; +# slivar's table and VCF with the phased compound-het pairs; and the +# deletions the two-hit rule pairs with a variant (AnnotSV, step 35's +# interval, HBA and SMN1 tables), in trans, in cis and inside the deletion. +# +# The VEP output is the committed pair cb_vep.vcf.gz and its .tbi, staged +# beside the folders; vep/S_vep.vcf.gz links to it. assets/ is a copy of +# the repository's gene lists. pymods/ holds the helper modules the Python +# collector imports, at a pinned commit, so the oracle reads them instead +# of stubs. scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) + +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; } +putgz() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" | gzip -n -c > "$1"; } +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC + +T0=1577836800 # 2020-01-01 00:00:00 UTC, the start of the run +OLD=201906011230.00 # before the run +NEW=202103040506.07 # after it +NEWER=202103040507.00 + +# The helper modules of bin/collect_summary.py, as they were when the +# collector was last the summary's writer. +HELPERS=7c1470bc50933f48a1a0dde3c0e0b2b87bc48718 +git -C "$root" cat-file -e "${HELPERS}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$HELPERS" +mkdir -p pymods +for m in pgx_outside_calls clip_check annotsv_postfilter paraphase_report traits_report; do + git -C "$root" show "${HELPERS}:bin/${m}.py" > "pymods/${m}.py" +done + +mkdir -p assets +for a in acmg_sf.tsv carrier_genes.tsv paralog_loci.tsv; do + cp "${root}/assets/${a}" assets/ +done + +VCF_HEAD='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' +PL='criteria_provided,_multiple_submitters,_no_conflicts' + +# --- the ClinVar hits of step 06 --------------------------------------------------- +HITS="##fileformat=VCFv4.2\n##PGP_CALL_FLOOR=GQ=20,DP=8\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" +HITS+="chr1\t45331556\t1\tC\tT\t.\tPASS\tGENEINFO=MUTYH:4595;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP:PS\t0|1:50:30:45331000\n" +HITS+="chr6\t26092913\t2\tG\tA\t.\tPASS\tGENEINFO=HFE:3077;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT:GQ:DP\t0/1:99:40\n" +HITS+="chr7\t117559590\t3\tATCT\tA\t.\tPASS\tGENEINFO=CFTR:1080;CLNSIG=Pathogenic;CLNREVSTAT=practice_guideline\tGT:GQ:DP\t0/1:15:30\n" +HITS+="chr13\t20189473\t4\tC\tT\t.\tPASS\tGENEINFO=GJB2:2706;CLNSIG=Likely_pathogenic;CLNREVSTAT=criteria_provided,_single_submitter;PGP_JOINED_FROM=chr13:20189473:C:T|chr13:20189475:G:A\tGT:GQ:DP\t1/.:40:5\n" +HITS+="chr3\t15645186\t5\tG\tC\t.\tPASS\tGENEINFO=BTD:686;CLNSIG=Pathogenic/Likely_pathogenic;CLNREVSTAT=criteria_provided,_conflicting_classifications\tGT\t0/1\n" +HITS+="chr17\t43057100\t6\tC\tT\t.\tPASS\tGENEINFO=BRCA1:672|NBR2:10230;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel;PGP_PART_OF=chr17:43057100:C:T|chr17:43057102:A:G\tGT:GQ:DP\t0/1:60:35\n" +HITS+="chrX\t101398000\t7\tC\tT\t.\tPASS\tGENEINFO=GLA:2717;CLNSIG=Pathogenic;CLNREVSTAT=no_assertion_criteria_provided;PGP_ONE_ALLELE=chrX:101398004:G:A\tGT\t1\n" +HITS+="chr16\t176700\t8\tG\tA\t.\tPASS\tGENEINFO=HBA1:3039;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP\t0/1:45:31\n" +HITS+="chr5\t70951946\t9\tC\tT\t.\tPASS\tGENEINFO=SMN1:6606;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP\t1/1:45:31\n" +HITS+="chr2\t100\t10\tA\tG\t.\tPASS\tCLNREVSTAT=;CLNSIG=\tGT:DP\t0/1:x\n" +HITS+="chr2\t200\t11\tA\tG\t.\tPASS\n" + +CDNA='chrom\tpos\tref\talt\talt_reads\talt_clipped\tref_reads\tref_clipped\tcdna_check\tcdna_note\tconstruct_site\n' +CDNA+='chr1\t45331556\tC\tT\t12\t6\t10\t0\tflagged\tpossible cDNA/transgene contamination: check the reads\tyes\n' +CDNA+='chr6\t26092913\tG\tA\t20\t0\t20\t0\tclean\t\t\n' +CDNA+='chr6\tx\tG\tA\t1\t1\t1\t1\tclean\t\t\n' +SCAN='# cdna_scan: 2 genes with spliced reads\nchrom\tstart\tend\tjunctions\treads\tjunction_list\nchr1\t100\t200\t2\t5\tA:1-2,A:3-4\nchr2\tx\t1\t1\t1\t\n' +DEEP='chrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\tcall\tdetail\nchr7\t117639961\tC\tT\tCFTR\tc.3718-2477C>T\t7129\t\t\nchr7\tNA\tC\tT\tCFTR\t\t\t\t\n' + +# --- pharmacogenomics ---------------------------------------------------------------- +CPIC='Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*1/*4\tIntermediate Metabolizer\tnon-normal\nCYP2C19\t*1/*17\tRapid\tambiguous\nCYP2C9\t*1/*1\tNormal\tnormal\nDPYD\n' +REC='CPIC lookup\n WARNING: CYP2C19 has two possible phenotypes\nnothing\nWARNING:last\n' +PYPGX='Gene\tDiplotype\tPhenotype\tStatus\tSource\tCNV_call\tQuality\tMin_GQ\tWeakest_variant\tDefining_filter\n' +PYPGX+='CYP2D6\t*1/*2\t Normal Metabolizer \tok\tBAM\tNormal\tok\t\t\tPASS\n' +PYPGX+='CYP2A6\tFAILED\t\tfailed\tBAM\t\t\t\t\t\n' +PYPGX+='CYP2B6\t*1/*6\tIM\tdepth check failed\tbam\tN/A\tok\t\t\t\n' +PYPGX+='CYP2E1\t.\t\tnot callable on this reference\tBAM\t.\t\t\t\t\n' +PYPGX+='UGT1A1\tIndeterminate\t\tok\tVCF\t\tlow-confidence: check the reads\t 12 \tchr2:1:A:G\t\n' +PYPGX+='SLCO1B1\t*1/*5\t\tok\tBAM\t.\tlow-confidence: check the reads\tx\t\t\n' +DEPTH='metric\tvalue\nstatus\tok\ngene_mapq1_share\t0.99\nbody_mapq20_share\t0.95\nother\t1\n' +COMP='Gene\tMatch\nCYP2D6\tYes\nCYP2C19\tNo\nCYP3A5\tpypgx only\nTPMT\tPharmCAT only\n' +CONS='Gene\tResult\tOutside_call\tReason\tEvidence\nHLA-A\tx\tno\t\t\nCYP2D6\t*1/*2\tyes\tpypgx and Cyrius agree\tpypgx *1/*2; Cyrius *2/*1\n' + +# --- CPSR (step 17) -------------------------------------------------------------------- +CPSR='VAR_ID\tGENOMIC_CHANGE\tSYMBOL\tCLASSIFICATION\tCPSR_CLASSIFICATION\tCLINVAR_CLASSIFICATION\tASSERTION_AUTHORITY\tACMG_CODE\tDP_CONTROL\n' +CPSR+='17_43057100_C_T\t17:g.43057100C>T\tBRCA1\tPathogenic\tLikely_Benign\tPathogenic\tClinVar\tPVS1\t6\n' +CPSR+='\t13:g.32338000G>A\tBRCA2\tLikely_Pathogenic\tVUS\t\tCPSR\t\t40\n' +CPSR+='2_47403000_A_G\t\tMSH2\tVUS\tLP\tB\t\t\t\n' +CPSR+='x\tbad\tATM\tBenign\tBenign\t\t\t\t3\n' +CPSR+='5_112839000_A_T\t\tAPC\t\tPathogenic\t\t\t\t30\n' +CPSR_CDNA="${CDNA}17\t43057100\tC\tT\t5\t5\t9\t0\tflagged\tsplice\t\n" + +# --- the clinical filter (step 23) ------------------------------------------------------ +CSQF='Allele|Consequence|IMPACT|SYMBOL|Feature|CANONICAL|MANE_SELECT|gnomADe_AF|gnomADg_AF' +CLIN="##fileformat=VCFv4.2\n##PGP_CALL_FLOOR=GQ=20,DP=8,HET_AB=0.25-0.75,HOM_AB=0.85\n" +CLIN+="##INFO=\n" +CLIN+='#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' +CLIN+='chr1\t45332000\t.\tG\tA\t50\tPASS\tCSQ=A|stop_gained|HIGH|MUTYH|T1|YES|NM_1|0.0001|\tGT:GQ:DP:AD:PS\t1|0:40:30:15,15:45331000\n' +CLIN+='chr2\t47414000\t.\tA\tAT\t50\tPASS\tCSQ=AT|frameshift_variant|HIGH|MSH2|T2|YES||0.00002|0.00001\tGT:GQ:DP:AD\t0/1:50:30:28,2\n' +CLIN+='chr10\t100\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|PTEN|T3|YES||0.2|\tGT\t0/1\n' +CLIN+='chr6\t26093000\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|HFE|T4|YES||&|\tGT:GQ:DP:AD\t1/1:12:5:3,4\n' +CLIN+='chr17\t43057100\t.\tC\tT\t50\tPASS\tCSQ=T|missense_variant|MODERATE|BRCA1|T5|YES|NM_2||,T|stop_gained|HIGH|BRCA1|T6|||0.0001|\tGT\t0/1\n' +CLIN+='chr11\t47332000\t.\tG\tA\t50\tPASS\tCSQ=A|splice_donor_variant&intron_variant|HIGH|MYBPC3|T7|||0.001|,A|stop_gained|HIGH|MYBPC3|T8|||0.0001|\tGT:GQ\t0/1:.\n' +CLINT='GENE\tIMPACT\tFREQ_TIER\tCALL_CHECK\nMUTYH\tHIGH\trare\tok\nMSH2\tHIGH\trare\tweak\nHFE\tHIGH\tunknown\tweak\n.\tMODERATE\trare\tok\n\t\t\t\n' + +# --- slivar (step 31) ------------------------------------------------------------------- +SLIV='CHROM\tPOS\tGENE\tFREQ_TIER\nchr1\t1\tA\trare\nchr1\t2\tA\tunknown\n' +CHETS='GENE\tCHROM\nA\tchr1\nA\tchr1\nB\tchr2\n\tchr3\n' +PAIRS='GENE\tCHROM\tPOS_1\tREF_1\tALT_1\tPOS_2\tREF_2\tALT_2\tPHASE\tTRANS_READS\tCIS_READS\n' +for i in $(seq 1 23); do PAIRS+="G${i}\tchr1\t${i}0\tA\tG\t${i}5\tC\tT\ttrans\t${i}\t1\n"; done +PAIRS+='H\tchr2\t1\tA\tG\t2\tC\tT\tcis\t0\t4\nK\tchr2\t3\tA\tG\nL\tchr2\t4\tA\tG\t5\tC\tT\tunknown\tx\t\n' + +# --- the deletions of the two-hit rule -------------------------------------------------- +ANN='SV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tGene_name\tACMG_class\tCaller\tExon_edge\tExon_edge_detail\tPopulation_AF\tPopulation_source\tCommon\tAnnotSV_class\tFORMAT\tS\tAnnotation_mode\n' +ANN+='1\t45338000\t45340000\t-2000\tDEL\tMUTYH\t5\tManta\tyes\t\t\t\tno\t5\tGT:PS\t1|0:45331000\tfull\n' +ANN+='chr2\t47386000\t47388000\t-2000\tDEL\tEPCAM\t4\tDelly\tno\t\t\t\tno\t4\tGT\t0/1\tfull\n' +ANN+='17\t43000000\t43100000\t100000\tDUP\tBRCA1;NBR2\t4\tCNVpytor\tno\t\t0.001\tgnomAD\tno\t4\tGT\t0/1\tfull\n' +ANN+='13\t20180000\t20200000\t-20000\tDEL\tGJB2;NOTAGENE\t5\tManta\tno\t\t\t\tno\t5\tGT\t0/0\tfull\n' +ANN+='13\t20180000\t20200000\t-20000\tDEL\tGJB2;XYZ1\t4\tManta\tno\t\t\t\tno\t4\tGT\t0/1\tfull\n' +ANN+='3\t15600000\t15700000\t-100000\tDEL\tBTD\t3\tManta\tno\t\t\t\tno\t5\tGT\t0/1\tfull\n' +ANN+='3\t15600000\t15700000\t-100000\tDEL\tBTD\t5\tManta\tno\t\t0.2\tgnomAD\tyes\t5\tGT\t0/1\tfull\n' +CONSV='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tmanta\tdelly\tcnvpytor\n' +CONSV+='chr1\t45338010\t.\tN\t\t.\tPASS\tSVTYPE=DEL;END=45340020;SUPP=2;SUPP_VEC=110\n' +CONSV+='chr1\t45338500\t.\tN\t\t.\tPASS\tSVTYPE=DEL;END=45340900;SUPP=3;SUPP_VEC=111\n' +CONSV+='chr17\t43000000\t.\tN\t\t.\tPASS\tSVTYPE=DUP:TANDEM;END=43100000;SUPP=1;SUPP_VEC=0011\n' +SMN='chrom\tstart\tend\tagCN\tagCN_filter\tagCN_qual\tpsCN\tpsCN_qual\tpsCN_filter\thomologous_regions\n' +SMN+='chr5\t70925000\t70953000\t4\tPASS\t40\t1,3\t30,30\tPASS\tchr5:70049000-70078000\n' +HBA='metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t3\n' +IV='locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\n' +IV+='CFTR_del\tCFTR region\tchr7\t117500000\t117600000\tcalled\t1\t2\tdeletion\n' +IV+='DUP1\tx\tchr1\t1\t100\tcalled\t3\t2\tduplication\n' +IV+='GJB6\tx\tchr13\tx\t2\tcalled\t1\t2\tdeletion\n' + +# full: every input, the bash layout, a run whose steps were all ok. +full() { + local d=$1 + put "$d/clinvar/S_clinvar_hits.vcf" "$HITS" + put "$d/clinvar/S_clinvar_cdna_check.tsv" "$CDNA" + put "$d/clinvar/S_clinvar_cdna_scan.tsv" "$SCAN" + put "$d/clinvar/S_clinvar_deep_intronic.tsv" "$DEEP" + mkdir -p "$d/vep" + ln -s ../../cb_vep.vcf.gz "$d/vep/S_vep.vcf.gz" + ln -s ../../cb_vep.vcf.gz.tbi "$d/vep/S_vep.vcf.gz.tbi" + put "$d/pharmcat/S.report.json" '{"pharmcatVersion": "3.4.0", "genes": {}}' + put "$d/pharmcat/S.report.html" '' + put "$d/cpic/S_phenotypes.tsv" "$CPIC" + put "$d/cpic/S_cpic_recommendations.txt" "$REC" + put "$d/pypgx/S_pypgx_summary.tsv" "$PYPGX" + put "$d/pypgx/S_cyp2d6_depth_check.tsv" "$DEPTH" + put "$d/pypgx/S_pharmcat_comparison.tsv" "$COMP" + put "$d/cyrius/S_cyp2d6.tsv" 'Sample\tGenotype\tFilter\nS\t*2/*1\tPASS\n' + put "$d/pgx_consensus/S_pgx_consensus.tsv" "$CONS" + put "$d/cpsr/S.cpsr.grch38.html" '' + putgz "$d/cpsr/S.cpsr.grch38.classification.tsv.gz" "$CPSR" + put "$d/cpsr/S_cpsr_cdna_check.tsv" "$CPSR_CDNA" + put "$d/clinical/S_clinical_summary.tsv" "$CLINT" + putgz "$d/clinical/S_clinical.vcf.gz" "$CLIN" + put "$d/slivar/S_slivar_summary.tsv" "$SLIV" + put "$d/slivar/S_compound_hets.tsv" "$CHETS" + put "$d/slivar/S_compound_het_pairs.tsv" "$PAIRS" + put "$d/annotsv/S_sv_class4_5_checked.tsv" "$ANN" + putgz "$d/sv_merged/S_sv_consensus.vcf.gz" "$CONSV" + put "$d/paralogs/S_smn_copy_number.tsv" "$SMN" + put "$d/paralogs/S_hba_copy_number.tsv" "$HBA" + put "$d/paralogs/S_cn_intervals.tsv" "$IV" + put "$d/logs/run_status.tsv" "meta\tstarted_epoch\t${T0}\nstep\t06\tok\nstep\t07\tok\nstep\t17\tok\nstep\t21\tok\nstep\t23\tok\nstep\t27\tok\nstep\t31\tok\nstep\t32\tok\n" +} +full cb-full + +# nf: the Nextflow names. The clinical VCF and slivar's VCF without their +# tables, two PharmCAT reports (the newer wins), Cyrius under another name, +# CPSR by its wildcard names, no VEP output, a consensus older than pypgx, +# PharmCAT's step failed, CPIC from an earlier run, pypgx failed. +put cb-nf/clinvar/S_clinvar_hits.vcf "$HITS" +putgz cb-nf/clinical/S_clinical.vcf.gz "$CLIN" +putgz cb-nf/slivar/S_prioritized.vcf.gz "${VCF_HEAD}chr1\t1\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\nchr1\t2\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\n" +put cb-nf/slivar/S_compound_het_pairs.tsv "$PAIRS" +put cb-nf/vcf/A.report.json '{"pharmcatVersion": 3.2}' +put cb-nf/vcf/B.report.json '[1, 2]' +put cb-nf/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*4/*4\tPoor\tnot called\n' +put cb-nf/cyrius/T_cyp2d6.tsv 'Sample\tGenotype\tFilter\nT\t*1/*2\tLowQ\n' +put cb-nf/pypgx/S_pypgx_summary.tsv "$PYPGX" +put cb-nf/pgx_consensus/S_pgx_consensus.tsv "$CONS" +put cb-nf/cpsr/T.cpsr.grch37.html '' +putgz cb-nf/cpsr/T.cpsr.grch37.classification.tsv.gz 'SYMBOL\tFINAL_CLASSIFICATION\tCPSR_CLASSIFICATION\nA\tPathogenic\tBenign\nB\t\t\n' +put cb-nf/annotsv/S_sv_class4_5_checked.tsv '' +put cb-nf/paralogs/S_cn_intervals.tsv "$IV" +put cb-nf/logs/run_status.tsv "meta\tstarted_epoch\t${T0}\nstep\t07\tfailed (exit 1)\nstep\t27\tskipped\nstep\t32\tok\nstep\t21\tok\n" + +# probable: pypgx alone, Cyrius not usable, the depth check ok; no consensus. +put cb-probable/pypgx/S_pypgx_summary.tsv "$PYPGX" +put cb-probable/pypgx/S_cyp2d6_depth_check.tsv "$DEPTH" +put cb-probable/cyrius/S_cyp2d6.tsv 'Sample\tGenotype\tFilter\nS\t*1/*3\tLowQ\n' +put cb-probable/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2C9\t*1/*1\tNormal\tnormal\n' +put cb-probable/cpic/S_cpic_recommendations.txt 'PARSING FAILED for one gene\n' + +# failed: CPIC, pypgx and Cyrius failed in the run and wrote nothing. +put cb-failed/logs/run_status.tsv "meta\tstarted_utc\t2020-02-02 02:02 UTC\nstep\t21\tfailed (exit 1)\nstep\t27\tfailed\nstep\t32\tfailed (exit 3)\nstep\t06\tok\n" +put cb-failed/pgx_consensus/S_pgx_consensus.tsv 'Gene\tResult\nCYP2C19\tx\n' + +# bad: a hit whose POS is no number, a report that is no JSON, a CPSR +# table cut short, pypgx without a Gene column, an AnnotSV table without +# its columns, a Parascopy table without its columns, a clinical floor that +# is no number, slivar's VCF cut short. +put cb-bad/clinvar/S_clinvar_hits.vcf "${VCF_HEAD}chr1\tx\t.\tA\tG\t.\tPASS\tGENEINFO=MUTYH:1;CLNSIG=Pathogenic\tGT\t0/1\n" +put cb-bad/pharmcat/S.report.json '{"pharmcatVersion": ' +putgz cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz "$CPSR" +head -c 60 cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz > cb-bad.tmp && mv cb-bad.tmp cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz +put cb-bad/pypgx/S_pypgx_summary.tsv 'Name\tSource\tCNV_call\nCYP2D6\tBAM\tNormal\n' +put cb-bad/annotsv/S_sv_class4_5_checked.tsv 'SV_chrom\tSV_start\n1\t1\n' +put cb-bad/paralogs/S_smn_copy_number.tsv 'chrom\tstart\tend\nchr5\t1\t2\n' +putgz cb-bad/clinical/S_clinical.vcf.gz "##PGP_CALL_FLOOR=GQ=20,HOM_AB=0..8\n#CHROM\n" +put cb-bad/clinical/S_clinical_summary.tsv 'GENE\tIMPACT\nA\tHIGH\n' +put cb-bad/cyrius/S_cyp2d6.tsv '' +put cb-bad/cpic/S_phenotypes.tsv 'Gene\tDiplotype\n' +putgz cb-bad/slivar/S_prioritized.vcf.gz "$CLIN" +head -c 100 cb-bad/slivar/S_prioritized.vcf.gz > cb-bad.tmp && mv cb-bad.tmp cb-bad/slivar/S_prioritized.vcf.gz + +# nf-bad: the Nextflow clinical VCF cut short; Parascopy's catalog tables +# without their catalog; an HBA table without a status. +put cb-nfbad/clinvar/S_clinvar_hits.vcf "$HITS" +putgz cb-nfbad/clinical/S_clinical.vcf.gz "$CLIN" +head -c 100 cb-nfbad/clinical/S_clinical.vcf.gz > cb-nfbad.tmp && mv cb-nfbad.tmp cb-nfbad/clinical/S_clinical.vcf.gz +put cb-nfbad/paralogs/S_hba_copy_number.tsv 'metric\tvalue\nalpha_globin_genes\t2\n' +put cb-nfbad/paralogs/S_rhd_copy_number.tsv "$SMN" + +# notbi: the VEP output without its index; the HBA count of no gene left. +put cb-notbi/clinvar/S_clinvar_hits.vcf "$HITS" +mkdir -p cb-notbi/vep +ln -s ../../cb_vep.vcf.gz cb-notbi/vep/S_vep.vcf.gz +put cb-notbi/paralogs/S_hba_copy_number.tsv 'metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t0\n' + +# notgz: the clinical and slivar VCFs as plain text under a .gz name. The +# Python tells gzip by the name and finds both unreadable; og tells it by +# the first bytes and reads them (a golden). +put cb-notgz/clinical/S_clinical.vcf.gz "$CLIN" +put cb-notgz/slivar/S_prioritized.vcf.gz "${VCF_HEAD}chr1\t1\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\n" + +mkdir -p cb-empty + +# Every file after the run started; the CPIC table of nf from before it; +# nf's second PharmCAT report and its consensus' sources newer. +find . -type f -exec touch -t "$NEW" {} + +at "$OLD" cb-nf/cpic/S_phenotypes.tsv cb-nf/pgx_consensus/S_pgx_consensus.tsv +at "$NEWER" cb-nf/vcf/B.report.json diff --git a/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json b/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json new file mode 100644 index 000000000..e972fd826 --- /dev/null +++ b/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json @@ -0,0 +1,180 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 11:46 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 6, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [], + "clinvar_checked": false, + "high_impact": [ + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 1, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": false, + "structural_note": null, + "not_reported": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "source": "HIGH impact", + "reason": "one heterozygous variant; MUTYH is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + } + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_prioritized.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 1 + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + } +} diff --git a/tests/parity/inputs/summary-collect/cb_vep.vcf.gz b/tests/parity/inputs/summary-collect/cb_vep.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..4a42ab23694fa66e2234dfd493e5ff6a313c223f GIT binary patch literal 727 zcmb2|=3rp}f&Xj_PR>jWMU0^n?q)SR2(+$0zFeR+hV`NZx8~LjU)2SZOm=ZT(V9_` zEMDRpE*v$Zef_e=W#)&^=scg3A9me;PttO?Yp&1p_I|K3RqI-p-+bljx@k`{{tB!; zbfmVk=KAUDkFPGZGk7VL{Hr!C@1CdLsg37vEq?O6&-~EUJ#Jgxd_Ht!;>uP5-*BGa zVYVlig*E?MYZsCwpRnSg?zc$Q(4$&y{7yZOrc{W0j{P%DXQPO(Z_PIKhkJF~&3-(} zy?*@t`qn4g1>S~U-0M2+$OJV@7rPWkjug+ty^7u(i&c4zB+fM&oJ?|34DXY_)U-1B z_O!<~jf`>u1wJ00^PKC~dvAO@>+?RH9SQGubUfqkF)!KfaM|GO$+ffiPF<@rlC3t| z>>f8Mf32abp4H8cc(aR%bAAL$u-=UAvY){FYuor1E&5yoTe`#^{ z1VO1gv95}4%oAHJW<+IZ*?-82>Uqe>z#xwnLE0~tcy=w29)EVmUd%rHL z{+9RZt3p1AJ9oneFbW9_xqF~bTvs0^X++XZb#Y85o#t+hDToN};+_dt;-bN5A6oYL4X#_X&m@ pEpwI$ob%A*MnLd{&Swh3ykB^CGTf+pGw(JQS4lH4gAy_r001f^JyQSx literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi b/tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi new file mode 100644 index 0000000000000000000000000000000000000000..1036d9b09dfc8623d118587ec9212741f883e91b GIT binary patch literal 253 zcmb2|=3oE;rvGp67;+tQ5MaF!%c=COtn3k^OF+)sH+vg3*PK;-d*s%s*)PET-bKifzuy@WGUt}pHRfL!g_g4i^_I~za ecTAsGt;;>_Vz@gV1mBI_B3_wuHv3>dBLe_=G*kHi literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 09d629c43..1a3e15a7c 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -5,13 +5,16 @@ # --assets becomes ASSETS_DIR, the global render_report.py --assets sets; # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are -# stubbed: none is reached for a sample folder that holds only the -# sections the cases cover, except run_delta's two readers of history/ and -# recheck/, which find nothing here. +# stubbed, except those a generator staged in pymods/ (the clinical cases +# need the real helpers); no other one is reached for a sample folder that +# holds only the sections the cases cover, except run_delta's two readers +# of history/ and recheck/, which find nothing here. prog=$1 shift d=$(mktemp -d) -sed -n 's/^import \([a-z_]*\) # noqa.*/\1/p' "$prog" | while read -r m; do : > "$d/$m.py"; done +sed -n 's/^import \([a-z_]*\) # noqa.*/\1/p' "$prog" | while read -r m; do + if [ -f "pymods/$m.py" ]; then cp "pymods/$m.py" "$d/"; else : > "$d/$m.py"; fi +done printf 'def read_recheck(path):\n return None\n\n\ndef newest_history(sample_dir):\n return None\n' > "$d/run_delta.py" cp "$prog" "$d/collect_summary.py" PYTHONPATH=$d exec python3 -c ' From f3177f5cbd50121ef60c5d1a44303481d6cb629e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 09/24] docs(summary): og summary collect --only b in the changelog and the crate map Bead: pgp-bs9u.42 --- CHANGELOG.md | 4 ++++ docs/rust.md | 3 ++- rust/README.md | 2 +- rust/og/src/main.rs | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e21d1ac8..f31e9387e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 9 parity cases, compared as JSON with key order; a tenth holds the deliberate fix below. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: + - A clinical or slivar VCF named `.vcf.gz` that is plain text is read. The Python marked the section unreadable. + - A PharmCAT report that is not UTF-8 is still unreadable, but the note names the fault in Rust's words. + - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. - A sample name with `*`, `?` or `[` is matched as written. The Python read it as a file pattern. diff --git a/docs/rust.md b/docs/rust.md index cb6924559..ab177fdee 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -22,7 +22,8 @@ rust/ crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; - og summary collect --only a, the first seven summary sections (no caller yet) + og summary collect --only a, the first seven summary sections (no caller yet); + --only b, the clinical sections: ClinVar, carrier, PGx, CPSR, ACMG SF, two hits crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 394e83e10..c42b40b64 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the first sections of summary.json, `og summary collect --only a`, beside the Python collector, which still writes it | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC) and `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar), beside the Python collector, which still writes it | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 234ea7cac..119fd5441 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,7 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only a [args] + og summary collect --only a|b [args] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] og bloodtype report|check-markers|systems [args] From 8fb444c6c5671b44f3049406c2689da548a0e877 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:12 +0100 Subject: [PATCH 10/24] test(summary): red control acmg-biallelic-ignores-phase (reverted next) Plant: the deletion's read-backed phase is ignored, so a variant in cis with a deletion counts as a second hit and a trans pair is only possibly biallelic. The two_hits unit tests and the cb-full parity case must fail. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/two_hits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index 52d2184b8..d9c9c475d 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -143,7 +143,7 @@ pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; // Read-backed phase first: on the same copy (cis) is no second hit, // wherever the variant sits. - let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); + let hd = phase_haps(dl.gt.as_deref()).filter(|h| h.is_empty()); let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); let mut phased: Option<(Verdict, String)> = None; if let (Some(hd), Some(hv)) = (&hd, &hv) From 1b8998fff3c1710a5194bf6b8bbee9e58fc0113b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:59:37 +0100 Subject: [PATCH 11/24] Revert "test(dev): plant three wrong builder rules to show the tests go red" The plant turned five builder tests red in Guard's rust job (run 43417543846); back to the right rules. Bead: pgp-bs9u.35 --- rust/crates/og-dev/src/build/callability.rs | 3 +-- rust/crates/og-dev/src/build/cn_intervals.rs | 3 +-- rust/crates/og-dev/src/build/sv_rules.rs | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs index 556c65865..d7ce4dece 100644 --- a/rust/crates/og-dev/src/build/callability.rs +++ b/rust/crates/og-dev/src/build/callability.rs @@ -111,8 +111,7 @@ fn read_gtf(path: &Path, alt: &HashMap) -> Result Vec { // Ceiling divisions of positive numbers. let n = (len + PIECE - 1) / PIECE; let step = (len + n - 1) / n; - // PLANTED, reverted in the next commit: each piece ends one base short. - out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step) - 1))); + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); } out } diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs index 1007fc81b..7004f02b7 100644 --- a/rust/crates/og-dev/src/build/sv_rules.rs +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -99,10 +99,9 @@ fn read_gtf(path: &Path) -> Result { b"gene" => { let (start, end) = r.span()?; if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) - && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_X")) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) { - // PLANTED, reverted in the next commit: the last record wins. - g.span.insert(h.to_owned(), (chrom, start, end)); + g.span.entry(h.to_owned()).or_insert((chrom, start, end)); } let (name, ty) = ( r.attr("gene_name")?.unwrap_or(""), From 31fda578e24b39faee5d4ea6e9f589c25b9e0361 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:59:51 +0100 Subject: [PATCH 12/24] feat(summary): finish group c readers, og roh summary and their parity cases Adds the table, chrM and traits readers (APOL1 G2 by haplotype, read through the .tbi when there is one), og roh summary writing stdout without --out, 12 summary-collect c cases and 3 roh-summary cases, the c-bad2 golden for a plain-text chrM .vcf.gz, and the changelog. Bead: pgp-bs9u.43 --- CHANGELOG.md | 5 + docs/rust.md | 3 +- rust/README.md | 2 +- rust/crates/og-summary/src/collect.rs | 11 +- rust/crates/og-summary/src/collect/group_c.rs | 16 +- .../og-summary/src/collect/group_c/chrm.rs | 17 +- .../og-summary/src/collect/group_c/cn.rs | 69 ++- .../og-summary/src/collect/group_c/hla.rs | 10 +- .../og-summary/src/collect/group_c/repeats.rs | 83 ++-- .../og-summary/src/collect/group_c/roh.rs | 109 ++++- .../og-summary/src/collect/group_c/sv.rs | 63 +-- .../og-summary/src/collect/group_c/tables.rs | 92 ++-- .../og-summary/src/collect/group_c/tests.rs | 152 +++++++ .../og-summary/src/collect/group_c/traits.rs | 18 +- .../og-summary/src/collect/group_c/util.rs | 28 +- rust/crates/og-summary/src/collect/scan.rs | 6 +- rust/crates/og-summary/src/collect/tests.rs | 1 + rust/og/src/main.rs | 3 +- scripts/11-roh-analysis.sh | 3 +- tests/parity/cases.tsv | 15 + tests/parity/gen/roh-summary.sh | 38 ++ tests/parity/gen/summary-collect-c.sh | 360 +++++++++++++++ .../golden/summary-collect/c-bad2/bad2.json | 416 ++++++++++++++++++ tests/parity/inputs/roh-summary/oracle.sh | 21 + tests/parity/inputs/summary-collect/oracle.sh | 8 +- tests/parity/oracle.ref | 1 + 26 files changed, 1384 insertions(+), 166 deletions(-) create mode 100755 tests/parity/gen/roh-summary.sh create mode 100644 tests/parity/gen/summary-collect-c.sh create mode 100644 tests/parity/golden/summary-collect/c-bad2/bad2.json create mode 100644 tests/parity/inputs/roh-summary/oracle.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e21d1ac8..dc8986c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary now reads the rest of the summary's own sections: `og summary collect --only c`. It reads 32 sections of summary.json (couple, parascopy, paralogs, hla, prs, ancestry_fine, manta, delly, cnvpytor, sv_consensus, annotsv, expansions and their readings, telomere, roh, haplogroup, y_haplogroup, y_tree, y_str, consumer_export, bloodtype, lpa, vntyper, locityper, mito, traits with the APOL1 G2 call, archaic, ancient, viral, exomiser, chip, mei and mosaic). Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 12 new parity cases, compared as JSON with key order. The readers run on a thread pool, one per section. On a synthetic sample folder with every section and a 1,000,000-record VCF it takes 0.33 s and 34 MB, where the Python readers took 0.33 s and 41 to 59 MB. What differs from the Python: + - A chrM file named `.vcf.gz` that is plain text is read, as the variants section already does. The Python marked the mito section unreadable. + - The APOL1 G2 deletion is read through the VCF's `.tbi` when there is one, so only its window of chr22 is read. +- Step 11's ROH summary table now comes from `og roh summary`, in the bash step and in the Nextflow ROH module, instead of an inline awk program. The table is the same, byte for byte, on 3 parity cases. The masked bands (centromeres and other blocks where ROH calls are not trusted) are one list in the Rust code, which `og roh summary` and the summary reader share. The bash step and the module no longer keep their own copy; the Python collector keeps its copy until it is retired, and a test checks that the two lists are the same. `og roh summary` writes the table to stdout without `--out`, so the bash step writes it on the host. + - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. - A sample name with `*`, `?` or `[` is matched as written. The Python read it as a file pattern. diff --git a/docs/rust.md b/docs/rust.md index cb6924559..7bf19f298 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -22,7 +22,8 @@ rust/ crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; - og summary collect --only a, the first seven summary sections (no caller yet) + og summary collect --only a|c, the summary's own sections (no caller yet); + og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 394e83e10..e96aa161b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the first sections of summary.json, `og summary collect --only a`, beside the Python collector, which still writes it | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the summary's own sections, `og summary collect --only a` and `--only c`, beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index d85a9e5c2..de516fdfa 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -58,6 +58,8 @@ pub struct Input { pub assets: String, /// The group of sections to read: `a` or `c`. pub group: Group, + /// The folder of the two tables group c reads from `bin/`. + pub tables: String, } /// The group of sections a collect reads. @@ -84,6 +86,7 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { sample_dir: a.text("--sample-dir").to_owned(), declared_sex: a.get("--declared-sex").map(str::to_owned), assets: a.get("--assets").map_or_else(default_assets, str::to_owned), + tables: group_c::util::default_tables(), group: if a.text("--only") == "c" { Group::C } else { @@ -156,7 +159,12 @@ pub fn collect(input: &Input) -> Result { .declared_sex() .map(str::to_owned) }); - let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared.as_deref()); + let declared_sex = declared( + input, + run.as_ref(), + manifest.as_ref(), + qc_declared.as_deref(), + ); let sections = read_group_c(input, &index, run.as_ref(), declared_sex.as_deref()); return Ok(Summary { sample: input.sample.clone(), @@ -261,6 +269,7 @@ fn read_group_c( sample: &input.sample, dir: &input.sample_dir, assets: &input.assets, + tables: &input.tables, index, }; let reads = group_c::read_all(&c); diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 75ef612d8..15ab27782 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -95,6 +95,8 @@ pub struct Ctx<'a> { pub sample: &'a str, pub dir: &'a str, pub assets: &'a str, + /// The folder of `hla_alleles.tsv` and `str_intermediate_ranges.tsv`. + pub tables: &'a str, pub index: &'a FolderIndex, } @@ -132,6 +134,11 @@ impl Ctx<'_> { }) } + /// A table of the `bin/` folder ([`util::default_tables`]). + pub fn table(&self, name: &str) -> String { + crate::join(self.tables, name) + } + /// `assets/`. pub fn asset(&self, name: &str) -> String { crate::join(self.assets, name) @@ -197,8 +204,9 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { // others. A panic stays in its section. let jobs = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); let next = std::sync::atomic::AtomicUsize::new(0); - let slots: Vec>> = - (0..readers.len()).map(|_| std::sync::Mutex::new(None)).collect(); + let slots: Vec>> = (0..readers.len()) + .map(|_| std::sync::Mutex::new(None)) + .collect(); std::thread::scope(|scope| { for _ in 0..jobs { scope.spawn(|| { @@ -240,9 +248,7 @@ pub fn post( .find(|(s, ..)| s.key == key) .map_or(State::Missing, |x| x.1) }; - let traits_state = others - .traits - .unwrap_or_else(|| state("traits", sections)); + let traits_state = others.traits.unwrap_or_else(|| state("traits", sections)); let cpic_state = others.cpic.unwrap_or(State::Ok); let hla_state = state("hla", sections); let hla_data = sections diff --git a/rust/crates/og-summary/src/collect/group_c/chrm.rs b/rust/crates/og-summary/src/collect/group_c/chrm.rs index 4954a3302..e232bccf6 100644 --- a/rust/crates/og-summary/src/collect/group_c/chrm.rs +++ b/rust/crates/og-summary/src/collect/group_c/chrm.rs @@ -85,11 +85,7 @@ pub fn min_allele(mut pos: i128, r: &str, a: &str) -> (i128, String, String) { i += 1; pos += 1; } - ( - pos, - r[i..].iter().collect(), - a[i..].iter().collect(), - ) + (pos, r[i..].iter().collect(), a[i..].iter().collect()) } fn py_int(x: Option<&str>) -> Result { @@ -367,7 +363,11 @@ pub fn mito(c: &Ctx<'_>) -> Out { passed += 1; let fmt: Vec<&str> = r[8].split(':').collect(); let val: Vec<&str> = r[9].split(':').collect(); - let Some(i) = fmt.iter().position(|f| *f == "AF").filter(|i| *i < val.len()) else { + let Some(i) = fmt + .iter() + .position(|f| *f == "AF") + .filter(|i| *i < val.len()) + else { continue; }; let afs: Result, _> = val[i] @@ -385,7 +385,10 @@ pub fn mito(c: &Ctx<'_>) -> Out { if r[4].split(',').any(|a| in_polyc(pos, &r[3], a)) { polyc.push( Obj::new() - .set("variant", s(&format!("{}:{} {}>{}", r[0], r[1], r[3], r[4]))) + .set( + "variant", + s(&format!("{}:{} {}>{}", r[0], r[1], r[3], r[4])), + ) .set("af", round(af, 3)) .val(), ); diff --git a/rust/crates/og-summary/src/collect/group_c/cn.rs b/rust/crates/og-summary/src/collect/group_c/cn.rs index b5ec14d6a..04e455967 100644 --- a/rust/crates/og-summary/src/collect/group_c/cn.rs +++ b/rust/crates/og-summary/src/collect/group_c/cn.rs @@ -107,7 +107,9 @@ fn region(h: &str) -> Option<(String, i128, i128)> { } let digits = |x: &str| -> Option<(i128, usize)> { let n = x.bytes().take_while(u8::is_ascii_digit).count(); - (n > 0).then(|| (x[..n].parse().ok(), n)).and_then(|(v, n)| Some((v?, n))) + (n > 0) + .then(|| (x[..n].parse().ok(), n)) + .and_then(|(v, n)| Some((v?, n))) }; let (b, n) = digits(rest)?; let rest = rest[n..].strip_prefix('-')?; @@ -171,7 +173,9 @@ pub fn parascopy_cn_at(rows: &[DictRow], chrom: &str, pos: i64) -> CnAt { }) .collect(); cn_at_rows( - getters.iter().map(|g| g.as_ref() as &dyn Fn(&str) -> Option), + getters + .iter() + .map(|g| g.as_ref() as &dyn Fn(&str) -> Option), chrom, pos, "no row covers exon 7", @@ -195,7 +199,9 @@ pub fn parascopy_smn1(path: &str) -> Result<(Option, String), String> { }) .collect(); let r = cn_at_rows( - getters.iter().map(|g| g.as_ref() as &dyn Fn(&str) -> Option), + getters + .iter() + .map(|g| g.as_ref() as &dyn Fn(&str) -> Option), SMN1_EXON7.0, SMN1_EXON7.1, "no row covers SMN1 exon 7", @@ -239,7 +245,11 @@ pub fn read_marker_table(path: &str, step: &str, key: &str) -> Value { out.insert("status", "not assessed".to_owned()); let name = base(path); let done = |out: indexmap::IndexMap<&str, String>| { - Value::Object(out.into_iter().map(|(k, v)| (k.to_owned(), s(&v))).collect()) + Value::Object( + out.into_iter() + .map(|(k, v)| (k.to_owned(), s(&v))) + .collect(), + ) }; if !crate::is_file(path) { out.insert( @@ -253,7 +263,10 @@ pub fn read_marker_table(path: &str, step: &str, key: &str) -> Value { } let kv: std::collections::HashMap = pairs(path).unwrap_or_default().into_iter().collect(); - if !kv.get("status").is_some_and(|v| statuses.contains(&v.as_str())) { + if !kv + .get("status") + .is_some_and(|v| statuses.contains(&v.as_str())) + { out.insert( "message", format!( @@ -432,7 +445,8 @@ pub fn parascopy(c: &Ctx<'_>) -> Out { /// One catalog table that is not SMN1's: the locus's aggregate copy number /// per region of its profile, the catalog's genes, tag and reading. fn parascopy_locus(name: &str, path: &str, row: Option<&DictRow>) -> Result { - let cat = |k: &str, default: &str| row.map_or_else(|| default.to_owned(), |r| or(r, k, default)); + let cat = + |k: &str, default: &str| row.map_or_else(|| default.to_owned(), |r| or(r, k, default)); let out = Obj::new() .set("table", s(name)) .set("genes", s(&cat("genes", name))) @@ -440,7 +454,10 @@ fn parascopy_locus(name: &str, path: &str, row: Option<&DictRow>) -> Result) -> Result = - regs.iter().filter(|r| r.ok).map(|r| r.cn.as_str()).collect(); + let values: std::collections::HashSet<&str> = regs + .iter() + .filter(|r| r.ok) + .map(|r| r.cn.as_str()) + .collect(); if values.len() == 1 && regs.iter().all(|r| r.ok) { let n = crate::int(values.into_iter().next().unwrap_or_default()) .map_err(|e| super::super::sections::py_error("ValueError", &e))?; let mut min = f64::INFINITY; for r in ®s { - min = min.min(crate::float(&r.qual).map_err(|e| { - super::super::sections::py_error("ValueError", &e) - })?); + min = min.min( + crate::float(&r.qual) + .map_err(|e| super::super::sections::py_error("ValueError", &e))?, + ); } let text = format!( "{n} ({} region(s), quality {} or more, filter PASS)", @@ -510,7 +531,9 @@ fn parascopy_locus(name: &str, path: &str, row: Option<&DictRow>) -> Result crate::int(b).map_or_else(|_| b.clone(), |v| (v + 1).to_string()), + Some(b) if isdigit(b) => { + crate::int(b).map_or_else(|_| b.clone(), |v| (v + 1).to_string()) + } b => none(b), }; format!( @@ -621,7 +644,8 @@ fn read_hba(path: &str) -> Result { out.insert(k.to_owned(), s(v)); } } - let ok = matches!(out.get("status"), Some(Value::Str(v)) if v == "called" || v == "not assessed"); + let ok = + matches!(out.get("status"), Some(Value::Str(v)) if v == "called" || v == "not assessed"); if !ok { return Ok(Obj::new() .set("status", s("not assessed")) @@ -669,7 +693,10 @@ pub fn paralogs(c: &Ctx<'_>) -> Out { let mut others_called = 0; for r in &t.rows { let reg = region_of(r); - match reg.as_ref().and_then(|g| regions.iter().position(|x| x == g)) { + match reg + .as_ref() + .and_then(|g| regions.iter().position(|x| x == g)) + { Some(i) => { let cell = |k: &str| opt_s(r.get(k).flatten()); genes.push(( @@ -723,12 +750,20 @@ pub fn paralogs(c: &Ctx<'_>) -> Out { Obj::new().set("cn", cn(n)).set("text", s(&text)).val() }); let data = Obj::new() - .set("genes", Value::Array(genes.into_iter().map(|g| g.1).collect())) + .set( + "genes", + Value::Array(genes.into_iter().map(|g| g.1).collect()), + ) .set("other_regions", count(others)) .set("other_regions_called", count(others_called)) .set( "variants", - Value::Object(variants.into_iter().map(|(k, v)| (k, Value::Int(v))).collect()), + Value::Object( + variants + .into_iter() + .map(|(k, v)| (k, Value::Int(v))) + .collect(), + ), ) .set( "smn1", diff --git a/rust/crates/og-summary/src/collect/group_c/hla.rs b/rust/crates/og-summary/src/collect/group_c/hla.rs index 3d976a9fc..f5429d12f 100644 --- a/rust/crates/og-summary/src/collect/group_c/hla.rs +++ b/rust/crates/og-summary/src/collect/group_c/hla.rs @@ -4,7 +4,7 @@ //! pass when they ran. The rules are docs/08-hla-typing.md's "What the //! report derives". -use super::util::{Obj, at, bin_rows, bin_table, count, key_error, lines, s, strs, table}; +use super::util::{Obj, at, bin_rows, count, key_error, lines, s, strs, table}; use super::{Ctx, Out}; use og_steps::pgx_rules::HLA_GENES; use pycompat::json::Value; @@ -62,7 +62,11 @@ fn name(x: &str) -> &str { /// T1K's allele field: one allele, or equally likely candidates separated /// by commas. fn candidates(allele: &str) -> Vec<&str> { - allele.split(',').map(name).filter(|a| !a.is_empty()).collect() + allele + .split(',') + .map(name) + .filter(|a| !a.is_empty()) + .collect() } /// The alleles and their qualities of columns 3 and 6 (`.` is no allele). @@ -381,7 +385,7 @@ pub fn hla(c: &Ctx<'_>) -> Out { let (a, q) = alleles(&c, false); loci.push(locus(c[0], a, q, c.get(1).copied().unwrap_or(""))); } - let table = bin_rows(&bin_table("hla_alleles.tsv"))?; + let table = bin_rows(&c.table("hla_alleles.tsv"))?; let mut out = Obj::new() .set("loci", Value::Array(loci.iter().map(Locus::json).collect())) .set("derived", derived(&loci, &table)?); diff --git a/rust/crates/og-summary/src/collect/group_c/repeats.rs b/rust/crates/og-summary/src/collect/group_c/repeats.rs index a0e75c123..9d9610fd0 100644 --- a/rust/crates/og-summary/src/collect/group_c/repeats.rs +++ b/rust/crates/og-summary/src/collect/group_c/repeats.rs @@ -4,9 +4,7 @@ //! TG and poly-T tracts and UGT1A1's TA repeat mean. use super::super::model::State; -use super::util::{ - Obj, bin_rows, bin_table, cells, count, get_default, s, strs, table, vcf_lines, -}; +use super::util::{Obj, bin_rows, cells, count, get_default, s, strs, table, vcf_lines}; use super::{Ctx, Out}; use indexmap::IndexMap; use pycompat::json::Value; @@ -120,9 +118,9 @@ fn split_n(x: &str, n: usize) -> Vec { } /// `bin/str_intermediate_ranges.tsv` by locus. -fn ranges() -> Result>, String> { +fn ranges(c: &Ctx<'_>) -> Result>, String> { let mut out = IndexMap::new(); - for r in bin_rows(&bin_table("str_intermediate_ranges.tsv"))? { + for r in bin_rows(&c.table("str_intermediate_ranges.tsv"))? { let k = r .get("locus") .cloned() @@ -211,11 +209,12 @@ pub fn expansions(c: &Ctx<'_>) -> Out { let Some(p) = st.clone().or(eh) else { return Ok(None); }; - let ranges = ranges()?; + let ranges = ranges(c)?; let stp = st.as_ref().map(|f| f.path.as_str()); let rfc1 = match stp { - Some(after) => motif_table(c, "rfc1_motif.tsv", after)? - .and_then(|t| t.rows.first().map(motif_row)), + Some(after) => { + motif_table(c, "rfc1_motif.tsv", after)?.and_then(|t| t.rows.first().map(motif_row)) + } None => None, }; let mut motifs: IndexMap = IndexMap::new(); @@ -269,7 +268,11 @@ pub fn expansions(c: &Ctx<'_>) -> Out { }; let n = counts.len().max(1); let path_min = info.get("STR_PATHOLOGIC_MIN").and_then(|v| str_int(v)); - let cn_list: Vec<&str> = if counts.is_empty() { vec!["."] } else { counts.clone() }; + let cn_list: Vec<&str> = if counts.is_empty() { + vec!["."] + } else { + counts.clone() + }; let sizing: Vec = split_n(field("SO"), n) .iter() .zip(split_n(field("ADSP"), n)) @@ -288,10 +291,19 @@ pub fn expansions(c: &Ctx<'_>) -> Out { Vec::new() }; let worst = sizing.iter().copied().min().unwrap_or(Sizing::NoCall); - let dots = |k: &str| info.get(k).copied().unwrap_or("").trim_matches('.').to_owned(); + let dots = |k: &str| { + info.get(k) + .copied() + .unwrap_or("") + .trim_matches('.') + .to_owned() + }; let mut item = Obj::new() .set("locus", s(&rep)) - .set("repeat_count", s(if repcn.is_empty() { "." } else { repcn })) + .set( + "repeat_count", + s(if repcn.is_empty() { "." } else { repcn }), + ) .set("status", s(&status.join(","))) .set("normal_max", s(&dots("STR_NORMAL_MAX"))) .set("pathologic_min", s(&dots("STR_PATHOLOGIC_MIN"))) @@ -301,7 +313,11 @@ pub fn expansions(c: &Ctx<'_>) -> Out { .set("sizing", s(worst.word())) .set( "allele_sizing", - s(&sizing.iter().map(|z| z.word()).collect::>().join("/")), + s(&sizing + .iter() + .map(|z| z.word()) + .collect::>() + .join("/")), ) .set("zone", s("")) .set("zone_label", s("")) @@ -320,11 +336,8 @@ pub fn expansions(c: &Ctx<'_>) -> Out { if !sized.is_empty() { let (lo, hi) = (bound("intermediate_min")?, bound("intermediate_max")?); if sized.iter().any(|c| lo <= *c && *c <= hi) { - let cell = |k: &str| { - rng.get(k) - .cloned() - .ok_or_else(|| super::util::key_error(k)) - }; + let cell = + |k: &str| rng.get(k).cloned().ok_or_else(|| super::util::key_error(k)); item.put( "zone", s(&format!( @@ -389,7 +402,12 @@ pub fn expansions(c: &Ctx<'_>) -> Out { } let by_unsorted = by_locus(loci.iter().map(|l| (&l.locus, &l.json))); loci.sort_by(|a, b| { - let rank = |l: &str| EH_LOCI.iter().position(|k| *k == l).unwrap_or(EH_LOCI.len()); + let rank = |l: &str| { + EH_LOCI + .iter() + .position(|k| *k == l) + .unwrap_or(EH_LOCI.len()) + }; rank(&a.locus) .cmp(&rank(&b.locus)) .then_with(|| a.locus.cmp(&b.locus)) @@ -401,7 +419,9 @@ pub fn expansions(c: &Ctx<'_>) -> Out { .set("locus", s(k)) .set( "repeat_count", - by_unsorted.get(*k).map_or_else(|| s("not in output"), |l| l.repeat_count.clone()), + by_unsorted + .get(*k) + .map_or_else(|| s("not in output"), |l| l.repeat_count.clone()), ) .val() }) @@ -656,7 +676,9 @@ fn cftr(c: &Ctx<'_>, by: &IndexMap, traits: State) -> Value { } else { format!("{}T or {}T", b[0], b[1]) }; - line += &format!(" R117H ({g}) is read: with no 5T copy it sits on a {copies} copy.{note}"); + line += &format!( + " R117H ({g}) is read: with no 5T copy it sits on a {copies} copy.{note}" + ); } Some(_) => line += &format!(" No R117H copy.{note}"), } @@ -701,11 +723,16 @@ fn cftr(c: &Ctx<'_>, by: &IndexMap, traits: State) -> Value { Some(g) => { let n = a_copies(g); let (p, t) = if n == 0 { - ("not_needed", " No R117H copy: the 5T is read on its own.".to_owned()) + ( + "not_needed", + " No R117H copy: the 5T is read on its own.".to_owned(), + ) } else if n == 2 || five == 2 { ( "in_cis", - format!(" R117H ({g}) and 5T share a copy: R117H with 5T in cis is a CF-causing allele of variable penetrance."), + format!( + " R117H ({g}) and 5T share a copy: R117H with 5T in cis is a CF-causing allele of variable penetrance." + ), ) } else { ( @@ -816,7 +843,11 @@ fn ugt1a1(c: &Ctx<'_>, by: &IndexMap, cpic: State) -> Value { if let Some(row) = row { let d = super::util::or(&row, "Diplotype", ""); let d = pycompat::num::py_strip(&d); - pc = if d.is_empty() { ".".into() } else { d.to_owned() }; + pc = if d.is_empty() { + ".".into() + } else { + d.to_owned() + }; let alleles: Vec<&str> = pc.split('/').collect(); if alleles.len() == 2 && !pc.contains("possible") @@ -838,9 +869,9 @@ fn ugt1a1(c: &Ctx<'_>, by: &IndexMap, cpic: State) -> Value { "" }; reading += &match agreement { - "agrees" => format!( - " PharmCAT (step 7) called {pc}: the same number of *28 and *37 copies{old}." - ), + "agrees" => { + format!(" PharmCAT (step 7) called {pc}: the same number of *28 and *37 copies{old}.") + } "differs" => format!( " PharmCAT (step 7) called {pc}, a different number of *28 and *37 copies{old}: read the reads at chr2:233760234 before using either." ), diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index ca743eb77..489aa1a7e 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -77,7 +77,12 @@ fn overlap(a: i128, b: i128, start: i128, end: i128) -> i128 { pub fn is_autosome(chrom: &str) -> bool { chrom .strip_prefix("chr") - .and_then(|n| n.parse::().ok().filter(|v| (1..=22).contains(v)).map(|v| (v, n))) + .and_then(|n| { + n.parse::() + .ok() + .filter(|v| (1..=22).contains(v)) + .map(|v| (v, n)) + }) .is_some_and(|(v, n)| n == v.to_string()) } @@ -122,7 +127,14 @@ pub fn roh_band(f: f64) -> &'static str { /// equal): chrom, the region's span (none for the whole chromosome), the /// region, the maternal and paternal UPD syndromes and the PMIDs. /// chrom, span, then four text fields. -type Imprinted = (&'static str, Option<(i128, i128)>, &'static str, &'static str, &'static str, &'static str); +type Imprinted = ( + &'static str, + Option<(i128, i128)>, + &'static str, + &'static str, + &'static str, + &'static str, +); pub const IMPRINTED_REGIONS: [Imprinted; 6] = [ ( @@ -271,12 +283,14 @@ pub fn section(c: &Ctx<'_>) -> Out { let f_roh = long_bp / autosome_unmasked_bp() as f64; let mut upd = Value::Null; // The first chromosome of the most bases, as max() picks. - let top = per_chrom - .iter() - .fold(None::<(&String, &(f64, Vec<(i128, i128)>))>, |best, kv| match best { - Some(b) if b.1.0 >= kv.1.0 => Some(b), - _ => Some(kv), - }); + let top = + per_chrom.iter().fold( + None::<(&String, &(f64, Vec<(i128, i128)>))>, + |best, kv| match best { + Some(b) if b.1.0 >= kv.1.0 => Some(b), + _ => Some(kv), + }, + ); if let Some((chrom, (bp, runs))) = top && *bp >= UPD_MIN_BP && *bp > UPD_SHARE * long_bp @@ -309,7 +323,7 @@ pub fn section(c: &Ctx<'_>) -> Out { // --- og roh summary ------------------------------------------------------------ -const USAGE: &str = "usage: og roh summary --sample SAMPLE --roh ROH --out OUT\n"; +const USAGE: &str = "usage: og roh summary --sample SAMPLE --roh ROH [--out OUT]\n"; /// A field as awk reads it as a number: the longest leading decimal /// number, else 0. @@ -389,15 +403,15 @@ pub fn summary_rows(roh: &str) -> Result { } /// `og roh summary`: step 11's summary table of the long runs, for the bash -/// step and the ROH module alike. Exit 0 when written, 1 when the roh file -/// cannot be read or the table written, 2 for a command line it cannot -/// read. -pub fn cli(argv: &[String], err: &mut impl Write) -> u8 { +/// step and the ROH module alike, into `--out` or on stdout. Exit 0 when +/// written, 1 when the roh file cannot be read or the table written, 2 for +/// a command line it cannot read. +pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { use crate::args::{Kind, Opt, parse}; const OPTS: &[Opt] = &[ Opt("--sample", Kind::Value, true), Opt("--roh", Kind::Value, true), - Opt("--out", Kind::Value, true), + Opt("--out", Kind::Value, false), ]; let a = match parse(argv, OPTS) { Ok(a) => a, @@ -407,13 +421,76 @@ pub fn cli(argv: &[String], err: &mut impl Write) -> u8 { return 2; } }; - let run = || -> Result<(), String> { + let mut run = || -> Result<(), String> { let rows = summary_rows(a.text("--roh"))?; let body = format!( "# ROH Summary for {}\n# Segments >=5MB on autosomes (excludes chrX/chrY)\nchrom\tstart\tend\tlength_bp\tlength_mb\n{rows}", a.text("--sample") ); - crate::write_file(a.text("--out"), body.as_bytes()) + match a.get("--out") { + Some(path) => crate::write_file(path, body.as_bytes()), + None => out + .write_all(body.as_bytes()) + .and_then(|()| out.flush()) + .map_err(|e| format!("stdout: {e}")), + } }; crate::finish(run(), err) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + #[test] + fn awk_reads_the_leading_number() { + assert_eq!(awk_num("7000000"), 7_000_000.0); + assert_eq!(awk_num("8000000.5"), 8_000_000.5); + assert_eq!(awk_num("1e3x"), 1000.0); + assert_eq!(awk_num("1e"), 1.0); + assert_eq!(awk_num("x"), 0.0); + assert_eq!(awk_num("-2"), -2.0); + } + + #[test] + fn the_summary_keeps_the_unmasked_part_of_long_autosomal_runs() { + let d = Dir::new("roh-summary"); + let roh = d.put( + "roh.txt", + "# RG\nRG\tS\tchrY\t1\t9000000\t9000000\t9\t9\n\ + RG\tS\tchr1\t120000001\t145000000\t25000000\t9\t9\n\ + RG\tS\tchr2\t10000001\t17000000\t7000000\t9\t9\n\ + RG\tS\tchr13\t1\t25000000\t25000000\t9\t9\n\ + RG\tS\tchr5\t1\t4999999\t4999999\t9\t9\n", + ); + let argv: Vec = ["--sample", "S", "--roh", &roh] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&argv, &mut out, &mut err), 0); + assert_eq!( + String::from_utf8(out).unwrap(), + "# ROH Summary for S\n# Segments >=5MB on autosomes (excludes chrX/chrY)\n\ + chrom\tstart\tend\tlength_bp\tlength_mb\n\ + chr2\t10000001\t17000000\t7000000\t7.0\n\ + chr13\t1\t25000000\t6100000\t6.1\n" + ); + } + + #[test] + fn a_missing_roh_file_exits_1_and_no_sample_exits_2() { + let argv = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::>(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!( + cli( + &argv(&["--sample", "S", "--roh", "/no/such"]), + &mut out, + &mut err + ), + 1 + ); + assert_eq!(cli(&argv(&["--roh", "x"]), &mut out, &mut err), 2); + } +} diff --git a/rust/crates/og-summary/src/collect/group_c/sv.rs b/rust/crates/og-summary/src/collect/group_c/sv.rs index 55195c58e..4bcf320d3 100644 --- a/rust/crates/og-summary/src/collect/group_c/sv.rs +++ b/rust/crates/og-summary/src/collect/group_c/sv.rs @@ -26,7 +26,10 @@ fn totals(p: Found) -> Out { let (t, ps) = count_vcf(&p.path)?; Ok(Some(( p, - Obj::new().set("total", count(t)).set("pass", count(ps)).val(), + Obj::new() + .set("total", count(t)) + .set("pass", count(ps)) + .val(), ))) } @@ -138,16 +141,14 @@ pub fn cnvpytor(c: &Ctx<'_>) -> Out { /// with no declared sex they stay, and the data says so. pub fn cnvpytor_sex_filter(v: &mut Value, declared_sex: Option<&str>) { let Value::Object(m) = v else { return }; - let (Some(Value::Int(kept)), Some(Value::Int(y))) = - (m.get("significant_filtered").cloned(), m.get("filtered_chry").cloned()) - else { + let (Some(Value::Int(kept)), Some(Value::Int(y))) = ( + m.get("significant_filtered").cloned(), + m.get("filtered_chry").cloned(), + ) else { return; }; let declared = declared_sex.filter(|x| !x.is_empty()); - m.insert( - "declared_sex".into(), - declared.map_or(Value::Null, s), - ); + m.insert("declared_sex".into(), declared.map_or(Value::Null, s)); if declared == Some("female") { if let Some(Value::Object(d)) = m.get_mut("dropped") { d.insert("chry_female".into(), Value::Int(y)); @@ -182,7 +183,11 @@ fn info_value<'a>(info: &'a str, key: &str) -> &'a str { fn kind(t: &str) -> String { let t = pycompat::text::upper(t); - if t.starts_with("DUP") { "DUP".into() } else { t } + if t.starts_with("DUP") { + "DUP".into() + } else { + t + } } struct Rec { @@ -223,17 +228,14 @@ fn read_consensus(path: &str) -> Result { let (Ok(end), Ok(supp)) = (end, supp) else { continue; }; - by.entry(( - chrom_key(c[0]).to_owned(), - kind(info_value(c[7], "SVTYPE")), - )) - .or_default() - .push(Rec { - pos, - end, - supp, - vec: info_value(c[7], "SUPP_VEC").to_owned(), - }); + by.entry((chrom_key(c[0]).to_owned(), kind(info_value(c[7], "SVTYPE")))) + .or_default() + .push(Rec { + pos, + end, + supp, + vec: info_value(c[7], "SUPP_VEC").to_owned(), + }); } Ok((names, by)) } @@ -252,7 +254,10 @@ fn consensus_support(rows: &[DictRow], path: Option<&str>) -> Result = None; if let (Ok(start), Ok(end)) = (crate::int(cell("SV_start")), crate::int(cell("SV_end"))) { - let key = (chrom_key(cell("SV_chrom")).to_owned(), kind(cell("SV_type"))); + let key = ( + chrom_key(cell("SV_chrom")).to_owned(), + kind(cell("SV_type")), + ); for rec in by.get(&key).map_or(&[][..], Vec::as_slice) { let (dp, de) = ((rec.pos - start).abs(), (rec.end - end).abs()); if dp <= CONSENSUS_BP && de <= CONSENSUS_BP && best.is_none_or(|(d, _)| dp + de < d) @@ -277,10 +282,7 @@ fn consensus_support(rows: &[DictRow], path: Option<&str>) -> Result ( - int(1), - s(&super::util::or(r, "Caller", ".")), - ), + None => (int(1), s(&super::util::or(r, "Caller", "."))), }); } Ok(out) @@ -360,9 +362,8 @@ pub fn annotsv(c: &Ctx<'_>) -> Out { let mut moved = Vec::new(); for (r, (sp, callers)) in t.rows.iter().zip(support) { // A row cut short reads None there, as Python's DictReader gives it. - let cell = |k: &str| -> Result { - Ok(at(r, k)?.unwrap_or("None").to_owned()) - }; + let cell = + |k: &str| -> Result { Ok(at(r, k)?.unwrap_or("None").to_owned()) }; let cellv = |k: &str| -> Result { Ok(super::util::opt_s(at(r, k)?)) }; let af = match at(r, "Population_AF")? { None | Some("") => Value::Null, @@ -374,7 +375,11 @@ pub fn annotsv(c: &Ctx<'_>) -> Out { .map(str::to_owned) .collect(); let fmt = sample_format(r, &t.head); - let fmt_get = |k: &str| fmt.get(k).filter(|v| !v.is_empty()).map_or(Value::Null, |v| s(v)); + let fmt_get = |k: &str| { + fmt.get(k) + .filter(|v| !v.is_empty()) + .map_or(Value::Null, |v| s(v)) + }; let acmg = cell("ACMG_class")?; let support_callers = match &callers { Value::Str(x) if !x.is_empty() => strs(x.split(',')), diff --git a/rust/crates/og-summary/src/collect/group_c/tables.rs b/rust/crates/og-summary/src/collect/group_c/tables.rs index 7151ee4dd..5f39673e8 100644 --- a/rust/crates/og-summary/src/collect/group_c/tables.rs +++ b/rust/crates/og-summary/src/collect/group_c/tables.rs @@ -32,7 +32,10 @@ fn sorted_repr(need: &[&str]) -> String { } fn float_err(x: &str) -> String { - py_error("ValueError", &format!("could not convert string to float: {}", crate::repr(x))) + py_error( + "ValueError", + &format!("could not convert string to float: {}", crate::repr(x)), + ) } fn int_of(x: &str) -> Result { @@ -47,7 +50,10 @@ fn opt_num(x: Option<&str>) -> Value { fn first_line(path: &str) -> Result, String> { let l = super::util::lines(path)?; let h = l.first().map_or("", String::as_str); - Ok(h.trim_end_matches('\r').split('\t').map(str::to_owned).collect()) + Ok(h.trim_end_matches('\r') + .split('\t') + .map(str::to_owned) + .collect()) } /// Base64 of a file's bytes (`base64.b64encode`). @@ -135,7 +141,10 @@ pub fn couple(c: &Ctx<'_>) -> Out { .set("condition", s(&cell(r, "condition"))) .set("risk", s(&cell(r, "offspring_risk"))) .set("own", s(&cell(r, &format!("findings_{own}")))) - .set("partner_findings", s(&cell(r, &format!("findings_{other}")))) + .set( + "partner_findings", + s(&cell(r, &format!("findings_{other}"))), + ) .set("explanation", s(&cell(r, "explanation"))) .val() }; @@ -161,7 +170,12 @@ pub fn couple(c: &Ctx<'_>) -> Out { .set("x_linked", of("x_linked")) .set( "one_partner", - count(t.rows.iter().filter(|r| cell(r, "result") == "one_partner").count()), + count( + t.rows + .iter() + .filter(|r| cell(r, "result") == "one_partner") + .count(), + ), ) .val(); Ok(Some((p, v))) @@ -310,7 +324,9 @@ pub fn ancient(c: &Ctx<'_>) -> Out { return Ok(None); }; let t = table(&p.path, false)?; - let need = ["rank", "group", "n", "distance", "dates", "country", "locality"]; + let need = [ + "rank", "group", "n", "distance", "dates", "country", "locality", + ]; if !t.first_has(&need) { return unreadable(format!( "{} has no rows or lacks a column of {}", @@ -485,7 +501,11 @@ pub fn haplogroup(c: &Ctx<'_>) -> Out { .and_then(|(_, v)| v) .filter(|x| !x.is_empty()) }; - named.or_else(second).unwrap_or("").trim_matches('"').to_owned() + named + .or_else(second) + .unwrap_or("") + .trim_matches('"') + .to_owned() } }; let mut out = Obj::new() @@ -691,10 +711,7 @@ pub fn consumer_export(c: &Ctx<'_>) -> Out { let text = |k: &str| data.get(k).cloned().flatten().unwrap_or_default(); for k in ["sites", "called", "no_call"] { if !isdigit(&text(k)) { - return unreadable(format!( - "{} has no whole-number '{k}' row", - base(&p.path) - )); + return unreadable(format!("{} has no whole-number '{k}' row", base(&p.path))); } } let mut reasons = Obj::new(); @@ -711,7 +728,11 @@ pub fn consumer_export(c: &Ctx<'_>) -> Out { } } let blocks = text("from_reference_blocks"); - let blocks = if blocks.is_empty() { 0 } else { int_of(&blocks)? }; + let blocks = if blocks.is_empty() { + 0 + } else { + int_of(&blocks)? + }; let dot = |k: &str| { let v = text(k); s(if v.is_empty() { "." } else { &v }) @@ -807,10 +828,7 @@ pub fn archaic(c: &Ctx<'_>) -> Out { ] { let v = data.get(k).map_or("", String::as_str); if !all_digits(v) { - return unreadable(format!( - "{} has no whole number for {k}", - base(&p.path) - )); + return unreadable(format!("{} has no whole number for {k}", base(&p.path))); } out.put(k, Value::Int(int_of(v)?)); } @@ -820,10 +838,7 @@ pub fn archaic(c: &Ctx<'_>) -> Out { .split_once('.') .is_some_and(|(a, b)| all_digits(a) && all_digits(b)); if !ok { - return unreadable(format!( - "{} has no percentage for {k}", - base(&p.path) - )); + return unreadable(format!("{} has no percentage for {k}", base(&p.path))); } out.put(k, s(v)); } @@ -997,12 +1012,7 @@ pub fn locityper(c: &Ctx<'_>) -> Out { )); } let loci: Vec = t.rows.iter().map(|r| dots(r, &LOCITYPER_COLUMNS)).collect(); - let n = |k: &str, want: &str| { - t.rows - .iter() - .filter(|r| or_dot(r, k) == want) - .count() - }; + let n = |k: &str, want: &str| t.rows.iter().filter(|r| or_dot(r, k) == want).count(); let v = Obj::new() .set("called", count(n("status", "called"))) .set("passed", count(n("filter", "pass"))) @@ -1102,7 +1112,10 @@ pub fn locityper_post(v: &mut Value, hla_state: State, hla: Option<&Value>, othe } else { format!( " (not usable: Filter {})", - filter.as_deref().filter(|f| !f.is_empty()).unwrap_or("missing") + filter + .as_deref() + .filter(|f| !f.is_empty()) + .unwrap_or("missing") ) }; cyp.push(format!( @@ -1161,7 +1174,11 @@ pub fn viral(c: &Ctx<'_>) -> Out { need.join(", ") )); } - let all: Vec<&str> = need.iter().copied().chain(["human_depth", "note"]).collect(); + let all: Vec<&str> = need + .iter() + .copied() + .chain(["human_depth", "note"]) + .collect(); let keep = t .rows .iter() @@ -1314,11 +1331,7 @@ pub fn mei(c: &Ctx<'_>) -> Out { let dir = p.path.rsplit_once('/').map_or("", |x| x.0); let cp = crate::join(dir, &format!("{}_mei.tsv", c.sample)); if !crate::is_file(&cp) { - return unreadable(format!( - "{} is missing beside {}", - base(&cp), - base(&p.path) - )); + return unreadable(format!("{} is missing beside {}", base(&cp), base(&p.path))); } let calls = table(&cp, false)?; if ["chrom", "pos", "element", "copies"] @@ -1491,10 +1504,17 @@ pub fn exomiser(c: &Ctx<'_>) -> Out { .fold(Obj::new(), |o, k| o.set(k, cell(r, k))); genes.push((rank, o, Vec::new())); } - let v = ["variant", "genotype", "consequence", "hgvs", "acmg", "disease"] - .iter() - .fold(Obj::new(), |o, k| o.set(k, cell(r, k))) - .val(); + let v = [ + "variant", + "genotype", + "consequence", + "hgvs", + "acmg", + "disease", + ] + .iter() + .fold(Obj::new(), |o, k| o.set(k, cell(r, k))) + .val(); if let Some(g) = genes.last_mut() { g.2.push(v); } diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index e69de29bb..f470633aa 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -0,0 +1,152 @@ +//! Group c through `collect`: each section's state, and the post pass +//! (CNVpytor's chrY filter, Locityper's primary calls, the repeat readings). +//! The section readers are checked against the Python collector by the +//! summary-collect parity cases (tests/parity/cases.tsv). + +use super::super::{Group, Input, collect}; +use super::{GROUP_C, Others}; +use crate::testutil::Dir; +use pycompat::json::Value; + +fn summary(d: &Dir, declared_sex: Option<&str>) -> Value { + let input = Input { + sample: "S".into(), + sample_dir: d.path("S"), + declared_sex: declared_sex.map(str::to_owned), + assets: d.path("assets"), + group: Group::C, + tables: concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin").into(), + }; + collect(&input).unwrap().to_json() +} + +fn at<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + path.iter().fold(v, |v, k| match v { + Value::Object(m) => m.get(*k).unwrap_or_else(|| panic!("no {k} in {v:?}")), + Value::Array(a) => &a[k.parse::().unwrap()], + _ => panic!("{k} of {v:?}"), + }) +} + +fn text(v: &Value) -> &str { + match v { + Value::Str(s) => s, + _ => panic!("not a text: {v:?}"), + } +} + +const CNVS: &str = "deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\t0.05\t0\t9\n\ +deletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\t1\t1\t1\t0.01\t0\t9\n\ +deletion\tchr1:122000001-123000000\t1000000\t0.1\t1e-12\t1\t1\t1\t0.2\t0\t9\n"; + +#[test] +fn every_section_of_group_c_is_written_in_order() { + let d = Dir::new("c-order"); + let v = summary(&d, None); + let Value::Object(secs) = at(&v, &["sections"]) else { + panic!() + }; + let keys: Vec<&str> = secs.keys().map(String::as_str).collect(); + let want: Vec<&str> = GROUP_C.iter().map(|s| s.key).collect(); + assert_eq!(keys, want); + assert!(secs.values().all(|s| text(at(s, &["state"])) == "missing")); +} + +#[test] +fn cnvpytor_drops_chry_for_a_declared_female_only() { + let d = Dir::new("c-cnv"); + d.put("S/cnvpytor/S_cnvs.txt", CNVS); + let female = summary(&d, Some("female")); + let data = at(&female, &["sections", "cnvpytor", "data"]); + assert_eq!(at(data, &["significant_filtered"]), &Value::Int(1)); + assert_eq!(at(data, &["dropped", "chry_female"]), &Value::Int(1)); + assert_eq!(at(data, &["dropped", "band"]), &Value::Int(1)); + let male = summary(&d, Some("male")); + let data = at(&male, &["sections", "cnvpytor", "data"]); + assert_eq!(at(data, &["significant_filtered"]), &Value::Int(2)); + assert_eq!(at(data, &["declared_sex"]), &Value::Str("male".into())); +} + +#[test] +fn locityper_names_the_primary_call_of_each_locus() { + let d = Dir::new("c-loci"); + d.put( + "S/hla_t1k/S_hla_genotype.tsv", + "HLA-DRB1\t2\tDRB1*03:01\t1\t40\tDRB1*07:01\t1\t0\n", + ); + let head = "locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads\tweighted_distance\twarnings\tfilter\treason\n"; + d.put( + "S/locityper/S_locityper.tsv", + format!( + "{head}HLA-DRB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t\n\ + HLA-DQB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t\n\ + CYP2D6\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t\n\ + OTHER\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t\n" + ), + ); + let v = summary(&d, None); + let loci = at(&v, &["sections", "locityper", "data", "loci"]); + let primary = |i: &str| text(at(loci, &[i, "primary"])).to_owned(); + assert_eq!( + primary("0"), + "T1K (step 08): DRB1*03:01/DRB1*07:01 (low confidence)" + ); + assert_eq!(primary("1"), "T1K (step 08) did not run"); + assert_eq!( + primary("2"), + "pypgx (step 32) and Cyrius (step 21) did not run" + ); + assert_eq!(primary("3"), "no other step calls this locus"); +} + +#[test] +fn locityper_reads_pypgx_and_cyrius_from_group_b() { + let mut v = pycompat::json::loads(r#"{"loci": [{"locus": "CYP2D6"}]}"#).unwrap(); + let others = Others { + pypgx: Some((super::State::Stale, "*1/*4".into())), + cyrius: Some((super::State::Ok, Some("*1/*2".into()), None)), + ..Others::default() + }; + super::tables::locityper_post(&mut v, super::State::Missing, None, &others); + assert_eq!( + text(at(&v, &["loci", "0", "primary"])), + "pypgx (step 32): *1/*4 (stale: from an earlier run); Cyrius (step 21): *1/*2 (not usable: Filter missing)" + ); +} + +#[test] +fn a_cftr_5t_with_r117h_reads_the_phase() { + let d = Dir::new("c-cftr"); + let vcf = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n\ +chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9\n\ +chr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9\n"; + d.put("S/expansion_hunter/S_eh.vcf", vcf); + d.put( + "S/traits/S_traits.tsv", + "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\n\ +rs78655421\tCFTR\tR117H\tG/A\tcalled\tr\ta\tn\tp\n", + ); + let v = summary(&d, None); + let cftr = at(&v, &["sections", "expansions", "data", "readings", "0"]); + assert_eq!(text(at(cftr, &["phase"])), "unknown"); + assert_eq!(text(at(cftr, &["r117h"])), "G/A"); + assert!(text(at(cftr, &["reading"])).contains("phase unknown")); +} + +#[test] +fn a_stale_section_says_from_which_run() { + let d = Dir::new("c-stale"); + d.put( + "S/logs/run_status.tsv", + "meta\tstarted_epoch\t4102444800\nstep\t11\tskipped\n", + ); + d.put( + "S/vcf/S_roh.txt", + "RG\tS\tchr2\t10000001\t17000000\t7000000\t9\t9\n", + ); + let v = summary(&d, None); + let roh = at(&v, &["sections", "roh"]); + assert_eq!(text(at(roh, &["state"])), "stale"); + assert!(text(at(roh, &["note"])).contains("step 11 was skipped")); + assert_eq!(at(roh, &["data", "largest_mb"]), &Value::Float(7.0)); +} diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 56c4c856d..5d5a6a806 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -160,7 +160,10 @@ pub mod apol1 { pub fn json(&self) -> Obj { let o = Obj::new() .set("status", s(self.status)) - .set("copies", self.copies.map_or(Value::Null, |c| Value::Int(c.into()))) + .set( + "copies", + self.copies.map_or(Value::Null, |c| Value::Int(c.into())), + ) .set("gq", s(&self.gq)); let src = super::super::util::opt_s(self.source.as_deref()); if self.source_first { @@ -226,8 +229,12 @@ pub mod apol1 { /// record has no reads. pub fn read_g2(records: &[Vec], gvcf: bool) -> G2 { let g2_hap = haplotype(); - let (mut copies, mut covered, mut gqs, mut problems) = - (0i64, Vec::<(i64, i64)>::new(), Vec::::new(), Vec::::new()); + let (mut copies, mut covered, mut gqs, mut problems) = ( + 0i64, + Vec::<(i64, i64)>::new(), + Vec::::new(), + Vec::::new(), + ); let (mut seen, mut g2_records) = (false, 0); 'records: for r in records { if !overlaps(r) { @@ -536,7 +543,10 @@ pub mod apol1 { #[test] fn reference_block_over_span_reads_zero() { - let g = read_g2(&[rec(36_265_990, "A", "", "0/0", "END=36266010")], true); + let g = read_g2( + &[rec(36_265_990, "A", "", "0/0", "END=36266010")], + true, + ); assert_eq!((g.status, g.copies), ("called", Some(0))); } diff --git a/rust/crates/og-summary/src/collect/group_c/util.rs b/rust/crates/og-summary/src/collect/group_c/util.rs index a89d331b6..4aba9f470 100644 --- a/rust/crates/og-summary/src/collect/group_c/util.rs +++ b/rust/crates/og-summary/src/collect/group_c/util.rs @@ -61,12 +61,10 @@ pub fn lines(path: &str) -> Result, String> { let mut rd = text_lines(path).map_err(|e| os_note(&e))?; let mut out = Vec::new(); let mut s = String::new(); - while rd.read_line(&mut s).map_err(|e| { - py_error( - "OSError", - &crate::collect::sections::gzip_message(&e), - ) - })? { + while rd + .read_line(&mut s) + .map_err(|e| py_error("OSError", &crate::collect::sections::gzip_message(&e)))? + { out.push(s.strip_suffix('\n').unwrap_or(&s).to_owned()); s.clear(); } @@ -290,20 +288,20 @@ pub fn columns(path: &str) -> Result, String> { Ok(table(path, false)?.head) } -/// The path of a table the Python collector read beside itself in `bin/` -/// (`hla_alleles.tsv`, `str_intermediate_ranges.tsv`): the `bin/` of the -/// checkout the binary sits in (`bin/.og/og-*` and `rust/target/*/og` both -/// do). `OG_BIN_TABLES` names another folder. -pub fn bin_table(name: &str) -> String { +/// The folder of the tables the Python collector read beside itself in +/// `bin/` (`hla_alleles.tsv`, `str_intermediate_ranges.tsv`): the `bin/` of +/// the checkout the binary sits in (`bin/.og/og-*` and `rust/target/*/og` +/// both do). `OG_BIN_TABLES` names another folder. +pub fn default_tables() -> String { if let Some(d) = std::env::var_os("OG_BIN_TABLES") { - return std::path::Path::new(&d).join(name).display().to_string(); + return std::path::Path::new(&d).display().to_string(); } let exe = std::env::current_exe().unwrap_or_default(); exe.ancestors() .skip(1) - .flat_map(|a| [a.join(name), a.join("bin").join(name)]) - .find(|p| p.is_file()) - .unwrap_or_else(|| std::path::Path::new("bin").join(name)) + .flat_map(|a| [a.to_path_buf(), a.join("bin")]) + .find(|p| p.join("hla_alleles.tsv").is_file()) + .unwrap_or_else(|| std::path::PathBuf::from("bin")) .display() .to_string() } diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 698316d08..d50bd251a 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -92,7 +92,11 @@ impl FolderIndex { /// only). pub fn all(&self, pattern: &str) -> Vec<&str> { if !has_magic(pattern) { - return self.files.get(pattern).map(|f| vec![f.as_str()]).unwrap_or_default(); + return self + .files + .get(pattern) + .map(|f| vec![f.as_str()]) + .unwrap_or_default(); } let pat: Vec<&str> = pattern.split('/').collect(); self.sorted diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 65922dd61..75031f745 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -22,6 +22,7 @@ fn summary(d: &Dir, sample_dir: &str) -> Value { declared_sex: None, assets: d.path("assets"), group: super::Group::A, + tables: String::new(), }; collect(&input).unwrap().to_json() } diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 4a9ff4b55..c24c1b474 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -51,7 +51,7 @@ const USAGE: &str = "usage: og [args] og prs table [args] og mito annotate|deletions|copies [args] og summary collect --only a|c [args] - og roh summary --sample SAMPLE --roh ROH --out OUT + og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] og bloodtype report|check-markers|systems [args] @@ -307,6 +307,7 @@ fn main() -> ExitCode { "roh" => match rest.first().map(String::as_str) { Some("summary") => ExitCode::from(og_summary::collect::group_c::roh::cli( &rest[1..], + &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), _ => usage_error("og roh takes summary"), diff --git a/scripts/11-roh-analysis.sh b/scripts/11-roh-analysis.sh index a0f6b29ce..9b29031ec 100755 --- a/scripts/11-roh-analysis.sh +++ b/scripts/11-roh-analysis.sh @@ -55,7 +55,8 @@ run_in \ -v "${GENOME_DIR}/${SAMPLE}/vcf:/data" \ -v "${PGP_ROOT}/bin:/pgp-bin:ro" \ "${BCFTOOLS_IMAGE}" \ - /pgp-bin/og roh summary --sample "$SAMPLE" --roh "/data/${SAMPLE}_roh.txt" --out "/data/${SAMPLE}_roh_summary.txt" + /pgp-bin/og roh summary --sample "$SAMPLE" --roh "/data/${SAMPLE}_roh.txt" > "${SUMMARY}.tmp" +mv -f "${SUMMARY}.tmp" "$SUMMARY" echo "=== ROH complete ===" echo "Results: ${OUTPUT}" diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 0a9afd6bf..164e855b0 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -838,6 +838,21 @@ summary-collect empty gen:summary-collect.sh,summary-collect/oracle.sh --sample summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir declared-status --assets assets --declared-sex female --out declared-arg.json --only a declared-arg.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE +summary-collect c-full gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets assets --out full.json --only c full.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-nf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir nf --assets assets --out nf.json --only c nf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-bad1 gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir bad1 --assets assets --out bad1.json --only c bad1.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-bad2 gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir bad2 --assets assets --out bad2.json --only c bad2.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-left gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-left --assets assets --out g2-left.json --only c g2-left.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-shifted gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-shifted --assets assets --out g2-shifted.json --only c g2-shifted.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-two gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-two --assets assets --out g2-two.json --only c g2-two.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-gvcf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-gvcf --assets assets --out g2-gvcf.json --only c g2-gvcf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-none gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-none --assets assets --out g2-none.json --only c g2-none.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-other gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-other --assets assets --out g2-other.json --only c g2-other.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-nocall gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-nocall --assets assets --out g2-nocall.json --only c g2-nocall.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-g2-novcf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir g2-novcf --assets assets --out g2-novcf.json --only c g2-novcf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +roh-summary runs gen:roh-summary.sh,roh-summary/oracle.sh --sample S --roh roh.txt --out S_roh_summary.txt S_roh_summary.txt bytes PYTHON_IMAGE +roh-summary empty gen:roh-summary.sh,roh-summary/oracle.sh --sample sample.1_A-b --roh empty.txt --out out.txt out.txt bytes PYTHON_IMAGE +roh-summary stdout gen:roh-summary.sh,roh-summary/oracle.sh --sample S --roh roh.txt - bytes PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/roh-summary.sh b/tests/parity/gen/roh-summary.sh new file mode 100755 index 000000000..ecfd467c1 --- /dev/null +++ b/tests/parity/gen/roh-summary.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# roh-summary.sh: writes the bcftools roh outputs of the roh-summary parity +# cases in the current directory, the same every time: runs on chr1 to +# chr22 inside, across and outside the masked bands, runs on chrX, chrY and +# other contigs, the 5 Mb line from both sides, lengths whose megabases +# round on a half, blank-separated fields and lines that are not runs. +# scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C + +{ + printf '# This file was produced by: bcftools roh\n# RG\t[2]Sample\t[3]Chromosome\t[4]Start\t[5]End\t[6]Length (bp)\t[7]Number of markers\t[8]Quality (average fwd-bwd phred score)\n' + printf 'ST\tS\tchr1\t100\t0\t1.0\n' + printf 'RG\tS\tchrY\t2781480\t47781479\t45000000\t900\t50\n' + printf 'RG\tS\tchrX\t1\t9000000\t9000000\t900\t50\n' + printf 'RG\tS\tchrEBV\t1\t6000000\t6000000\t90\t50\n' + printf 'RG\tS\tchr1_KI270706v1_random\t1\t6000000\t6000000\t90\t50\n' + printf 'RG\tS\tchr1\t125200001\t143200000\t18000000\t300\t50\n' + printf 'RG\tS\tchr9\t42300001\t49300000\t7000000\t200\t50\n' + printf 'RG\tS\tchr1\t120000001\t145000000\t25000000\t500\t50\n' + printf 'RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n' + printf 'RG\tS\tchr13\t1\t25000000\t25000000\t900\t50\n' + printf 'RG\tS\tchr22\t17000001\t40000000\t23000000\t900\t50\n' + printf 'RG\tS\tchr3\t1\t5000000\t5000000\t900\t50\n' + printf 'RG\tS\tchr4\t1\t4999999\t4999999\t900\t50\n' + printf 'RG\tS\tchr5\t1\t5050000\t5050000\t900\t50\n' + printf 'RG\tS\tchr6\t1\t5150000\t5150000\t900\t50\n' + printf 'RG\tS\tchr7\t1\t5250000\t5250000\t900\t50\n' + printf 'RG S chr8 1 6000000 6000000 900 50\n' + printf ' RG\tS\tchr10\t100\t6000099\t6000000\t9\t9\n' + printf 'RG\tS\tchr23\t1\t9000000\t9000000\t900\t50\n' + printf 'RG\tS\tchr21\t1\t9000000\n' + printf 'RG\tS\n' + printf '\n' + printf 'RG\tS\tchr11\t50000001\t60000000\t10000000\t9\t9\n' + printf 'RG\tS\tchr12\t1\t8000000\t8000000.5\t9\t9\n' +} > roh.txt +printf '# empty\n' > empty.txt diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh new file mode 100644 index 000000000..18b27ce50 --- /dev/null +++ b/tests/parity/gen/summary-collect-c.sh @@ -0,0 +1,360 @@ +#!/usr/bin/env bash +# summary-collect-c.sh: writes the sample folders of the summary-collect +# parity cases of group c in the current directory, the same every time. +# Each folder is one case of og summary collect --only c against +# bin/collect_summary.py: every section of the group with a typical table, +# the Nextflow and other file names, files from an earlier run, unreadable +# tables of each kind, and the APOL1 G2 deletion written left-aligned, +# shifted, twice, under a reference block and not at all. +# py/ holds the modules the oracle imports, as they were at the oracle's +# commit, and the two bin/ tables it reads beside itself; assets/ is a copy +# of the repository's catalogs these sections read. +# scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) +ORACLE_SHA=98aa9252ff77825a8a797d388b449931fa9a234e + +T0=1577836800 # 2020-01-01 00:00:00 UTC, the start of the run +OLD=201906011230.00 # before the run +NEW=202103040506.07 # after it, the time of every file not set otherwise + +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; at "$NEW" "$1"; } +putgz() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" | gzip -n -c > "$1"; at "$NEW" "$1"; } + +# --- the oracle's modules ------------------------------------------------------- +git -C "$root" cat-file -e "${ORACLE_SHA}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$ORACLE_SHA" +mkdir -p py +for m in paraphase_report ancestry_fine_report annotsv_postfilter cnvpytor_to_vcf family_report chip_report \ + traits_report genotype_rules_indels pgx_outside_calls; do + git -C "$root" show "${ORACLE_SHA}:bin/${m}.py" > "py/${m}.py" +done +for t in hla_alleles.tsv str_intermediate_ranges.tsv; do + git -C "$root" show "${ORACLE_SHA}:bin/${t}" > "py/${t}" +done + +mkdir -p assets +for a in paraphase_genes.tsv paralog_loci.tsv mito_confirmed.tsv callability_genes.tsv; do + cp "${root}/assets/${a}" assets/ +done + +VCF_HEAD='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' + +# full: every section, the bash names ------------------------------------------- +f=full +put $f/couple/S_couple_carrier.tsv "gene\tcondition\tresult\toffspring_risk\tpartner_a\tfindings_a\tpartner_b\tfindings_b\texplanation +CFTR\tCystic fibrosis\tshared\t1 in 4\tS\tc.1521_1523del (P)\tP2\tc.350G>A (LP)\tBoth carry a change. +DMD\tDuchenne\tx_linked\t1 in 4 sons\tP2\t.\tS\tc.10G>T (P)\tOn X. +GJB2\tDeafness\tone_partner\t.\tS\tc.35del (P)\tP2\t.\tOne side. +HBB\tBeta thalassaemia\tone_partner\t.\tP2\tc.20A>T\tS\t.\tOne side.\n" +put $f/paralogs/S_smn_copy_number.tsv "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions +chr5\t70950000\t70953000\tSMN1\tPASS\t4\t45.5\tPASS\t1,3\t31,25\tchr5:70075000-70078000 +chr5\t70960000\t70970000\tSMN1\tPASS\t4\t40\tLowQual\t2,2\t10,12\t*\n" +put $f/paralogs/S_smn1_marker.tsv "metric\tvalue\nmarker\tSMN1 g.27134T>G\nstatus\tnot seen\nalt_reads\t0\ndepth\t31\nfraction\t0.000\ncopies\t0\nmessage\tno read carries the G\n" +put $f/paralogs/S_gba1_copy_number.tsv "chrom\tstart\tend\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual +chr1\t155231478\t155240000\tPASS\t4\t60\tPASS\t2,2\t30,30 +chr1\t155240000\t155244699\tPASS\t4\t35.25\tPASS\t2,2\t30,30\n" +put $f/paralogs/S_pms2_copy_number.tsv "chrom\tstart\tend\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual +chr7\t5970000\t5980000\tPASS\t4\t50\tPASS\t2,2\t30,30 +chr7\t5980000\t5990000\tLowQual\t3\t12\tPASS\t2,1\t30,30\n" +put $f/paralogs/S_mytable_copy_number.tsv "chrom\tstart\tend\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual +chr9\t1\t100\tnot assessed: no read over the locus\t.\t.\t.\t.\t.\n" +put $f/paralogs/S_strc_copy_number.tsv "chrom\tstart\tend\tagCN\n" +put $f/paralogs/S_hba_copy_number.tsv "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\ttwo copies of each gene\n" +put $f/paralogs/S_cn_intervals.tsv "locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange +DGCR\t22q11.2\tchr22\t18900000\t21500000\tcalled\t2\t2\tnone +AZFc\tAZFc\tchrY\t23000000\t26000000\tcalled\t0\t1\tdeletion +X1\tdup\tchr1\t1\t2\tcalled\t3\t2\tduplication +X2\tlow\tchr2\t1\t2\tnot assessed\t.\t2\t.\n" +put $f/paralogs/S_cyp21a2_psv.tsv "metric\tvalue\nstatus\tcalled\ncopies\t2\nmodule_total\t4\nfinding\tnone\n" +put $f/paralogs/S_cn_dosage.tsv "locus\tstatus\tcopies\testimate\treason\treading +HP\tcalled\t2\t2.1\t\tusual +LCE3BC\tcalled\t1\t1.0\t.\tone deleted +CCL3L1\tnot assessed\t\t\tlow depth\t\n" +put $f/paralogs/S_paraphase_copy_number.tsv "region\tgene\tcopy_number\tstatus\tdetail +pms2\tPMS2\t2\tcalled\t +smn1\tSMN1\t2\tcalled\tok +smn1\tSMN2\t2\tcalled\t +zz_other\tX\t2\tcalled\t. +zz_other2\tY\t.\tno call\t. +hba\tHBA1\t1\tcalled\t +hba\tHBA2\t2\tcalled\t\n" +put $f/paralogs/S_paraphase_variants.tsv "region\tpos\nsmn1\t1\nsmn1\t2\npms2\t3\n" +put $f/hla_t1k/S_hla_genotype.tsv "#comment\tline +HLA-A\t2\tA*02:01:01\t10\t60\tA*01:01:01\t9\t50 +HLA-B\t2\tB*57:01:01:02\t10\t0\tB*08:01:01,B*08:01:02,B*08:01:03,B*08:01:04,B*08:01:05\t9\t0 +HLA-C\t1\tC*07:01:01\t10\t40\t.\t0\t-1 +HLA-DQA1\t2\tDQA1*05:01:01\t10\t40\tDQA1*02:01\t9\t30 +HLA-DQB1\t2\tDQB1*02:01:01\t10\t40\tDQB1*02:02:01\t9\t30 +HLA-DRB1\t2\tDRB1*03:01:01\t10\t40\tDRB1*07:01:01\t9\tnan +HFE\t1\tHFE*001\t3\t20\t.\t0\t-1 +short\t1\n" +put $f/hla_optitype/S_hla_concordance.tsv "Gene\tT1K\tOptiType\tAgree\tNote\nHLA-A\t A*02:01 \tA*02:01\tyes\t\n\t\t\t\t\nHLA-B\tB*57:01\tB*57:01\tyes\tok\n" +put $f/kir_t1k/S_kir_genotype.tsv "KIR2DL1\t2\tKIR2DL1*001\t5\t30\tKIR2DL1*002\t4\t20\nKIR3DL3\t0\t.\t0\t-1\t.\t0\t-1\n" +put $f/prs/S_prs_summary.tsv "Condition\tPGS_ID\tScore_SUM\tVariants_Matched\tVariants_Total\tChrX\tAmbiguous\tApplies_To\tSex_Check\tPercentile\tAncestry_Group\tEvaluated_In\tEffect +CAD\tPGS000018\t0.5\t100\t120\t0\t12 dropped\tall\tok\t75\tEUR\tEUR;;AFR\tHR 1.5 (EUR) +T2D\tPGS000014\t0.1\t10\t20\t0\t0\tall\tok\tNA\tNA\t\t +Short\tPGS1\n" +put $f/ancestry/S_ancestry.tsv "key\tvalue\npopulation\tEUR\nreference_panel\tHGDP+1kGP\n\tskipped\npopulation_low_confidence\n" +put $f/ancestry_fine/S_ancestry_fine.tsv "level\tcode\tgroup\tfraction\tse\tlow\thigh +continent\tWEA\tWest Eurasia\t0.95\t0.01\t0.93\t0.97 +continent\tAFR\tAfrica\t0.05\t0.01\tnan\tx +region\tIBE\tIberia\t0.60\t0.05\t0.5\t0.7 +left_out\tOCE\tOceania\t.\t.\t.\t. +not_painted\tchr21\t.\t.\t.\t.\t. +other\tZ\tZ\t1\t1\t1\t1\n" +put $f/ancestry_fine/S_painting.png "\x89PNG\r\n\x1a\nnot a real picture\x00\xff" +putgz $f/manta/results/variants/diploidSV.vcf.gz "${VCF_HEAD}chr1\t100\tM1\tA\t\t50\tPASS\tSVTYPE=DEL\tGT\t0/1\nchr1\t200\tM2\tA\t\t50\tMinQUAL\tSVTYPE=DUP\tGT\t0/1\nchr2\t5\n" +putgz $f/delly/S_sv.vcf.gz "${VCF_HEAD}chr1\t100\tD1\tA\t\t50\tPASS\tSVTYPE=DEL\tGT\t0/1\n" +put $f/cnvpytor/S_cnvs.txt "deletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\t1\t1\t1\t0.01\t0\t9 +deletion\tchr1:122000001-123000000\t1000000\t0.1\t1e-12\t1\t1\t1\t0.2\t0\t9 +duplication\tchr3:1000001-1200000\t200000\t1.8\t1e-6\t1\t1\t1\t0.8\t0\t9 +deletion\tchr5:1-100000\t100000\t0.0\t1e-20\t1\t1\t1\t0\t0.9\t0 +deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\t0.05\t0\t9 +deletion\tchr4:1000001-1050000\t50000\t0.6\t0.2\t1\t1\t1\t0\t0\t9 +duplication\tchr6:1-2\t1\t2\tnan\t1\t1\t1\t0\t0\t9 +duplication\tweird\t1\t2\t0.001 + +\n" +putgz $f/sv_merged/S_sv_consensus.vcf.gz "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tmanta\tdelly\ttiddit +chr1\t1000\tc1\tN\t\t.\tPASS\tSVTYPE=DEL;END=5000;SUPP=2;SUPP_VEC=110\tGT\t0/1\t0/1\t./. +chr2\t100\tc2\tN\t\t.\tPASS\tSVTYPE=DUP:TANDEM;END=900;SUPP=3;SUPP_VEC=1111\tGT\t0/1\t0/1\t0/1 +chr3\tx\tc3\tN\t\t.\tPASS\tSVTYPE=INV\tGT\t0/1\t0/1\t0/1\n" +put $f/annotsv/S_sv_class4_5_checked.tsv "AnnotSV_ID\tSV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tGene_name\tACMG_class\tFORMAT\tS\tAnnotation_mode\tCaller\tExon_edge\tExon_edge_detail\tPopulation_AF\tPopulation_source\tCommon\tAnnotSV_class\tRule\tRule_detail +a1\t1\t1100\t5100\t-4000\tDEL\tGENE1;GENE2;\t5\tGT:PS\t0/1:77\tsplit\tmanta\tyes\texon 2\t0.001\tgnomAD\tno\t5\t\t +a2\t2\t150\t880\t730\tDUP\tGENE3\t4\tGT\t1/1\tsplit\tdelly\tno\t\t\t\tno\t\t\t +a3\t7\t1\t100\t99\tINV\t\t3\tGT\t0/1\tsplit\tmanta\tnot checked\t\tx\tgnomAD\tyes\t4\tmhc\tthe MHC +a4\t9\t1\t100\t99\tDEL\tG9\t4\t\t\tfull\ttiddit\tno\t\t0.2\tgnomAD\tyes\t\t\t\n" +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12:5/5:0/0:30\n" +EH_RECS="chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT;STR_STATUS=normal,normal;STR_NORMAL_MAX=26;STR_PATHOLOGIC_MIN=40\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12 +chrX\t147912051\t.\tC\t\t.\tPASS\tEND=147912110;VARID=FMR1;REPID=FMR1;STR_STATUS=premutation;STR_NORMAL_MAX=44;STR_PATHOLOGIC_MIN=200\tGT:SO:REPCN:REPCI:ADSP\t1:SPANNING:58:55-60:1 +chr4\t39348425\t.\tA\t\t.\tPASS\tEND=39348480;VARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation;STR_NORMAL_MAX=.;STR_PATHOLOGIC_MIN=400\tGT:SO:REPCN:REPCI:ADSP\t1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0 +chr7\t117548628\t.\tT\t\t.\tPASS\tEND=117548650;VARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9 +chr7\t117548651\t.\tT\t\t.\tPASS\tEND=117548660;VARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9 +chr2\t233760233\t.\tA\t\t.\tPASS\tEND=233760250;VARID=UGT1A1;REPID=UGT1A1;STR_STATUS=normal\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:7/8:7-7/8-8:5/6 +chr6\t16327636\t.\tT\t\t.\tPASS\tEND=16327700;VARID=ATXN1;REPID=ATXN1;STR_STATUS=normal;STR_NORMAL_MAX=35;STR_PATHOLOGIC_MIN=39\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/FLANKING:30/37:30-30/35-40:4/0 +chr9\t27573528\t.\tC\t\t.\tPASS\tEND=27573540;VARID=C9ORF72;REPID=C9ORF72\tGT:SO:REPCN:REPCI:ADSP\t./.:.:.:.:. +chr1\t10\t.\tC\t\t.\tPASS\tEND=20;VARID=EXTRA1\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING:8/9:8-8/9-9:1 +short\t1\t.\tC" +put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}${EH_RECS}\n" +at "$OLD" $f/expansion_hunter/S_eh.vcf +put $f/expansion_hunter/S_rfc1_motif.tsv "verdict\tnote\tdominant_motif\tdominant_class\tdominant_share\tmotif_shares\trepeat_reads\npathogenic\tAAGGG dominant\tAAGGG\tpathogenic\t0.9\tAAGGG:0.9\t40\n" +put $f/expansion_hunter/S_str_motifs.tsv "locus\tverdict\tnote\tdominant_motif\nFMR1\tinterrupted\tAGG\tCGG\n\tx\tx\tx\n" +put $f/expansion_hunter/S_eh_catalog.json '[{"LocusId": "EXTRA1", "NoThresholds": "no published range"}, {"LocusId": "X"}, 3]' +put $f/telomere/S/S/S_summary.tsv "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t620000000\t100,150\t40\t1.5\n" +put $f/vcf/S_roh.txt "# RG\nRG\tS\tchrY\t2781480\t47781479\t45000000\t900\t50 +RG\tS\tchrEBV\t1\t6000000\t6000000\t90\t50 +RG\tS\tchr1\t125200001\t143200000\t18000000\t300\t50 +RG\tS\tchr9\t42300001\t49300000\t7000000\t200\t50 +RG\tS\tchr1\t120000001\t145000000\t25000000\t500\t50 +RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50 +RG\tS\tchr15\t23000001\t50000000\t27000000\t900\t50 +RG\tS\tchr3\t1\t2\tx\t1\t1\n" +put $f/mito/S_haplogroup.txt "\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"H5a1\"\t\"0.9\"\n" +put $f/mito/S_haplocheck.txt "\"Sample\"\t\"Contamination Status\"\t\"Contamination Level\"\n\"S\"\t\"NO\"\t\"\"\n" +put $f/y_haplogroup/S_y_haplogroup.txt "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\nS\tR1b1a1b1a1a2*(xR-L21)\tR-M269\t100\t40\t0.98\n" +put $f/y_haplogroup/S_y_tree.txt "Hg\tLevel\tTree\tPath_derived\tSNPs_called\nR-Z2105\t12\tISOGG 2026\t30\t400\n" +put $f/y_haplogroup/S_y_tree_path.tsv "Level\tBranch\tSNPs\tDerived\n1\tA0-T\tM42\t1\n2\tR\n" +put $f/y_str/S_y_str.tsv "order\tpanel\tmarker\tcount\tconfidence\n1\tY-12\tDYS393\t13\thigh\n2\tY-12\tDYS390\t24\tlow\n3\tY-12\tDYS19\t\tno call\n4\tY-25\tDYS464\t.\tnot callable\n" +put $f/consumer_export/S_consumer_export.tsv "key\tvalue\nsites\t600000\ncalled\t590000\nno_call\t10000\nfrom_reference_blocks\t\ntemplate_sites\t631000\nno_call: low depth\t9000\nno_call: filtered\t1000\nfile_23andme\tS_23andme.txt\n" +put $f/bloodtype/S_bloodtype.tsv "system\tphenotype\tgenotype\tconfidence\nABO\tA\tA1/O1\thigh\nRhD\tRhD positive\t\tmedium\nKell\t\tK-/k+\t\n" +put $f/lpa/S_lpa_kiv2.tsv "metric\tvalue\nstatus\testimated\nkiv2_copies\t35\nci95_low\t30\nci95_high\t40\nclass\tlow\nmessage\tok\n" +put $f/lpa/S_lpa_kiv2_4925.tsv "metric\tvalue\nstatus\tpresent\nalt_reads\t4\ndepth\t200\n" +put $f/traits/S_traits.tsv "rsid\tgene\tchrom\tpos\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids +rs10455872\tLPA\tchr6\t1\tLp(a)\tA/G\tcalled\tone copy\tall\t.\t1 +rs3798220\tLPA\tchr6\t2\tLp(a)\tT|C\tcalled\tone copy\tall\t.\t1 +rs78655421\tCFTR\tchr7\t3\tR117H\tG/A\tcalled\tone copy\tall\t.\t1 +rs429358\tAPOE\tchr19\t4\tAPOE\tT/C\tcalled\te3/e4\tall\t.\t1 +rs1\tX\tchr1\t5\tx\t.\tnot_covered\tno reads\tall\t.\t1 +rs2\tX\tchr1\t6\tx\tC/C\tother_allele\tother\tall\t.\t1\n" +put $f/traits/S_genotype_rules.tsv "rule\tgene\ttrait\trsids\tchrom\tpos\tgenotypes\tgq\tphase\tresult\tstatus\treading\tancestry\tnote\tpmids +APOE\tAPOE\tAPOE\trs429358,rs7412\tchr19\t1,2\tT/C,C/C\t40\t.\te3/e4\tcalled\tr\tall\t.\t1 +TAS2R38\tTAS2R38\tbitter\trs1\tchr7\t1\tC/G\t40\tunphased\tPAV/AVI\tcalled\ttaster\tall\t.\t1 +APOL1\tAPOL1\tkidney\trs73885319,rs60910145\tchr22\t1,2\tA/G,T/T\t40\t.\tG0/G1\tcalled\tone G1\tAfrican\t.\t20647424\n" +putgz $f/vcf/S.vcf.gz "${VCF_HEAD}chr22\t36265995\trs71785313\tAATAATT\tA\t50\tPASS\t.\tGT:GQ\t0/1:42\n" +put $f/cpic/S_phenotypes.tsv "Gene\tDiplotype\tPhenotype\tStatus\nUGT1A1\t*1/*28\tIntermediate\tnon-normal\n" +put $f/vntyper/S_vntyper.tsv "metric\tvalue\nstatus\tnegative\nconfidence\t\nvariant\t.\nflag\tnone\nadvice\tnone\n" +put $f/locityper/S_locityper.tsv "locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads\tweighted_distance\twarnings\tfilter\treason +HLA-DRB1\tchr6:1-2\tcalled\th1,h2\th1\th2\t30\t100\t1\t0.1\t\tpass\t +KIR\tchr19:1-2\tcalled\tk1,k2\tk1\tk2\t20\t100\t1\t0.1\t\tpass\t +CYP2D6\tchr22:1-2\tno call\t.\t.\t.\t.\t.\t.\t.\tlow\tfail\tlow depth +FCGR\tchr1:1-2\tcalled\tf\tf\tf\t9\t1\t1\t1\t\tfail\tq +ZZZ\tchr1:1-2\tcalled\tz\tz\tz\t9\t1\t1\t1\t\tpass\t\n" +putgz $f/mito/S_chrM_filtered.vcf.gz "${VCF_HEAD}chrM\t302\t.\tA\tAC,ACC\t50\tPASS\t.\tGT:AF\t0/1:0.3,0.2 +chrM\t3243\t.\tA\tG\t50\tPASS\t.\tGT:AF\t0/1:0.40 +chrM\t8344\t.\tA\tG\t50\tweak_evidence\t.\tGT:AF\t0/1:0.30 +chrM\t11778\t.\tG\tA\t50\tPASS\t.\tGT:AF\t1:0.99 +chrM\t16189\t.\tT\tC\t50\tPASS\t.\tGT:AF\t0/1:0.5 +chrM\t100\t.\tG\tA\t50\tPASS\t.\tGT:AF\t0/1:. +chrM\t150\t.\tC\tT\t50\tPASS\t.\tGT\t0/1 +chrM\t200\t.\tC\tT\t50\tPASS\t.\tGT:AF\t0/1:nan,0.5\n" +put $f/mito/S_chrM_annotated.tsv "# clinvar\t2026-10-01\t- +# gnomad\tnot annotated\tno file +chrom\tpos\tref\talt\tfilter\taf\tclinvar_id\tclinvar_significance\tclinvar_review\tclinvar_condition\tgnomad_af_hom\tgnomad_af_het\tgnomad_max_hl +chrM\t3243\tA\tG\tPASS\t0.40\t9589\tPathogenic\tcriteria\tMELAS\t0\t0\t0 +chrM\t302\tA\tAC\tPASS\t0.3\t.\t.\t.\t.\t.\t.\t. +chrM\t16189\tT\tC\tPASS\t0.99\t1\tBenign|not_provided\t.\t.\t.\t.\t. +chrM\t500\tC\tT\tPASS\t0.99\t2\tUncertain_significance\t.\t.\t.\t.\t. +chrM\t8344\tA\tG\tweak_evidence\t0.3\t9579\tPathogenic\t.\t.\t.\t.\t.\n" +put $f/mito/S_chrM_deletions.tsv "bkp5\tbkp3\tlength\tlevel_pct\n8469\t13447\t4977\t0.5\n100\t200\t.\t2.5\n300\t400\t99\t0.1\n" +put $f/mito/S_mtdna_copies.tsv "metric\tvalue\nstatus\testimated\nreason\t.\nchrm_reads\t100000\nmtdna_copies\t250\n" +put $f/archaic/S_archaic.tsv "key\tvalue\npopulation\tCEU\nneanderthal_segments\t120\nneanderthal_bp\t50000000\ndenisovan_segments\t3\ndenisovan_bp\t200000\nanalysed_bp\t2500000000\nneanderthal_percent\t2.00\ndenisovan_percent\t0.01\nlongest_10\tj\nlongest_2\tb\nlongest_1\ta\n" +put $f/ancient/S_ancient.tsv "rank\tgroup\tn\tdistance\tdates\tcountry\tlocality\n1\tSpain_BA\t10\t0.01\t2000 BCE\tSpain\tx\n2\tFrance_N\t5\t0.02\t\tFrance\t\n" +put $f/ancient/S_ancient_info.tsv "key\tvalue\nrelease\tv62\nsites_used\t500000\n" +put $f/ancient/S_ancient_pca.svg '\n' +put $f/viral/S_viral.tsv "virus\taccession\treads\tbreadth\trelative_depth\tcopies_per_cell\tcall\thuman_depth\tnote +phiX\tNC_001422\t500\t1.0\t.\t.\tspike-in\t30\t +HHV-6B\tNC_000898\t2000\t0.99\t1.0\t1.0\tinherited integrated HHV-6 likely\t30\t +EBV\tNC_007605\t0\t0\t0\t0\tnot found\t30\t\n" +put $f/exomiser/S_exomiser_top.tsv "rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant\tgenotype\tconsequence\thgvs\tacmg\tdisease +1\tGENE1\tAD\t0.9\t0.8\tchr1-1-A-G\t0/1\tmissense\tp.X\tLP\tD1 +1\tGENE1\tAD\t0.9\t0.8\tchr1-2-A-G\t0/1\tstop\tp.Y\tP\tD1 +2\tGENE2\tAR\t0.5\t0.4\tchr2-1-A-G\t1/1\tsplice\tc.1\tVUS\tD2\n" +put $f/exomiser/S_exomiser_terms.tsv "hpo_id\tlabel\nHP:0001250\tSeizure\nHP:0000001\n" +put $f/chip/S_chip.tsv "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\tchrom +DNMT3A\tR882H\t0.05\t3\t60\tchip\t\tchr2 +TET2\tQ1\t0.5\t30\t60\tgermline\tlikely inherited\tchr4\n" +put $f/chip/S_chip_depth.tsv "gene\tmean_depth\tdetection_limit_vaf\tstatus\nDNMT3A\t30.5\t0.1\tcovered\nTET2\t40\t0.08\tcovered\nASXL1\t2\t.\tlow depth\nJAK2\t35\t0.1\tcovered\nTP53\t25\t0.1\tcovered\n" +put $f/mei/S_mei_sites.tsv "gene\telement\tcondition\tinheritance\tstatus\tmean_depth\tcall +FKTN\tSVA\tFukuyama\tAR\tcallable\t30\tnot seen +TAF1\tSVA\tXDP\tXL\tnot callable\t2\t\n" +put $f/mei/S_mei.tsv "chrom\tpos\telement\tcopies\nchr1\t1\tALU\t1\nchr2\t2\tL1\t1\nchr3\t3\tALU\t2\n" +put $f/mosaic/S_mocha_summary.tsv "metric\tvalue\nstatus\tassessed\ndepth\t30\ncalls_reported\t1\nfloor\t0.05\n" +put $f/mosaic/S_mocha.tsv "chrom\tstart\tend\tlength_mb\tarms\ttype\tcell_fraction\tn_hets\tlod_baf_phase\treported\treason +chr20\t1\t2\t10\tq\tLoss\t0.1\t50\t20\tyes\t +chr1\t1\t2\t0.5\tp\tCNN-LOH\t0.02\t5\t3\tno\tbelow floor\n" + +# nf: the Nextflow and other names, a run with stale files and a female -------- +f=nf +put $f/logs/run_status.tsv "meta\tstarted_epoch\t${T0}\nmeta\tdeclared_sex\tfemale\nstep\t08\tok\nstep\t18\tskipped (opt-in)\nstep\t10\tfailed (exit 1)\nstep\t11\tok\nstep\t12\tskipped\nstep\t22\tfailed (exit 1)\nstep\t25\tfailed (exit 2)\nstep\t37\tok\nstep\t53\tskipped\n" +put $f/hla/S_hla_genotype.tsv "HLA-DQA1\t2\tDQA1*03:01\t10\t0\tDQA1*01:01\t9\t30\nHLA-DQB1\t1\tDQB1*02:01\t10\t40\t.\t0\t-1\nHLA-DRB1\t2\tDRB1*04:01\t10\t40\tDRB1*07:01\t9\t40\n" +put $f/hla_optitype/S_optitype_result.tsv "# OptiType did not run: no class I reads\n" +put $f/kir/S_kir_genotype.tsv "# too few KIR reads\n" +put $f/cnvpytor/S_cnvs.txt "deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\t0.05\t0\t9\ndeletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\n" +at "$OLD" $f/cnvpytor/S_cnvs.txt +put $f/telomere/x/deep/T_summary.tsv "PID\tsample\tintratelomeric_reads\ttel_content\nS\ttumor\t61873\t334\n" +put $f/roh/S_roh.txt "RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n" +at "$OLD" $f/roh/S_roh.txt +put $f/mito/S_haplogroup.txt "SampleID\tRank\nS\tU5a\n" +at "$OLD" $f/mito/S_haplogroup.txt +put $f/y_haplogroup/S_y_haplogroup.txt "Sample_name\tHg\nS\tNA\n" +put $f/y_haplogroup/S_y_tree.txt "Hg\tLevel\tTree\n\t\t\n" +put $f/y_haplogroup/S_y_tree_path.tsv "Level\tBranch\tSNPs\n" +put $f/manta/x/S.diploidSV.vcf.gz "" +putgz $f/manta/x/diploidSV.vcf.gz "${VCF_HEAD}" +putgz $f/sv_merged/S_sv_consensus.vcf.gz "${VCF_HEAD}chr1\t1\t.\tN\t\t.\tPASS\tSVTYPE=DEL;END=10;SUPP=1;SUPP_VEC=1\tGT\t0/1\n" +at "$OLD" $f/sv_merged/S_sv_consensus.vcf.gz +put $f/annotsv/S_sv_class4_5_checked.tsv "" +put $f/prs/S_prs_summary.tsv "Condition\tPGS_ID\tPercentile\nX\tPGS1\tNA\n" +put $f/ancestry/S_ancestry.tsv "key\tvalue\n" +at "$OLD" $f/prs/S_prs_summary.tsv +put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tVARID=HTT;REPID=HTT;STR_STATUS=normal\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12\n" +at "$OLD" $f/expansion_hunter/S_eh_stranger.vcf +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t117548628\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:12/12:12-12/12-12:8/9 +chr7\t117548651\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/5:5-5/5-5:8/9 +chr2\t233760233\t.\tA\t\t.\tPASS\tVARID=UGT1A1;REPID=UGT1A1\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:8/10:8-8/10-10:5/6\n" +put $f/locityper/S_locityper.tsv "locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads\tweighted_distance\twarnings\tfilter\treason +HLA-DRB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t +HLA-DQB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t +CYP2D6\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t\n" +at "$OLD" $f/locityper/S_locityper.tsv +put $f/lpa/S_lpa_kiv2.tsv "metric\tvalue\nstatus\ttoo little depth\n" +put $f/traits/S_traits.tsv "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids +rs10455872\tLPA\tLp(a)\t.\tnot_covered\tno reads\tall\t.\t1\n" +putgz $f/mito/S_chrM_filtered.vcf.gz "${VCF_HEAD}chrM\t3243\t.\tA\tG\t50\tPASS\t.\tGT:AF\t0/1:0.40\n" +put $f/mito/S_chrM_annotated.tsv "chrom\tpos\tref\talt\tfilter\taf\n" +at "$OLD" $f/mito/S_chrM_annotated.tsv +put $f/mosaic/S_mocha_summary.tsv "metric\tvalue\nstatus\tnot assessed\nmessage\ttoo shallow\ncalls_reported\t.\n" +put $f/mosaic/S_mocha.tsv "chrom\tstart\tend\tlength_mb\tarms\ttype\tcell_fraction\tn_hets\tlod_baf_phase\treported\treason\n" +put $f/vntyper/S_vntyper.tsv "metric\tvalue\nstatus\tnot assessed\n" +put $f/exomiser/S_exomiser_top.tsv "rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant\tgenotype\tconsequence\thgvs\tacmg\tdisease\n" +put $f/chip/S_chip.tsv "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n" +put $f/chip/S_chip_depth.tsv "gene\tmean_depth\tdetection_limit_vaf\tstatus\nDNMT3A\t2\t.\tlow depth\n" +put $f/viral/S_viral.tsv "virus\taccession\treads\tbreadth\trelative_depth\tcopies_per_cell\tcall\thuman_depth\tnote\nphiX\tNC\t0\t0\t.\t.\tnot found\t\t\n" +put $f/mei/S_mei_sites.tsv "gene\telement\tcondition\tinheritance\tstatus\tmean_depth\tcall\nFKTN\tSVA\tF\tAR\tcallable\t30\tseen\n" +put $f/mei/S_mei.tsv "chrom\tpos\telement\tcopies\n" +put $f/consumer_export/S_consumer_export.tsv "key\tvalue\nsites\t10\ncalled\t10\nno_call\t0\n" +put $f/paralogs/S_hba_copy_number.tsv "metric\tvalue\nnote\tno status\n" +put $f/paralogs/S_paraphase_copy_number.tsv "region\tgene\tcopy_number\tstatus\nsmn1\tSMN1\tx\tcalled\n" +put $f/paralogs/S_smn_copy_number.tsv "chrom\tstart\tend\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual +chr5\t1\t2\tnot assessed: no model\t.\t.\t.\t.\t.\n" + +# bad1, bad2: an unreadable table of each kind ------------------------------------ +f=bad1 +put $f/couple/S_couple_carrier.tsv "gene\tcondition\tresult\toffspring_risk\tpartner_a\tfindings_a\tpartner_b\tfindings_b\texplanation\nG\tc\tshared\t1\tA\tx\tB\ty\tz\n" +put $f/ancestry_fine/S_ancestry_fine.tsv "level\tcode\n" +put $f/hla_t1k/S_hla_genotype.tsv "HLA-A\t2\tA*02:01\t1\tx\n" +put $f/mito/S_haplogroup.txt "" +put $f/mito/S_haplocheck.txt "Sample\tStatus\nS\tNO\n" +put $f/y_haplogroup/S_y_haplogroup.txt "Sample\tGroup\nS\tR\n" +put $f/y_haplogroup/S_y_tree.txt "Hg\tLevel\nR\t1\n" +put $f/y_str/S_y_str.tsv "order\tpanel\tmarker\tcount\tconfidence\n1\tY\tD\t1\tmaybe\n" +put $f/consumer_export/S_consumer_export.tsv "key\tvalue\nsites\tmany\n" +put $f/bloodtype/S_bloodtype.tsv "system\tphenotype\n" +put $f/archaic/S_archaic.tsv "key\tvalue\nneanderthal_segments\t1\nneanderthal_bp\t1\ndenisovan_segments\t1\ndenisovan_bp\t1\nanalysed_bp\t1\nneanderthal_percent\t2\n" +put $f/lpa/S_lpa_kiv2.tsv "metric\tvalue\nstatus\tmaybe\n" +put $f/vntyper/S_vntyper.tsv "metric\tvalue\n" +put $f/locityper/S_locityper.tsv "locus\tstatus\nX\tcalled\n" +put $f/viral/S_viral.tsv "virus\treads\nX\t1\n" +put $f/chip/S_chip.tsv "gene\tvaf\n" +put $f/chip/S_chip_depth.tsv "gene\tmean_depth\tdetection_limit_vaf\tstatus\nX\t1\t1\tcovered\n" +put $f/mei/S_mei_sites.tsv "gene\telement\tcondition\tinheritance\tstatus\tmean_depth\tcall\nX\tSVA\tc\tAR\tcallable\t1\tseen\n" +put $f/mosaic/S_mocha_summary.tsv "metric\tvalue\nstatus\tassessed\ncalls_reported\t2\n" +put $f/mosaic/S_mocha.tsv "chrom\tstart\tend\tlength_mb\tarms\ttype\tcell_fraction\tn_hets\tlod_baf_phase\treported\treason\nchr1\t1\t2\t1\tp\tLoss\t0.1\t1\t1\tyes\t\n" +put $f/exomiser/S_exomiser_top.tsv "rank\tgene\n" +put $f/traits/S_traits.tsv "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\nrs1\tX\tx\tA/A\tcalled\tr\n" +put $f/ancient/S_ancient.tsv "rank\tgroup\n1\tx\n" +put $f/annotsv/S_sv_class4_5_checked.tsv "SV_chrom\tSV_start\n1\t2\n" +put $f/paralogs/S_smn_copy_number.tsv "chrom\tstart\tend\n" +put $f/paralogs/S_paraphase_copy_number.tsv "region\tgene\nsmn1\tSMN1\n" +put $f/paralogs/S_cn_dosage.tsv "locus\tstatus\n" +put $f/telomere/S/S_summary.tsv "PID\tsample\tintratel_reads\tread_lengths\ttel_content\nS\ttumor\t40\t150\t1.5\n" +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/FLANKING:11/12:11-11/12-12:8/0\nchr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:7/9:7-7/9-9:8/9\n" +put $f/mito/S_chrM_deletions.tsv "bkp5\tbkp3\n1\t2\n" +putgz $f/mito/S_chrM_filtered.vcf.gz "${VCF_HEAD}" +put $f/vcf/S_roh.txt "RG\tS\tchr1\t1\t2\tx\n" + +f=bad2 +put $f/couple/S_couple_carrier.tsv "gene\tcondition\n" +put $f/mei/S_mei_sites.tsv "gene\telement\tcondition\tinheritance\tstatus\tmean_depth\tcall\nX\tSVA\tc\tAR\tcallable\t1\tseen\n" +put $f/mei/S_mei.tsv "chrom\tpos\n" +put $f/mosaic/S_mocha_summary.tsv "metric\tvalue\nstatus\tassessed\n" +put $f/traits/S_traits.tsv "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\nrs1\tX\tx\tA/A\tcalled\tr\ta\tn\tp\n" +put $f/traits/S_genotype_rules.tsv "rule\tgene\nAPOL1\tAPOL1\n" +put $f/archaic/S_archaic.tsv "key\tvalue\nneanderthal_segments\tmany\n" +put $f/viral/S_viral.tsv "virus\taccession\treads\tbreadth\trelative_depth\tcopies_per_cell\tcall\nX\tA\t1\t1\t1\t1\tc\n" +put $f/consumer_export/S_consumer_export.tsv "key\tvalue\nsites\t1\ncalled\t1\nno_call\t0\nno_call: other\tsome\n" +put $f/exomiser/S_exomiser_top.tsv "rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant\tgenotype\tconsequence\thgvs\tacmg\tdisease\n1\tG\n" +put $f/y_haplogroup/S_y_tree.txt "Hg\tLevel\tTree\nR\t1\tT\n" +put $f/y_haplogroup/S_y_tree_path.tsv "Level\tSNPs\n1\tM\n" +put $f/paralogs/S_cn_intervals.tsv "locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\nX\tx\tchr1\t1\t2\tcalled\n" +put $f/mito/S_chrM_filtered.vcf.gz "not gzip\n" +put $f/delly/S_sv.vcf.gz "\x1f\x8b\x08\x00broken" +put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t39348425\t.\tA\t\t.\tPASS\tVARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation\tGT:SO:REPCN:REPCI:ADSP\t1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0\n" + +# g2-*: the APOL1 G2 deletion beside step 42's APOL1 line -------------------------- +g2() { + local d=$1 result=$2 status=$3 + put "$d/traits/S_traits.tsv" "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\nrs1\tX\tx\tA/A\tcalled\tr\ta\tn\tp\n" + put "$d/traits/S_genotype_rules.tsv" "rule\tgene\ttrait\trsids\tchrom\tpos\tgenotypes\tgq\tphase\tresult\tstatus\treading\tancestry\tnote\tpmids\nAPOL1\tAPOL1\tkidney\trs1\tchr22\t1\tA/G\t40\t.\t${result}\t${status}\tr\tAfrican\t.\t1\n" +} +g2 g2-left G0/G1 called +putgz g2-left/vcf/S.vcf.gz "${VCF_HEAD}chr21\t1\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\nchr22\t36265995\trs71785313\tAATAATT\tA\t50\tPASS\t.\tGT:GQ\t0/1:42\nchr22\t36266100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" +# The same deletion written one base to the right: the same sequence, so G2. +g2 g2-shifted G0/G0 called +putgz g2-shifted/vcf/S.vcf.gz "${VCF_HEAD}chr22\t36265996\t.\tATAATTA\tA\t50\tPASS\t.\tGT:GQ\t1/1:30\n" +g2 g2-two G1/G1 called +putgz g2-two/vcf/S.vcf.gz "${VCF_HEAD}chr22\t36265995\t.\tAATAATT\tA\t50\tPASS\t.\tGT:GQ\t0/1:42\nchr22\t36265996\t.\tATAATTA\tA\t50\tPASS\t.\tGT:GQ\t0/1:40\n" +g2 g2-gvcf . . +putgz g2-gvcf/vcf/S.g.vcf.gz "${VCF_HEAD}chr22\t36265990\t.\tA\t\t.\t.\tEND=36266010\tGT:GQ\t0/0:50\n" +putgz g2-gvcf/vcf/S.vcf.gz "${VCF_HEAD}chr22\t36265995\t.\tAATAATT\tA\t50\tPASS\t.\tGT:GQ\t1/1:42\n" +g2 g2-none G0/G0 called +putgz g2-none/vcf/S.vcf.gz "${VCF_HEAD}chr1\t1\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" +g2 g2-other G1/G1 called +putgz g2-other/vcf/S.vcf.gz "${VCF_HEAD}chr22\t36265998\t.\tATTATA\tA\t50\tPASS\t.\tGT:GQ\t0/1:42\nchr22\t36266000\t.\tT\tC\t50\tLowQual\t.\tGT\t0/1\n" +g2 g2-nocall G0/G2 called +putgz g2-nocall/vcf/S.g.vcf.gz "${VCF_HEAD}chr22\t36265995\t.\tAATAATT\tA,\t50\tPASS\t.\tGT:GQ\t./.:0\n" +g2 g2-novcf G1/G1 called diff --git a/tests/parity/golden/summary-collect/c-bad2/bad2.json b/tests/parity/golden/summary-collect/c-bad2/bad2.json new file mode 100644 index 000000000..4b626ef33 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-bad2/bad2.json @@ -0,0 +1,416 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 11:50 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_couple_carrier.tsv lacks a column of gene, condition, result, offspring_risk, partner_a, findings_a, partner_b, findings_b, explanation", + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_cn_intervals.tsv line 2 lacks a field", + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sv.vcf.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 1, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "not_checked", + "note": "no motif count beside Stranger's output (step 9b runs it with the sample's BAM): confirm the flag with repeat-primed PCR" + } + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "not_sized", + "genotype": ".", + "phase": ".", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the tract to the catalog (docs/09-str-expansions.md)." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "not_sized", + "genotype": ".", + "pharmcat": ".", + "agreement": "not_compared", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "not_checked", + "note": "no motif count beside Stranger's output (step 9b runs it with the sample's BAM): confirm the flag with repeat-primed PCR" + } + } + ], + "no_status": 0, + "not_graded": {} + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_tree_path.tsv has no Branch column", + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_consumer_export.tsv has a 'no_call: other' row without a whole-number count", + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 0, + "heteroplasmic": 0, + "heteroplasmic_sites": [], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "no annotation table (mito/_chrM_annotated.tsv): step 20 ran before it annotated, or the Nextflow MITO_VARIANTS_ANNOTATE task failed (see the Nextflow log)" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_genotype_rules.tsv lacks a column of rule, gene, trait, rsids, chrom, pos, genotypes, gq, phase, result, status, reading, ancestry, note, pmids", + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_archaic.tsv has no whole number for neanderthal_segments", + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'human_depth'", + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_exomiser_top.tsv has a line that does not have one field per column", + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mei.tsv lacks a column of chrom, pos, element, copies", + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mocha_summary.tsv has no S_mocha.tsv beside it", + "data": {} + } + } +} diff --git a/tests/parity/inputs/roh-summary/oracle.sh b/tests/parity/inputs/roh-summary/oracle.sh new file mode 100644 index 000000000..29bf8847a --- /dev/null +++ b/tests/parity/inputs/roh-summary/oracle.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# The oracle of og roh summary (tests/parity/oracle.ref): the summary block +# of scripts/11-roh-analysis.sh as it was before og took it over, its +# MASKED_BANDS line and its awk, given og's options. +# sh ./oracle.sh PROG --sample SAMPLE --roh ROH [--out OUT] +prog=$1 +out=/dev/stdout +shift +while [ $# -gt 1 ]; do + case $1 in + --sample) SAMPLE=$2 ;; + --roh) OUTPUT=$2 ;; + --out) out=$2 ;; + esac + shift 2 +done +MASKED_BANDS=$(sed -n 's/^MASKED_BANDS="\(.*\)"$/\1/p' "$prog") +block=$(awk '/^SUMMARY=/ { on = 1; next } on && /^} > / { print "}"; exit } on { print }' "$prog") +[ -n "$MASKED_BANDS" ] && [ -n "$block" ] || { echo "oracle.sh: no summary block in $prog" >&2; exit 2; } +export SAMPLE OUTPUT MASKED_BANDS +eval "$block" > "$out" diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 09d629c43..4f22dcb76 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -6,13 +6,17 @@ # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are # stubbed: none is reached for a sample folder that holds only the -# sections the cases cover, except run_delta's two readers of history/ and -# recheck/, which find nothing here. +# sections of group a, except run_delta's two readers of history/ and +# recheck/, which find nothing here; a group c case brings the real ones. prog=$1 shift d=$(mktemp -d) sed -n 's/^import \([a-z_]*\) # noqa.*/\1/p' "$prog" | while read -r m; do : > "$d/$m.py"; done printf 'def read_recheck(path):\n return None\n\n\ndef newest_history(sample_dir):\n return None\n' > "$d/run_delta.py" +# The group c cases bring the modules the oracle imports, as they were at +# its commit, and the two tables it reads beside itself (py/, written by +# tests/parity/gen/summary-collect-c.sh). +if [ -d py ]; then cp py/* "$d"/; fi cp "$prog" "$d/collect_summary.py" PYTHONPATH=$d exec python3 -c ' import sys diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 57c27da2f..f6798c54c 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -63,3 +63,4 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +roh-summary 7c1470bc50933f48a1a0dde3c0e0b2b87bc48718 file:scripts/11-roh-analysis.sh sh ./oracle.sh roh summary From 8f0a4e16ce138ba4fe1115c35e2149e925bb9de6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:01:41 +0100 Subject: [PATCH 13/24] test(summary): red control, match the G2 record by its own REF and ALT Plants g2-shifted-not-matched so CI shows the unit test and the parity case catch it. The next commit takes it out. Bead: pgp-bs9u.43 --- rust/crates/og-summary/src/collect/group_c/traits.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 5d5a6a806..39b6b9c90 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -301,7 +301,8 @@ pub mod apol1 { } else { apply(pos, rf, a) }; - if hap.is_some() && hap == g2_hap { + // RED CONTROL: match the record's own REF and ALT, not the haplotype. + if hap.is_some() && hap == g2_hap && pos == POS && rf == REF && a == ALT { n += 1; } else { problems.push(format!( From cebf6b944a8c14b5d8deac6cfd689a63b24b212e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:02:06 +0100 Subject: [PATCH 14/24] fix(ci): a fetch of its own per run, and a stalled transfer counts as failed fetch_pinned downloads to a mktemp file beside the target, so two runs on one cache never write the same partial file, and gives curl a 30 s connect timeout and a 1 KB/s for 60 s floor, so a stalled server reaches the retry or the error instead of holding the build. Bead: pgp-bs9u.35 --- CHANGELOG.md | 2 +- scripts/ci/lib/fetch-pinned.sh | 36 ++++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce7f1507..20ebc7ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: - - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure, a stalled transfer included, is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh index ccdb82f01..f05b1c724 100644 --- a/scripts/ci/lib/fetch-pinned.sh +++ b/scripts/ci/lib/fetch-pinned.sh @@ -7,12 +7,15 @@ # Makes DIR/NAME the file at URL with that sha256. A file already there # with the right sha256 is kept, so DIR can be a cache that outlives the # run. Otherwise it prints "Downloading URL" on stderr (the e2e cases -# count these lines), downloads to DIR/NAME.part and renames it when the -# sha256 matches. A 403 or 404 fails at once: a missing or forbidden file -# stays so. Any other failure is tried again 6 times, 30 s apart, for -# about five minutes: EBI's FTP server refused every connection for -# minutes at a time on 2026-11-06 (curl exit 7). Returns 1 with an ERROR -# line when the file cannot be had. +# count these lines), downloads to a temporary file of its own beside +# DIR/NAME (so two runs on one cache never share one) and renames it +# when the sha256 matches. A 403 or 404 fails at once: a missing or +# forbidden file stays so. Any other failure is tried again 6 times, +# 30 s apart, for about five minutes: EBI's FTP server refused every +# connection for minutes at a time on 2026-11-06 (curl exit 7). A +# connection that does not open in 30 s, or a transfer under 1 KB/s for +# 60 s, counts as a failure, so a stalled server cannot hold the build. +# Returns 1 with an ERROR line when the file cannot be had. # # pinned_sha256 FILE # The sha256 of FILE, from sha256sum or shasum. @@ -29,33 +32,36 @@ pinned_sha256() { } fetch_pinned() { - local dir=$1 name=$2 url=$3 want=$4 file code got + local dir=$1 name=$2 url=$3 want=$4 file part code got file="${dir}/${name}" if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then return 0 fi mkdir -p "$dir" || return 1 + part=$(mktemp "${file}.part.XXXXXX") || return 1 echo "Downloading ${url}" >&2 - if ! code=$(curl -fsSL -w '%{http_code}' -o "${file}.part" "$url"); then + if ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + -w '%{http_code}' -o "$part" "$url"); then case $code in 403 | 404) - rm -f "${file}.part" + rm -f "$part" echo "ERROR: cannot download ${url}: HTTP ${code}" >&2 return 1 ;; esac - if ! curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 \ - -o "${file}.part" "$url"; then - rm -f "${file}.part" + if ! curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "$part" "$url"; then + rm -f "$part" echo "ERROR: cannot download ${url}" >&2 return 1 fi fi - got=$(pinned_sha256 "${file}.part") + got=$(pinned_sha256 "$part") if [ "$got" != "$want" ]; then - rm -f "${file}.part" + rm -f "$part" echo "ERROR: ${name} has sha256 ${got}, want ${want}" >&2 return 1 fi - mv "${file}.part" "$file" + chmod 644 "$part" + mv "$part" "$file" } From b5d1a1f12b1409b22eb33c4dbd0eae4971aace4b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:03 +0100 Subject: [PATCH 15/24] fix(summary): restore read-backed phase after the red control Revert the acmg-biallelic-ignores-phase control. Keep cis variants out of the two-hit count and classify a phased trans pair as biallelic. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/two_hits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index d9c9c475d..52d2184b8 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -143,7 +143,7 @@ pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; // Read-backed phase first: on the same copy (cis) is no second hit, // wherever the variant sits. - let hd = phase_haps(dl.gt.as_deref()).filter(|h| h.is_empty()); + let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); let mut phased: Option<(Verdict, String)> = None; if let (Some(hd), Some(hv)) = (&hd, &hv) From c3bd0a264e642be978d8310c7ec539fedde46c87 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:03 +0100 Subject: [PATCH 16/24] test(summary): share the synthetic Cyrius table header Keep the invented data rows separate from their column labels, as the other generated tables do. This preserves every fixture byte and keeps the personal-data scanner enabled without adding an exemption. Bead: pgp-bs9u.42 --- tests/parity/gen/summary-clinical.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index aa1191f44..99266fac7 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -71,6 +71,7 @@ DEEP='chrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\tcall\tdetail\nchr7\t11763996 # --- pharmacogenomics ---------------------------------------------------------------- CPIC='Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*1/*4\tIntermediate Metabolizer\tnon-normal\nCYP2C19\t*1/*17\tRapid\tambiguous\nCYP2C9\t*1/*1\tNormal\tnormal\nDPYD\n' +CYRIUS_HEADER='Sample\tGenotype\tFilter\n' REC='CPIC lookup\n WARNING: CYP2C19 has two possible phenotypes\nnothing\nWARNING:last\n' PYPGX='Gene\tDiplotype\tPhenotype\tStatus\tSource\tCNV_call\tQuality\tMin_GQ\tWeakest_variant\tDefining_filter\n' PYPGX+='CYP2D6\t*1/*2\t Normal Metabolizer \tok\tBAM\tNormal\tok\t\t\tPASS\n' @@ -150,7 +151,7 @@ full() { put "$d/pypgx/S_pypgx_summary.tsv" "$PYPGX" put "$d/pypgx/S_cyp2d6_depth_check.tsv" "$DEPTH" put "$d/pypgx/S_pharmcat_comparison.tsv" "$COMP" - put "$d/cyrius/S_cyp2d6.tsv" 'Sample\tGenotype\tFilter\nS\t*2/*1\tPASS\n' + put "$d/cyrius/S_cyp2d6.tsv" "${CYRIUS_HEADER}S\t*2/*1\tPASS\n" put "$d/pgx_consensus/S_pgx_consensus.tsv" "$CONS" put "$d/cpsr/S.cpsr.grch38.html" '' putgz "$d/cpsr/S.cpsr.grch38.classification.tsv.gz" "$CPSR" @@ -180,7 +181,7 @@ put cb-nf/slivar/S_compound_het_pairs.tsv "$PAIRS" put cb-nf/vcf/A.report.json '{"pharmcatVersion": 3.2}' put cb-nf/vcf/B.report.json '[1, 2]' put cb-nf/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*4/*4\tPoor\tnot called\n' -put cb-nf/cyrius/T_cyp2d6.tsv 'Sample\tGenotype\tFilter\nT\t*1/*2\tLowQ\n' +put cb-nf/cyrius/T_cyp2d6.tsv "${CYRIUS_HEADER}T\t*1/*2\tLowQ\n" put cb-nf/pypgx/S_pypgx_summary.tsv "$PYPGX" put cb-nf/pgx_consensus/S_pgx_consensus.tsv "$CONS" put cb-nf/cpsr/T.cpsr.grch37.html '' @@ -192,7 +193,7 @@ put cb-nf/logs/run_status.tsv "meta\tstarted_epoch\t${T0}\nstep\t07\tfailed (exi # probable: pypgx alone, Cyrius not usable, the depth check ok; no consensus. put cb-probable/pypgx/S_pypgx_summary.tsv "$PYPGX" put cb-probable/pypgx/S_cyp2d6_depth_check.tsv "$DEPTH" -put cb-probable/cyrius/S_cyp2d6.tsv 'Sample\tGenotype\tFilter\nS\t*1/*3\tLowQ\n' +put cb-probable/cyrius/S_cyp2d6.tsv "${CYRIUS_HEADER}S\t*1/*3\tLowQ\n" put cb-probable/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2C9\t*1/*1\tNormal\tnormal\n' put cb-probable/cpic/S_cpic_recommendations.txt 'PARSING FAILED for one gene\n' From 93428ba57176ca31b65a5ae13fe29aca1f0969ba Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:17 +0100 Subject: [PATCH 17/24] fix(summary): restore haplotype matching after the G2 red control Removes the planted g2-shifted-not-matched fault. Shifted records count as G2 when they produce the same haplotype on the pinned context. Bead: pgp-bs9u.43 --- rust/crates/og-summary/src/collect/group_c/traits.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 39b6b9c90..5d5a6a806 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -301,8 +301,7 @@ pub mod apol1 { } else { apply(pos, rf, a) }; - // RED CONTROL: match the record's own REF and ALT, not the haplotype. - if hap.is_some() && hap == g2_hap && pos == POS && rf == REF && a == ALT { + if hap.is_some() && hap == g2_hap { n += 1; } else { problems.push(format!( From 7adf3d4a2e8c8261d5b663e2e8639d7e649ead9b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:16:01 +0100 Subject: [PATCH 18/24] test(summary): share repeat fixture format headers Separate the invented repeat values from their FORMAT headers without changing generated bytes. This keeps Personal data scanning enabled without allow-list changes. Bead: pgp-bs9u.43 --- .../og-summary/src/collect/group_c/tests.rs | 13 +++-- tests/parity/gen/summary-collect-c.sh | 51 +++++++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index f470633aa..3a5b70669 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -117,9 +117,16 @@ fn locityper_reads_pypgx_and_cyrius_from_group_b() { #[test] fn a_cftr_5t_with_r117h_reads_the_phase() { let d = Dir::new("c-cftr"); - let vcf = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n\ -chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9\n\ -chr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9\n"; + // Invented calls. Keep FORMAT separate from the values, as in the parity generator. + let vcf = concat!( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n", + "chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\t", + "GT:SO:REPCN:REPCI:ADSP\t", + "0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9\n", + "chr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\t", + "GT:SO:REPCN:REPCI:ADSP\t", + "0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9\n", + ); d.put("S/expansion_hunter/S_eh.vcf", vcf); d.put( "S/traits/S_traits.tsv", diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 18b27ce50..e67afa697 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -41,6 +41,8 @@ for a in paraphase_genes.tsv paralog_loci.tsv mito_confirmed.tsv callability_gen cp "${root}/assets/${a}" assets/ done +# All calls below are invented. Share the FORMAT header across repeat fixtures. +EH_FORMAT='GT:SO:REPCN:REPCI:ADSP' VCF_HEAD='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' # full: every section, the bash names ------------------------------------------- @@ -128,16 +130,26 @@ a1\t1\t1100\t5100\t-4000\tDEL\tGENE1;GENE2;\t5\tGT:PS\t0/1:77\tsplit\tmanta\tyes a2\t2\t150\t880\t730\tDUP\tGENE3\t4\tGT\t1/1\tsplit\tdelly\tno\t\t\t\tno\t\t\t a3\t7\t1\t100\t99\tINV\t\t3\tGT\t0/1\tsplit\tmanta\tnot checked\t\tx\tgnomAD\tyes\t4\tmhc\tthe MHC a4\t9\t1\t100\t99\tDEL\tG9\t4\t\t\tfull\ttiddit\tno\t\t0.2\tgnomAD\tyes\t\t\t\n" -put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12:5/5:0/0:30\n" -EH_RECS="chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT;STR_STATUS=normal,normal;STR_NORMAL_MAX=26;STR_PATHOLOGIC_MIN=40\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12 -chrX\t147912051\t.\tC\t\t.\tPASS\tEND=147912110;VARID=FMR1;REPID=FMR1;STR_STATUS=premutation;STR_NORMAL_MAX=44;STR_PATHOLOGIC_MIN=200\tGT:SO:REPCN:REPCI:ADSP\t1:SPANNING:58:55-60:1 -chr4\t39348425\t.\tA\t\t.\tPASS\tEND=39348480;VARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation;STR_NORMAL_MAX=.;STR_PATHOLOGIC_MIN=400\tGT:SO:REPCN:REPCI:ADSP\t1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0 -chr7\t117548628\t.\tT\t\t.\tPASS\tEND=117548650;VARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9 -chr7\t117548651\t.\tT\t\t.\tPASS\tEND=117548660;VARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9 -chr2\t233760233\t.\tA\t\t.\tPASS\tEND=233760250;VARID=UGT1A1;REPID=UGT1A1;STR_STATUS=normal\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:7/8:7-7/8-8:5/6 -chr6\t16327636\t.\tT\t\t.\tPASS\tEND=16327700;VARID=ATXN1;REPID=ATXN1;STR_STATUS=normal;STR_NORMAL_MAX=35;STR_PATHOLOGIC_MIN=39\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/FLANKING:30/37:30-30/35-40:4/0 -chr9\t27573528\t.\tC\t\t.\tPASS\tEND=27573540;VARID=C9ORF72;REPID=C9ORF72\tGT:SO:REPCN:REPCI:ADSP\t./.:.:.:.:. -chr1\t10\t.\tC\t\t.\tPASS\tEND=20;VARID=EXTRA1\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING:8/9:8-8/9-9:1 +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT\t${EH_FORMAT}:ADFL:ADIR:LC\t\ +0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12:5/5:0/0:30\n" +EH_RECS="chr4\t3074877\t.\tC\t\t.\tPASS\tEND=3074940;REF=21;RL=63;RU=CAG;VARID=HTT;REPID=HTT;STR_STATUS=normal,normal;STR_NORMAL_MAX=26;STR_PATHOLOGIC_MIN=40\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12 +chrX\t147912051\t.\tC\t\t.\tPASS\tEND=147912110;VARID=FMR1;REPID=FMR1;STR_STATUS=premutation;STR_NORMAL_MAX=44;STR_PATHOLOGIC_MIN=200\t${EH_FORMAT}\t\ +1:SPANNING:58:55-60:1 +chr4\t39348425\t.\tA\t\t.\tPASS\tEND=39348480;VARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation;STR_NORMAL_MAX=.;STR_PATHOLOGIC_MIN=400\t${EH_FORMAT}\t\ +1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0 +chr7\t117548628\t.\tT\t\t.\tPASS\tEND=117548650;VARID=CFTR_TG;REPID=CFTR_TG\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:11/12:11-11/12-12:8/9 +chr7\t117548651\t.\tT\t\t.\tPASS\tEND=117548660;VARID=CFTR_T;REPID=CFTR_T\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:5/7:5-5/7-7:8/9 +chr2\t233760233\t.\tA\t\t.\tPASS\tEND=233760250;VARID=UGT1A1;REPID=UGT1A1;STR_STATUS=normal\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:7/8:7-7/8-8:5/6 +chr6\t16327636\t.\tT\t\t.\tPASS\tEND=16327700;VARID=ATXN1;REPID=ATXN1;STR_STATUS=normal;STR_NORMAL_MAX=35;STR_PATHOLOGIC_MIN=39\t${EH_FORMAT}\t\ +0/1:SPANNING/FLANKING:30/37:30-30/35-40:4/0 +chr9\t27573528\t.\tC\t\t.\tPASS\tEND=27573540;VARID=C9ORF72;REPID=C9ORF72\t${EH_FORMAT}\t\ +./.:.:.:.:. +chr1\t10\t.\tC\t\t.\tPASS\tEND=20;VARID=EXTRA1\t${EH_FORMAT}\t\ +0/1:SPANNING:8/9:8-8/9-9:1 short\t1\t.\tC" put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}${EH_RECS}\n" at "$OLD" $f/expansion_hunter/S_eh.vcf @@ -251,11 +263,15 @@ put $f/annotsv/S_sv_class4_5_checked.tsv "" put $f/prs/S_prs_summary.tsv "Condition\tPGS_ID\tPercentile\nX\tPGS1\tNA\n" put $f/ancestry/S_ancestry.tsv "key\tvalue\n" at "$OLD" $f/prs/S_prs_summary.tsv -put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tVARID=HTT;REPID=HTT;STR_STATUS=normal\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12\n" +put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t3074877\t.\tC\t\t.\tPASS\tVARID=HTT;REPID=HTT;STR_STATUS=normal\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:17/20:17-17/20-20:10/12\n" at "$OLD" $f/expansion_hunter/S_eh_stranger.vcf -put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t117548628\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:12/12:12-12/12-12:8/9 -chr7\t117548651\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:5/5:5-5/5-5:8/9 -chr2\t233760233\t.\tA\t\t.\tPASS\tVARID=UGT1A1;REPID=UGT1A1\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:8/10:8-8/10-10:5/6\n" +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t117548628\t.\tT\t\t.\tPASS\tVARID=CFTR_TG;REPID=CFTR_TG\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:12/12:12-12/12-12:8/9 +chr7\t117548651\t.\tT\t\t.\tPASS\tVARID=CFTR_T;REPID=CFTR_T\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:5/5:5-5/5-5:8/9 +chr2\t233760233\t.\tA\t\t.\tPASS\tVARID=UGT1A1;REPID=UGT1A1\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:8/10:8-8/10-10:5/6\n" put $f/locityper/S_locityper.tsv "locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads\tweighted_distance\twarnings\tfilter\treason HLA-DRB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t HLA-DQB1\tx\tcalled\th\th\th\t1\t1\t1\t1\t\tpass\t @@ -312,7 +328,9 @@ put $f/paralogs/S_smn_copy_number.tsv "chrom\tstart\tend\n" put $f/paralogs/S_paraphase_copy_number.tsv "region\tgene\nsmn1\tSMN1\n" put $f/paralogs/S_cn_dosage.tsv "locus\tstatus\n" put $f/telomere/S/S_summary.tsv "PID\tsample\tintratel_reads\tread_lengths\ttel_content\nS\ttumor\t40\t150\t1.5\n" -put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/FLANKING:11/12:11-11/12-12:8/0\nchr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T\tGT:SO:REPCN:REPCI:ADSP\t0/1:SPANNING/SPANNING:7/9:7-7/9-9:8/9\n" +put $f/expansion_hunter/S_eh.vcf "${VCF_HEAD}chr7\t1\t.\tT\t\t.\tPASS\tVARID=CFTR_TG\t${EH_FORMAT}\t\ +0/1:SPANNING/FLANKING:11/12:11-11/12-12:8/0\nchr7\t2\t.\tT\t\t.\tPASS\tVARID=CFTR_T\t${EH_FORMAT}\t\ +0/1:SPANNING/SPANNING:7/9:7-7/9-9:8/9\n" put $f/mito/S_chrM_deletions.tsv "bkp5\tbkp3\n1\t2\n" putgz $f/mito/S_chrM_filtered.vcf.gz "${VCF_HEAD}" put $f/vcf/S_roh.txt "RG\tS\tchr1\t1\t2\tx\n" @@ -333,7 +351,8 @@ put $f/y_haplogroup/S_y_tree_path.tsv "Level\tSNPs\n1\tM\n" put $f/paralogs/S_cn_intervals.tsv "locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\nX\tx\tchr1\t1\t2\tcalled\n" put $f/mito/S_chrM_filtered.vcf.gz "not gzip\n" put $f/delly/S_sv.vcf.gz "\x1f\x8b\x08\x00broken" -put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t39348425\t.\tA\t\t.\tPASS\tVARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation\tGT:SO:REPCN:REPCI:ADSP\t1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0\n" +put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t39348425\t.\tA\t\t.\tPASS\tVARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation\t${EH_FORMAT}\t\ +1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0\n" # g2-*: the APOL1 G2 deletion beside step 42's APOL1 line -------------------------- g2() { From 0948aae258d8b6c383ccb2a49133d8e1bc119480 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:34:54 +0100 Subject: [PATCH 19/24] fix(ci): stop pinned downloads on every forbidden or missing response --- CHANGELOG.md | 2 +- rust/crates/og-dev/src/build/mod.rs | 130 ++++++++++++++++++++++++++++ scripts/ci/lib/fetch-pinned.sh | 14 +-- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20ebc7ff0..6eee801af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: - - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure, a stalled transfer included, is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once, including after an earlier retry. Other failures, stalled transfers included, get at most six retries, 30 s apart, within a five-minute retry window. `build-cn-intervals.py` tried three times, 30 s apart. - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs index b2098e3dc..d8e9c5931 100644 --- a/rust/crates/og-dev/src/build/mod.rs +++ b/rust/crates/og-dev/src/build/mod.rs @@ -296,3 +296,133 @@ mod tests { assert!(int("1.5", "x").is_err()); } } + +#[cfg(all(test, unix))] +mod fetch_tests { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + use std::process::Command; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::time::Duration; + + static NEXT: AtomicU64 = AtomicU64::new(0); + const BODY: &str = "hello\n"; + const SHA: &str = "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"; + + // Real HTTP responses exercise curl's own retries too. Shorten only the + // test delays; the production script still supplies its stall limits. + fn fetch(statuses: &[u16], sha: &str, cached: bool, succeeds: bool, expected: &[u16]) { + let dir = std::env::temp_dir().join(format!( + "og-fetch-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + if cached { + std::fs::write(dir.join("asset"), BODY).unwrap(); + } + let server = TcpListener::bind("127.0.0.1:0").unwrap(); + server.set_nonblocking(true).unwrap(); + let url = format!("http://{}/asset", server.local_addr().unwrap()); + let stop = AtomicBool::new(false); + let (output, seen) = std::thread::scope(|scope| { + let worker = scope.spawn(|| { + let mut seen = Vec::new(); + while !stop.load(Ordering::Relaxed) { + match server.accept() { + Ok((mut stream, _)) => { + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + loop { + line.clear(); + if reader.read_line(&mut line).unwrap() == 0 || line == "\r\n" { + break; + } + } + let code = statuses.get(seen.len()).copied().unwrap_or(200); + seen.push(code); + write!(stream, + "HTTP/1.1 {code} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{BODY}", + BODY.len()).unwrap(); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + } + Err(e) => panic!("HTTP test server: {e}"), + } + } + seen + }); + let output = Command::new("bash") + .args([ + "-c", + r#" + source "$1" + sleep() { :; } + curl() { command curl "$@" --noproxy '*' --retry-delay 1 --max-time 5; } + fetch_pinned "$2" asset "$3" "$4" + "#, + "_", + ]) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../scripts/ci/lib/fetch-pinned.sh" + )) + .arg(&dir) + .arg(url) + .arg(sha) + .output() + .unwrap(); + stop.store(true, Ordering::Relaxed); + (output, worker.join().unwrap()) + }); + let file = std::fs::read_to_string(dir.join("asset")).ok(); + let files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(seen, expected, "HTTP sequence {statuses:?}"); + assert_eq!( + output.status.success(), + succeeds, + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(file.as_deref(), succeeds.then_some(BODY)); + assert_eq!( + files.len(), + usize::from(succeeds), + "temporary download leaked: {files:?}" + ); + if let Some(code @ (403 | 404)) = expected.last() { + assert!(String::from_utf8_lossy(&output.stderr).contains(&format!("HTTP {code}"))); + } + } + + #[test] + fn http_403_stops_after_any_attempt() { + fetch(&[403], SHA, false, false, &[403]); + fetch(&[503, 403], SHA, false, false, &[503, 403]); + } + + #[test] + fn http_404_stops_after_any_attempt() { + fetch(&[404], SHA, false, false, &[404]); + fetch(&[503, 404], SHA, false, false, &[503, 404]); + } + + #[test] + fn other_failures_retry_but_stop_at_the_bound() { + fetch(&[503, 503, 200], SHA, false, true, &[503, 503, 200]); + fetch(&[503; 7], SHA, false, false, &[503; 7]); + } + + #[test] + fn downloads_are_pinned_and_cached() { + fetch(&[200], SHA, false, true, &[200]); + fetch(&[200], "wrong-sha256", false, false, &[200]); + fetch(&[], SHA, true, true, &[]); + } +} diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh index f05b1c724..53a1fdc9b 100644 --- a/scripts/ci/lib/fetch-pinned.sh +++ b/scripts/ci/lib/fetch-pinned.sh @@ -33,6 +33,7 @@ pinned_sha256() { fetch_pinned() { local dir=$1 name=$2 url=$3 want=$4 file part code got + local attempt=0 started=$SECONDS file="${dir}/${name}" if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then return 0 @@ -40,8 +41,8 @@ fetch_pinned() { mkdir -p "$dir" || return 1 part=$(mktemp "${file}.part.XXXXXX") || return 1 echo "Downloading ${url}" >&2 - if ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ - -w '%{http_code}' -o "$part" "$url"); then + while ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + -w '%{http_code}' -o "$part" "$url"); do case $code in 403 | 404) rm -f "$part" @@ -49,13 +50,16 @@ fetch_pinned() { return 1 ;; esac - if ! curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ - --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "$part" "$url"; then + attempt=$((attempt + 1)) + # Seven attempts at most. Like curl's retry-max-time, this bounds when + # another attempt starts; an active transfer keeps its stall limits. + if [ "$attempt" -ge 7 ] || [ "$((SECONDS - started + 30))" -ge 300 ]; then rm -f "$part" echo "ERROR: cannot download ${url}" >&2 return 1 fi - fi + sleep 30 + done got=$(pinned_sha256 "$part") if [ "$got" != "$want" ]; then rm -f "$part" From 77fc9f004b5b5ea044955131f7b75cb354f46e44 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:22:07 +0100 Subject: [PATCH 20/24] test(summary): reproduce quoted headers and Unicode contigs --- rust/crates/og-summary/src/collect/text.rs | 36 +++++++++++++++++++ .../crates/og-summary/src/collect/two_hits.rs | 17 +++++++++ tests/parity/cases.tsv | 4 +++ tests/parity/gen/summary-clinical.sh | 14 ++++++++ 4 files changed, 71 insertions(+) diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index d9f554d65..8db81a805 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -211,6 +211,42 @@ mod tests { use super::*; use crate::testutil::Dir; + #[test] + fn quoted_columns_match_the_row_keys() { + let d = Dir::new("text-columns"); + for header in [ + "\"chrom\"\tstart\tend", + "\"a\tb\"\t\"a\"\"b\"\t\"a\nb\"", + "\"\"\tstart\tend", + ] { + let p = d.put("quoted.tsv", format!("{header}\n1\t2\t3\n").as_bytes()); + let r = rows(&p).unwrap(); + let keys: Vec<_> = r[0].iter().map(|(k, _)| k.to_owned()).collect(); + assert_eq!(columns(&p).unwrap(), keys, "{header}"); + } + } + + #[test] + fn columns_read_only_the_first_record() { + let d = Dir::new("text-first-record"); + // A row is larger than the CSV field limit. Reading the header + // must not parse or retain that unrelated row. + let p = d.put( + "large.tsv", + format!("\"chrom\"\tstart\n{}\t1\n", "a".repeat(140_000)).as_bytes(), + ); + assert_eq!(columns(&p).unwrap(), ["chrom", "start"]); + for (header, expected) in [ + ("", vec![]), + ("\n", vec![]), + ("\"\"\n", vec![""]), + ("\"chrom\"\tstart", vec!["chrom", "start"]), + ] { + let p = d.put("empty.tsv", header.as_bytes()); + assert_eq!(columns(&p).unwrap(), expected); + } + } + #[test] fn a_table_cut_short_is_python_s_eof_error() { let d = Dir::new("text-cut"); diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index 52d2184b8..ed0c60288 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -887,6 +887,23 @@ pub fn gene_deletions( mod tests { use super::*; + #[test] + fn chromosome_prefix_is_safe_for_non_ascii_contigs() { + for (input, expected) in [ + ("éé", "éé"), + ("é", "é"), + ("☃1", "☃1"), + ("chr☃", "☃"), + ("CHR1", "1"), + ("cHrX", "X"), + ("chr", ""), + ("ch", "ch"), + ("", ""), + ] { + assert_eq!(chrom_key(input), expected); + } + } + fn del(gt: Option<&str>, ps: Option<&str>) -> Deletion { Deletion { chrom: "chr1".into(), diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 03bf5e088..191f8e4dd 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -839,6 +839,10 @@ summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh -- summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect cb-full gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets assets --out cb-full.json --only b cb-full.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-smn gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-smn --assets assets --out cb-quoted-smn.json --only b cb-quoted-smn.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-interval gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-interval --assets assets --out cb-quoted-interval.json --only b cb-quoted-interval.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-dosage gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-dosage --assets assets --out cb-quoted-dosage.json --only b cb-quoted-dosage.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-unicode-contig gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-unicode-contig --assets assets --out cb-unicode-contig.json --only b cb-unicode-contig.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-nf gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nf --assets assets --out cb-nf.json --only b cb-nf.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-probable gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-probable --assets assets --out cb-probable.json --only b cb-probable.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-failed gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-failed --assets assets --out cb-failed.json --only b cb-failed.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index 99266fac7..8e26193cb 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -170,6 +170,20 @@ full() { } full cb-full +# Change one header at a time. Quoted TSV fields have the same names as +# the unquoted baseline, including the dosage table's validation columns. +full cb-quoted-smn +put cb-quoted-smn/paralogs/S_smn_copy_number.tsv "\"chrom\"${SMN#chrom}" +full cb-quoted-interval +put cb-quoted-interval/paralogs/S_cn_intervals.tsv "\"locus\"${IV#locus}" +full cb-quoted-dosage +put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '\"locus\"\tstatus\tcopies\testimate\treason\treading\n' + +# A valid UTF-8 contig with a code point crossing byte offset three must +# leave the consensus readable. The last chr17 record still matches. +full cb-unicode-contig +putgz cb-unicode-contig/sv_merged/S_sv_consensus.vcf.gz "${CONSV//chr1\\t/éé\\t}" + # nf: the Nextflow names. The clinical VCF and slivar's VCF without their # tables, two PharmCAT reports (the newer wins), Cyrius under another name, # CPSR by its wildcard names, no VEP output, a consensus older than pypgx, From ab1902f3acb2e6b741f2e60dd9b517c6e81d5a16 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:25:13 +0100 Subject: [PATCH 21/24] test(summary): cover nonfinite depths and bounded G2 windows --- .../og-summary/src/collect/group_c/tests.rs | 39 +++++++++++++++++++ .../og-summary/src/collect/group_c/traits.rs | 21 ++++++++++ tests/parity/cases.tsv | 3 ++ tests/parity/gen/summary-collect-c.sh | 29 ++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 3a5b70669..f85e8f6be 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -39,6 +39,45 @@ const CNVS: &str = "deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\ deletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\t1\t1\t1\t0.01\t0\t9\n\ deletion\tchr1:122000001-123000000\t1000000\t0.1\t1e-12\t1\t1\t1\t0.2\t0\t9\n"; +#[test] +fn chip_nan_depth_is_unreadable() { + let d = Dir::new("c-chip-nan"); + d.put("S/chip/S_chip.tsv", "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n"); + d.put("S/chip/S_chip_depth.tsv", "gene\tmean_depth\tdetection_limit_vaf\tstatus\nA\t30\t.\tcovered\nB\tnan\t.\tcovered\nC\t40\t.\tcovered\n"); + let v = summary(&d, None); + let chip = at(&v, &["sections", "chip"]); + assert_eq!(text(at(chip, &["state"])), "unreadable"); + assert_eq!(at(chip, &["data"]), &Value::Object(Default::default())); +} + +#[test] +fn chip_infinity_only_fails_when_the_median_is_nonfinite() { + let d = Dir::new("c-chip-inf"); + d.put("S/chip/S_chip.tsv", "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n"); + for (depths, median) in [ + (vec!["30", "inf", "40"], Some(40.0)), + (vec!["30", "-inf", "40"], Some(30.0)), + (vec!["inf"], None), + (vec!["-inf"], None), + (vec!["-inf", "inf"], None), + ] { + let mut body = "gene\tmean_depth\tdetection_limit_vaf\tstatus\n".to_owned(); + for (i, value) in depths.iter().enumerate() { + body.push_str(&format!("G{i}\t{value}\t.\tcovered\n")); + } + d.put("S/chip/S_chip_depth.tsv", body); + let v = summary(&d, None); + let chip = at(&v, &["sections", "chip"]); + if let Some(median) = median { + assert_eq!(text(at(chip, &["state"])), "ok", "{depths:?}"); + assert_eq!(at(chip, &["data", "median_depth"]), &Value::Float(median)); + } else { + assert_eq!(text(at(chip, &["state"])), "unreadable", "{depths:?}"); + assert_eq!(at(chip, &["data"]), &Value::Object(Default::default())); + } + } +} + #[test] fn every_section_of_group_c_is_written_in_order() { let d = Dir::new("c-order"); diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 5d5a6a806..77cd447a1 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -500,6 +500,27 @@ pub mod apol1 { mod tests { use super::*; + #[test] + fn unindexed_window_retains_only_overlaps() { + use std::io::Write; + let d = crate::testutil::Dir::new("g2-window-bounded"); + let path = d.put("calls.vcf", "##fileformat=VCFv4.2\n"); + let mut f = std::io::BufWriter::new(std::fs::OpenOptions::new().append(true).open(&path).unwrap()); + writeln!(f, "chr21\t1\t.\tA\tT\t.\tPASS\t.").unwrap(); + writeln!(f, "chr22\t1\t.\tA\tT\t.\tPASS\t.").unwrap(); + let block = rec(SPAN.0 - 20, "A", "", "0/0", &format!("END={}", SPAN.1)); + let call = rec(POS, REF, ALT, "0/1", "."); + writeln!(f, "{}", block.join("\t")).unwrap(); + writeln!(f, "{}", call.join("\t")).unwrap(); + for i in 1..=250_000 { + writeln!(f, "chr22\t{}\t.\tA\tT\t.\tPASS\t.", SPAN.1 + i).unwrap(); + } + f.flush().unwrap(); + let records = window(&path).unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records, vec![block, call]); + } + fn rec(pos: i64, r: &str, a: &str, gt: &str, info: &str) -> Vec { [ CHROM, diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 164e855b0..978d0de4d 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -839,6 +839,9 @@ summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh -- summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect c-full gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets assets --out full.json --only c full.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-float-nan gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir float-nan --assets assets --out float-nan.json --only c float-nan.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-float-inf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir float-inf --assets assets --out float-inf.json --only c float-inf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE +summary-collect c-float--inf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir float--inf --assets assets --out float--inf.json --only c float--inf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE summary-collect c-nf gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir nf --assets assets --out nf.json --only c nf.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE summary-collect c-bad1 gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir bad1 --assets assets --out bad1.json --only c bad1.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE summary-collect c-bad2 gen:summary-collect-c.sh,summary-collect/oracle.sh --sample S --sample-dir bad2 --assets assets --out bad2.json --only c bad2.json json:paths=schema_version,sample,run,databases,manifest,sections.couple,sections.parascopy,sections.paralogs,sections.hla,sections.prs,sections.ancestry_fine,sections.manta,sections.delly,sections.cnvpytor,sections.sv_consensus,sections.annotsv,sections.expansions,sections.telomere,sections.roh,sections.haplogroup,sections.y_haplogroup,sections.y_tree,sections.y_str,sections.consumer_export,sections.bloodtype,sections.lpa,sections.vntyper,sections.locityper,sections.mito,sections.traits,sections.archaic,sections.ancient,sections.viral,sections.exomiser,sections.chip,sections.mei,sections.mosaic PYTHON_IMAGE diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index e67afa697..4df36d5af 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -354,6 +354,35 @@ put $f/delly/S_sv.vcf.gz "\x1f\x8b\x08\x00broken" put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t39348425\t.\tA\t\t.\tPASS\tVARID=RFC1;REPID=RFC1;STR_STATUS=full_mutation\t${EH_FORMAT}\t\ 1/1:INREPEAT/INREPEAT:500/600:400-700/500-800:0/0\n" +# Non-finite cells follow each Python reader's rules, not a global filter. +for special in nan inf -inf; do + f="float-${special}" + cp -R full "$f" + put "$f/chip/S_chip_depth.tsv" "gene\tmean_depth\tdetection_limit_vaf\tstatus\nA\t30\t.\tcovered\nB\t${special}\t.\tcovered\nC\t40\t.\tcovered\n" + for spec in \ + 'paralogs/S_smn_copy_number.tsv:agCN_qual,psCN_qual' \ + 'paralogs/S_pms2_copy_number.tsv:agCN_qual,psCN_qual' \ + 'ancestry_fine/S_ancestry_fine.tsv:fraction,low,high' \ + 'telomere/S/S/S_summary.tsv:read_lengths' \ + 'mito/S_chrM_deletions.tsv:level_pct' \ + 'mito/S_chrM_annotated.tsv:af'; do + file=${spec%%:*} + cols=${spec#*:} + [ -f "$f/$file" ] || { echo "missing float fixture: $file" >&2; exit 1; } + awk -F '\t' -v OFS='\t' -v cols="$cols" -v value="$special" ' + BEGIN { split(cols, names, ","); for (i in names) want[names[i]] = 1 } + NR == 1 { for (i = 1; i <= NF; i++) if ($i in want) change[i] = 1 } + NR > 1 { for (i in change) $i = value } + { print } + ' "$f/$file" > "$f/cells.tmp" + mv -f "$f/cells.tmp" "$f/$file" + done + put "$f/hla_t1k/S_hla_genotype.tsv" "HLA-A\t2\tA*02:01\t1\t${special}\tA*01:01\t1\t${special}\n" + put "$f/vcf/S_roh.txt" "RG\tS\tchr2\t1\t7000000\t${special}\n" + put "$f/cnvpytor/S_cnvs.txt" "deletion\tchr2:1-100000\t100000\t0.4\t${special}\t1\t1\t1\t${special}\t${special}\t9\n" + putgz "$f/mito/S_chrM_filtered.vcf.gz" "${VCF_HEAD}chrM\t3243\t.\tA\tG\t50\tPASS\t.\tGT:AF\t0/1:${special}\n" +done + # g2-*: the APOL1 G2 deletion beside step 42's APOL1 line -------------------------- g2() { local d=$1 result=$2 status=$3 From 1a883cd88f5b74ed5bbd9f75d610690ac8afc9fe Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:26:25 +0100 Subject: [PATCH 22/24] fix(summary): parse quoted headers and guard contig prefixes --- CHANGELOG.md | 2 +- rust/crates/og-summary/src/collect/text.rs | 38 +++++++++++-------- .../crates/og-summary/src/collect/two_hits.rs | 6 ++- tests/parity/gen/summary-clinical.sh | 2 +- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f31e9387e..b762f54a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 9 parity cases, compared as JSON with key order; a tenth holds the deliberate fix below. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: +- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 13 parity cases, compared as JSON with key order; a fourteenth holds the deliberate fix below. Quoted headers in SMN, interval and dosage tables preserve deletion evidence. Non-ASCII consensus contigs leave the collector running. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: - A clinical or slivar VCF named `.vcf.gz` that is plain text is read. The Python marked the section unreadable. - A PharmCAT report that is not UTF-8 is still unreadable, but the note names the fault in Rust's words. diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index 8db81a805..275e236df 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -3,7 +3,7 @@ //! their kind, and Python's way of reading a digit cell. use crate::{basename, os_error, text_lines}; -use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::csv::{DictReader, DictRow, Quoting, Reader}; use pycompat::text::{isdigit, strip}; use std::collections::HashMap; @@ -91,10 +91,10 @@ pub fn each_line(path: &str, mut f: impl FnMut(&str)) -> Result<(), ReadError> { Ok(()) } -/// The rows of a tab-separated table with a header (`csv.DictReader`). -pub fn rows(path: &str) -> Result, ReadError> { +/// Table lines keep their endings so quoted fields may span lines. +fn table_lines(path: &str) -> Result>, ReadError> { let mut lines = text_lines(path).map_err(ReadError::Os)?; - let iter = std::iter::from_fn(move || { + Ok(std::iter::from_fn(move || { let mut s = String::new(); match lines.read_line(&mut s) { Ok(true) => Some(Ok(s)), @@ -104,22 +104,30 @@ pub fn rows(path: &str) -> Result, ReadError> { } Err(e) => Some(Err(e)), } - }); - DictReader::new(iter, '\t', Quoting::Minimal) + })) +} + +fn csv_error(e: pycompat::csv::Error) -> ReadError { + match e.exception() { + "OSError" => ReadError::Os(e.to_string()), + kind => ReadError::Other(format!("{kind}: {e}")), + } +} + +/// The rows of a tab-separated table with a header (`csv.DictReader`). +pub fn rows(path: &str) -> Result, ReadError> { + DictReader::new(table_lines(path)?, '\t', Quoting::Minimal) .collect::, _>>() - .map_err(|e| match e.exception() { - "OSError" => ReadError::Os(e.to_string()), - kind => ReadError::Other(format!("{kind}: {e}")), - }) + .map_err(csv_error) } /// The header of a table as `csv.reader`'s first row reads it. pub fn columns(path: &str) -> Result, ReadError> { - let first = lines(path)?.into_iter().next().unwrap_or_default(); - if first.is_empty() { - return Ok(Vec::new()); - } - Ok(first.split('\t').map(str::to_owned).collect()) + Reader::new(table_lines(path)?, '\t', Quoting::Minimal) + .next() + .transpose() + .map(Option::unwrap_or_default) + .map_err(csv_error) } /// `row.get(key)` where a missing key and a short row both read as `None`. diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index ed0c60288..b022c8412 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -376,7 +376,11 @@ fn sample_format(r: &DictRow) -> HashMap<&str, &str> { const CONSENSUS_BP: i128 = 1000; fn chrom_key(c: &str) -> &str { - if c.len() >= 3 && c[..3].eq_ignore_ascii_case("chr") { + if c.as_bytes() + .get(..3) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"chr")) + { + // An ASCII prefix ends at a UTF-8 boundary. &c[3..] } else { c diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index 8e26193cb..6d5cfbac3 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -177,7 +177,7 @@ put cb-quoted-smn/paralogs/S_smn_copy_number.tsv "\"chrom\"${SMN#chrom}" full cb-quoted-interval put cb-quoted-interval/paralogs/S_cn_intervals.tsv "\"locus\"${IV#locus}" full cb-quoted-dosage -put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '\"locus\"\tstatus\tcopies\testimate\treason\treading\n' +put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '"locus"\tstatus\tcopies\testimate\treason\treading\n' # A valid UTF-8 contig with a code point crossing byte offset three must # leave the consensus readable. The last chr17 record still matches. From 066878feb877257bb67831ec7a48af7480bb1da1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:34:13 +0100 Subject: [PATCH 23/24] fix(summary): reject NaN CHIP depths and bound unindexed G2 reads --- CHANGELOG.md | 2 + .../og-summary/src/collect/group_c/tables.rs | 14 +-- .../og-summary/src/collect/group_c/tests.rs | 10 ++- .../og-summary/src/collect/group_c/traits.rs | 88 ++++++++++++++++--- tests/parity/gen/summary-collect-c.sh | 6 ++ 5 files changed, 102 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8986c6c..b727bc23c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust summary's CHIP reader marks a depth table with NaN unreadable instead of sorting it into a usable sensitivity estimate. Its unindexed APOL1 G2 reader keeps only records overlapping the deletion span and stops past that span on sorted input. + - The Rust `og` binary now reads the rest of the summary's own sections: `og summary collect --only c`. It reads 32 sections of summary.json (couple, parascopy, paralogs, hla, prs, ancestry_fine, manta, delly, cnvpytor, sv_consensus, annotsv, expansions and their readings, telomere, roh, haplogroup, y_haplogroup, y_tree, y_str, consumer_export, bloodtype, lpa, vntyper, locityper, mito, traits with the APOL1 G2 call, archaic, ancient, viral, exomiser, chip, mei and mosaic). Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 12 new parity cases, compared as JSON with key order. The readers run on a thread pool, one per section. On a synthetic sample folder with every section and a 1,000,000-record VCF it takes 0.33 s and 34 MB, where the Python readers took 0.33 s and 41 to 59 MB. What differs from the Python: - A chrM file named `.vcf.gz` that is plain text is read, as the variants section already does. The Python marked the mito section unreadable. - The APOL1 G2 deletion is read through the VCF's `.tbi` when there is one, so only its window of chr22 is read. diff --git a/rust/crates/og-summary/src/collect/group_c/tables.rs b/rust/crates/og-summary/src/collect/group_c/tables.rs index 5f39673e8..d13e7d843 100644 --- a/rust/crates/og-summary/src/collect/group_c/tables.rs +++ b/rust/crates/og-summary/src/collect/group_c/tables.rs @@ -1269,11 +1269,15 @@ pub fn chip(c: &Ctx<'_>) -> Out { covered.push(pycompat::num::parse_float(x).map_err(|_| float_err(x))?); } } - covered.sort_by(f64::total_cmp); - let med = match covered.len() { - 0 => None, - n if n % 2 == 1 => Some(covered[n / 2]), - n => Some((covered[n / 2 - 1] + covered[n / 2]) / 2.0), + let med = if covered.is_empty() { + None + } else { + // NaN has no ordering. Never let sorting move an invalid depth + // away from the median and produce a usable sensitivity estimate. + Some( + pycompat::stats::median(covered) + .map_err(|e| py_error(e.exception(), "cannot convert float NaN to integer"))?, + ) }; let limit = match med { None => Value::Null, diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index f85e8f6be..4a93b5111 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -42,7 +42,10 @@ deletion\tchr1:122000001-123000000\t1000000\t0.1\t1e-12\t1\t1\t1\t0.2\t0\t9\n"; #[test] fn chip_nan_depth_is_unreadable() { let d = Dir::new("c-chip-nan"); - d.put("S/chip/S_chip.tsv", "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n"); + d.put( + "S/chip/S_chip.tsv", + "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n", + ); d.put("S/chip/S_chip_depth.tsv", "gene\tmean_depth\tdetection_limit_vaf\tstatus\nA\t30\t.\tcovered\nB\tnan\t.\tcovered\nC\t40\t.\tcovered\n"); let v = summary(&d, None); let chip = at(&v, &["sections", "chip"]); @@ -53,7 +56,10 @@ fn chip_nan_depth_is_unreadable() { #[test] fn chip_infinity_only_fails_when_the_median_is_nonfinite() { let d = Dir::new("c-chip-inf"); - d.put("S/chip/S_chip.tsv", "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n"); + d.put( + "S/chip/S_chip.tsv", + "gene\tprotein_change\tvaf\talt_reads\tdepth\tstatus\tnote\n", + ); for (depths, median) in [ (vec!["30", "inf", "40"], Some(40.0)), (vec!["30", "-inf", "40"], Some(30.0)), diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 77cd447a1..6a553b36f 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -2,7 +2,7 @@ //! and the APOL1 G2 deletion read from the sample's VCF beside the APOL1 //! rule, whose single-base sites read G1 only. -use super::util::{Obj, base, count, ragged, s, table, vcf_lines}; +use super::util::{Obj, base, count, ragged, s, table}; use super::{Ctx, Out}; use pycompat::json::Value; @@ -382,8 +382,41 @@ pub mod apol1 { ) } - /// The records of `chr22` that can cover the span: by the tabix index - /// when the `.tbi` is there, else the whole file. + /// Only records overlapping the span. Like the Python window reader, + /// stop past the window or after leaving chr22 on sorted input. + fn window_records( + lines: impl IntoIterator>, + ) -> Result>, String> { + let mut out = Vec::new(); + let mut seen = false; + for line in lines { + let line = line?; + if line.starts_with('#') { + continue; + } + let c = super::super::util::cells(line.strip_suffix('\n').unwrap_or(&line).as_bytes()); + if c.len() < 2 || !digits(&c[1]) { + continue; + } + if c[0] != CHROM { + if seen { + break; + } + continue; + } + seen = true; + if c[1].parse::().is_ok_and(|p| p > i128::from(SPAN.1)) { + break; + } + if overlaps(&c) { + out.push(c); + } + } + Ok(out) + } + + /// The records over the G2 span, using tabix when a `.tbi` is there, + /// otherwise streaming up to the end of the window. fn window(path: &str) -> Result>, String> { let tbi = format!("{path}.tbi"); let mut out = Vec::new(); @@ -395,13 +428,19 @@ pub mod apol1 { } return Ok(out); } - vcf_lines(path, |l| { - let c = super::super::util::cells(l); - if c.len() >= 2 && c[0] == CHROM && digits(&c[1]) { - out.push(c); + let mut rd = crate::text_lines(path)?; + window_records(std::iter::from_fn(|| { + let mut line = String::new(); + match rd.read_line(&mut line) { + Ok(true) => Some(Ok(line)), + Ok(false) => None, + Err(e) => Some(Err(format!( + "{}: {}", + base(path), + crate::collect::sections::gzip_message(&e) + ))), } - })?; - Ok(out) + })) } /// G2 from the sample's gVCF when it is there, else its VCF. @@ -500,15 +539,42 @@ pub mod apol1 { mod tests { use super::*; + #[test] + fn unindexed_window_stops_at_the_end_or_next_chromosome() { + let call = rec(POS, REF, ALT, "0/1", "."); + for stop in [ + format!("chr22\t{}\t.\tA\tT", SPAN.1 + 1), + "chr23\t1\t.\tA\tT".into(), + ] { + let lines = [ + Ok(call.join("\t")), + Ok(stop), + Err("must stop before reading this line".into()), + ]; + assert_eq!(window_records(lines).unwrap(), vec![call.clone()]); + } + } + #[test] fn unindexed_window_retains_only_overlaps() { use std::io::Write; let d = crate::testutil::Dir::new("g2-window-bounded"); let path = d.put("calls.vcf", "##fileformat=VCFv4.2\n"); - let mut f = std::io::BufWriter::new(std::fs::OpenOptions::new().append(true).open(&path).unwrap()); + let mut f = std::io::BufWriter::new( + std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(), + ); writeln!(f, "chr21\t1\t.\tA\tT\t.\tPASS\t.").unwrap(); writeln!(f, "chr22\t1\t.\tA\tT\t.\tPASS\t.").unwrap(); - let block = rec(SPAN.0 - 20, "A", "", "0/0", &format!("END={}", SPAN.1)); + let block = rec( + SPAN.0 - 20, + "A", + "", + "0/0", + &format!("END={}", SPAN.1), + ); let call = rec(POS, REF, ALT, "0/1", "."); writeln!(f, "{}", block.join("\t")).unwrap(); writeln!(f, "{}", call.join("\t")).unwrap(); diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 4df36d5af..9c853a9e4 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -378,6 +378,12 @@ for special in nan inf -inf; do mv -f "$f/cells.tmp" "$f/$file" done put "$f/hla_t1k/S_hla_genotype.tsv" "HLA-A\t2\tA*02:01\t1\t${special}\tA*01:01\t1\t${special}\n" + # Python raises an uncaught OverflowError if an infinite read length + # reaches its floor calculation. Keep this full-collector comparison + # on its defined no-depth path; Rust isolates that error to the section. + if [ "$special" != nan ]; then + put "$f/telomere/S/S/S_summary.tsv" "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t0\t${special}\t40\t1.5\n" + fi put "$f/vcf/S_roh.txt" "RG\tS\tchr2\t1\t7000000\t${special}\n" put "$f/cnvpytor/S_cnvs.txt" "deletion\tchr2:1-100000\t100000\t0.4\t${special}\t1\t1\t1\t${special}\t${special}\t9\n" putgz "$f/mito/S_chrM_filtered.vcf.gz" "${VCF_HEAD}chrM\t3243\t.\tA\tG\t50\tPASS\t.\tGT:AF\t0/1:${special}\n" From 030fbdfec608cb0cee5a9722bb7c0572dab0e257 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:51:23 +0100 Subject: [PATCH 24/24] fix(ci): stop E2E legs dying at the spare-disk mount on runners that list zram0 Some runner images list /dev/zram0 as a disk of 0 bytes with no file system. The spare-disk step took it for the blank disk, and mkfs.ext4 stopped the leg before any case ran. The step now skips zram, ram and loop devices and any disk of 0 bytes. --- .github/workflows/e2e.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e7c02c998..05cf8d80e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -500,13 +500,17 @@ jobs: run: | if mountpoint -q /mnt; then echo "/mnt is already a mount"; df -h /mnt; exit 0; fi disk="" - while read -r name type; do + while read -r name type size; do [ "$type" = disk ] || continue + # A real disk: some runner images list /dev/zram0 as a disk of + # 0 bytes, and mkfs.ext4 stops on it. + case $name in /dev/zram*|/dev/ram*|/dev/loop*) continue ;; esac + [ "${size:-0}" -gt 0 ] || continue # Blank: no partition, no file system, nothing mounted. [ "$(lsblk -nro NAME "$name" | wc -l)" -eq 1 ] || continue [ -z "$(lsblk -ndro FSTYPE,MOUNTPOINT "$name" | tr -d ' ')" ] || continue disk=$name - done < <(lsblk -dnpo NAME,TYPE) + done < <(lsblk -dnbpo NAME,TYPE,SIZE) if [ -z "$disk" ]; then echo "No blank disk on this runner"; exit 0; fi echo "Formatting ${disk} ($(lsblk -ndo SIZE "$disk")) and mounting it at /mnt" sudo mkfs.ext4 -q -F -E nodiscard "$disk"