From 985b2f1402fe3cf7a11deba22ff8f39c038837fb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:05:35 +0100 Subject: [PATCH 1/8] feat(traits): evaluate trait and haplotype catalogs in Rust Bead: pgp-bs9u.25 --- rust/crates/og-steps/src/lib.rs | 3 + rust/crates/og-steps/src/traits.rs | 1351 ++++++++++++++++++++++++++++ rust/og/src/main.rs | 4 + tests/parity/cases.tsv | 27 + tests/parity/gen/traits.sh | 42 + tests/parity/oracle.ref | 1 + 6 files changed, 1428 insertions(+) create mode 100644 rust/crates/og-steps/src/traits.rs create mode 100644 tests/parity/gen/traits.sh diff --git a/rust/crates/og-steps/src/lib.rs b/rust/crates/og-steps/src/lib.rs index f72a1b34..fae33897 100644 --- a/rust/crates/og-steps/src/lib.rs +++ b/rust/crates/og-steps/src/lib.rs @@ -26,3 +26,6 @@ pub mod pgx_rules; pub mod prs; pub mod pypgx_summary; pub mod sv; + +/// Step 42 traits and genotype rules. +pub mod traits; diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs new file mode 100644 index 00000000..145e6915 --- /dev/null +++ b/rust/crates/og-steps/src/traits.rs @@ -0,0 +1,1351 @@ +//! Step 42: bounded site genotypes, haplotype rules and read-backed phase. +use indexmap::IndexMap; +use og_io::{AtomicFile, Lines, Utf8}; +use pycompat::csv::{Quoting, Reader, Writer}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::io::Write; + +const COLUMNS: &[&str] = &[ + "chrom", "pos", "rsid", "gene", "ref", "alt", "trait", "hom_ref", "het", "hom_alt", "ancestry", + "note", "pmids", +]; +const OUT: &[&str] = &[ + "rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", + "pmids", +]; +const RULE_OUT: &[&str] = &[ + "rule", + "gene", + "trait", + "rsids", + "chrom", + "pos", + "genotypes", + "gq", + "phase", + "result", + "status", + "reading", + "ancestry", + "note", + "pmids", +]; +type Row = BTreeMap; +type Key = (String, String); +type Genotypes = HashMap; +type Phased = HashMap; +type Result = std::result::Result; + +#[derive(Clone, Debug)] +struct Site { + row: Row, + line: usize, +} +impl Site { + fn key(&self) -> Key { + (self.row["chrom"].clone(), self.row["pos"].clone()) + } +} +#[derive(Clone, Debug)] +struct Hap { + name: String, + alleles: Vec, + line: usize, +} +#[derive(Clone, Debug)] +struct Combo { + key: String, + label: String, + reading: String, + line: usize, +} +#[derive(Clone, Debug)] +struct Rule { + id: String, + meta: Row, + line: usize, + sites: Vec, + haps: Vec, + combos: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq)] +enum Status { + Called, + NotCovered, + OtherAllele, + Unphased, + OtherHaplotype, +} +impl Status { + fn text(&self) -> &'static str { + match self { + Self::Called => "called", + Self::NotCovered => "not_covered", + Self::OtherAllele => "other_allele", + Self::Unphased => "unphased", + Self::OtherHaplotype => "other_haplotype", + } + } +} +#[derive(Clone, Debug)] +struct SiteGt { + alleles: Vec, + status: Status, + gq: String, + snv: bool, +} +impl Default for SiteGt { + fn default() -> Self { + Self { + alleles: vec![], + status: Status::NotCovered, + gq: ".".into(), + snv: false, + } + } +} +fn row(keys: &[&str], values: &[String]) -> Row { + keys.iter() + .zip(values) + .map(|(k, v)| (k.to_string(), v.clone())) + .collect() +} +fn records(path: &str) -> Result)>> { + // Parse even the header as a CSV record. Comments do not enter the parser. + let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut text = String::new(); + let mut number = 0; + let mut out = Vec::new(); + while lines.read_line(&mut text).map_err(|e| e.to_string())? { + number += 1; + if text.trim_end_matches('\n').is_empty() || text.starts_with('#') { + continue; + } + let cells = Reader::new( + std::iter::once(Ok::<_, std::io::Error>(text.clone())), + '\t', + Quoting::Minimal, + ) + .next() + .ok_or("empty record")? + .map_err(|e| e.to_string())?; + out.push((number, cells)); + } + Ok(out) +} +fn read_table(path: &str) -> Result> { + let mut input = records(path)?.into_iter(); + let (n, header) = input + .next() + .ok_or_else(|| format!("{path}: no header line"))?; + if header != COLUMNS { + return Err(format!("{path}:{n}: unexpected traits header")); + } + input + .map(|(line, c)| { + if c.len() != COLUMNS.len() { + return Err(format!( + "{path}:{line}: {} columns, expected {}", + c.len(), + COLUMNS.len() + )); + } + Ok(Site { + row: row(COLUMNS, &c), + line, + }) + }) + .collect() +} +fn read_rules(path: &str) -> Result> { + let mut rules: IndexMap = IndexMap::new(); + for (line, c) in records(path)? { + let fields: &[&str] = match c[0].as_str() { + "rule" => &["gene", "trait", "needs_phase", "ancestry", "note", "pmids"], + "site" => &["rsid", "chrom", "pos", "ref", "alt"], + "haplotype" => &["name", "alleles"], + "combination" => &["diplotypes", "label", "reading"], + kind => { + return Err(format!( + "{path}:{line}: kind '{kind}' is not one of rule, site, haplotype, combination" + )); + } + }; + if c.len() != fields.len() + 2 { + return Err(format!( + "{path}:{line}: a {} line has {} columns, expected {}", + c[0], + c.len(), + fields.len() + 2 + )); + } + let id = &c[1]; + if c[0] == "rule" { + if rules.contains_key(id) { + return Err(format!("{path}:{line}: rule {id} has two rule lines")); + } + rules.insert( + id.clone(), + Rule { + id: id.clone(), + meta: row(fields, &c[2..]), + line, + sites: vec![], + haps: vec![], + combos: vec![], + }, + ); + } else { + let r = rules.get_mut(id).ok_or_else(|| { + format!( + "{path}:{line}: a {} line of rule '{id}' before its rule line", + c[0] + ) + })?; + match c[0].as_str() { + "site" => r.sites.push(Site { + row: row(fields, &c[2..]), + line, + }), + "haplotype" => r.haps.push(Hap { + name: c[2].clone(), + alleles: c[3].split(',').map(str::to_owned).collect(), + line, + }), + _ => r.combos.push(Combo { + key: c[2].clone(), + label: c[3].clone(), + reading: c[4].clone(), + line, + }), + } + } + } + if rules.is_empty() { + return Err(format!("{path}: no rule line")); + } + Ok(rules.into_values().collect()) +} +fn positive(s: &str) -> bool { + s.as_bytes() + .first() + .is_some_and(|c| matches!(c, b'1'..=b'9')) + && s.bytes().all(|c| c.is_ascii_digit()) +} +fn pmids(s: &str) -> bool { + s.split(',').all(positive) +} +fn text_errors(w: &str, text: &str, e: &mut Vec) { + if text.to_lowercase().contains("snpedia") { + e.push(format!( + "{w}: mentions SNPedia; the table takes no text from it" + )); + } + if text.contains('"') || !text.is_ascii() { + e.push(format!( + "{w}: a double quote or a non-ASCII character (the reports read the table as plain TSV)" + )); + } +} +fn site_errors(w: &str, s: &Site, e: &mut Vec) { + let r = &s.row; + if !(1..=22).any(|n| r["chrom"] == format!("chr{n}")) { + e.push(format!("{w}: chrom '{}' is not chr1 to chr22", r["chrom"])); + } + if !positive(&r["pos"]) { + e.push(format!("{w}: pos '{}' is not a positive integer", r["pos"])); + } + if !r["rsid"].strip_prefix("rs").is_some_and(positive) { + e.push(format!("{w}: rsid is not rs")); + } + for k in ["ref", "alt"] { + if !["A", "C", "G", "T"].contains(&r[k].as_str()) { + e.push(format!("{w}: {k} '{}' is not one of A, C, G, T", r[k])); + } + } + if r["ref"] == r["alt"] { + e.push(format!("{w}: ref and alt are the same base")); + } +} +fn check_rows(rows: &[Site]) -> Vec { + let mut e = vec![]; + let mut ids = HashSet::new(); + let mut sites = HashSet::new(); + for s in rows { + let r = &s.row; + let w = format!("line {} ({})", s.line, r["rsid"]); + site_errors(&w, s, &mut e); + if !ids.insert(r["rsid"].as_str()) { + e.push(format!("{w}: rsid appears twice")); + } + if !sites.insert(s.key()) { + e.push(format!( + "{w}: position {}:{} appears twice", + r["chrom"], r["pos"] + )); + } + for k in ["gene", "trait", "hom_ref", "het", "hom_alt", "note"] { + if r[k].trim().is_empty() { + e.push(format!("{w}: {k} is empty")); + } + } + if !["yes", "no"].contains(&r["ancestry"].as_str()) { + e.push(format!( + "{w}: ancestry '{}' is not yes or no", + r["ancestry"] + )); + } + if !pmids(&r["pmids"]) { + e.push(format!( + "{w}: pmids '{}' is not a comma-separated list of PubMed ids", + r["pmids"] + )); + } + text_errors( + &w, + &COLUMNS + .iter() + .map(|k| r[*k].as_str()) + .collect::>() + .join(" "), + &mut e, + ); + } + for (id, a, b) in [("rs429358", "T", "C"), ("rs7412", "C", "T")] { + if let Some(s) = rows.iter().rev().find(|s| s.row["rsid"] == id) { + let r = &s.row; + if r["chrom"] != "chr19" || r["ref"] != a || r["alt"] != b { + e.push(format!( + "APOE row {id} must be chr19 {a}>{b}, is {} {}>{}", + r["chrom"], r["ref"], r["alt"] + )); + } + } else { + e.push(format!("APOE row {id} is missing")); + } + } + e +} +fn allowed(rule: &Rule, gt: &[Vec]) -> Vec { + let mut out = vec![]; + for (i, a) in rule.haps.iter().enumerate() { + for b in &rule.haps[i..] { + if gt.iter().enumerate().all(|(k, g)| { + g.len() == 2 + && ((g[0] == a.alleles[k] && g[1] == b.alleles[k]) + || (g[1] == a.alleles[k] && g[0] == b.alleles[k])) + }) { + out.push(format!("{}/{}", a.name, b.name)); + } + } + } + out +} +fn check_rules(rules: &[Rule], rows: Option<&[Site]>) -> Vec { + let mut e = vec![]; + for r in rules { + let n0 = e.len(); + let w = format!("rule {} (line {})", r.id, r.line); + if r.id.is_empty() || !r.id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + e.push(format!("{w}: the rule id is not letters, digits and _")); + } + for k in ["gene", "trait", "note"] { + if r.meta[k].trim().is_empty() { + e.push(format!("{w}: {k} is empty")); + } + } + for k in ["needs_phase", "ancestry"] { + if !["yes", "no"].contains(&r.meta[k].as_str()) { + e.push(format!("{w}: {k} '{}' is not yes or no", r.meta[k])); + } + } + if !pmids(&r.meta["pmids"]) { + e.push(format!( + "{w}: pmids '{}' is not a comma-separated list of PubMed ids", + r.meta["pmids"] + )); + } + if !(2..=8).contains(&r.sites.len()) { + e.push(format!("{w}: {} sites, expected 2 to 8", r.sites.len())); + } + let mut ids = HashSet::new(); + let mut positions = HashSet::new(); + for s in &r.sites { + let sw = format!("{w}, site line {} ({})", s.line, s.row["rsid"]); + site_errors(&sw, s, &mut e); + if !ids.insert(&s.row["rsid"]) || !positions.insert(s.key()) { + e.push(format!("{sw}: the site appears twice in the rule")); + } + if let Some(t) = + rows.and_then(|rows| rows.iter().rev().find(|t| t.row["rsid"] == s.row["rsid"])) + { + if ["chrom", "pos", "ref", "alt"] + .iter() + .any(|k| t.row[*k] != s.row[*k]) + { + e.push(format!( + "{sw}: the traits table has {}:{} {}>{}, the rule {}:{} {}>{}", + t.row["chrom"], + t.row["pos"], + t.row["ref"], + t.row["alt"], + s.row["chrom"], + s.row["pos"], + s.row["ref"], + s.row["alt"] + )); + } + } + } + if r.sites + .iter() + .map(|s| &s.row["chrom"]) + .collect::>() + .len() + > 1 + { + e.push(format!("{w}: the sites are on more than one chromosome")); + } + let mut names = Vec::new(); + let mut tuples = HashSet::new(); + for h in &r.haps { + let hw = format!("{w}, haplotype line {} ({})", h.line, h.name); + if h.name.is_empty() + || !h + .name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"*.".contains(&b)) + { + e.push(format!("{hw}: the name is not letters, digits, * and .")); + } + if names.contains(&h.name.as_str()) { + e.push(format!("{hw}: the name appears twice")); + } + names.push(h.name.as_str()); + if h.alleles.len() != r.sites.len() { + e.push(format!( + "{hw}: {} alleles for {} sites", + h.alleles.len(), + r.sites.len() + )); + continue; + } + for (a, s) in h.alleles.iter().zip(&r.sites) { + if a != &s.row["ref"] && a != &s.row["alt"] { + e.push(format!( + "{hw}: allele {a} at {} is neither its ref {} nor its alt {}", + s.row["rsid"], s.row["ref"], s.row["alt"] + )); + } + } + if !tuples.insert(&h.alleles) { + e.push(format!("{hw}: the same alleles as another haplotype")); + } + } + if names.len() < 2 { + e.push(format!("{w}: fewer than two haplotypes")); + } + let mut keys = BTreeSet::new(); + for c in &r.combos { + let cw = format!("{w}, combination line {} ({})", c.line, c.key); + for (k, v) in [("label", &c.label), ("reading", &c.reading)] { + if v.trim().is_empty() { + e.push(format!("{cw}: {k} is empty")); + } + } + text_errors(&cw, &format!("{} {} {}", c.key, c.label, c.reading), &mut e); + if !keys.insert(c.key.clone()) { + e.push(format!("{cw}: the diplotypes appear twice")); + } + for d in c.key.split('|') { + let pair: Vec<_> = d.split('/').collect(); + if pair.len() != 2 || pair.iter().any(|p| !names.contains(p)) { + e.push(format!( + "{cw}: '{d}' is not two of the rule's haplotypes joined by /" + )); + } else if names.iter().position(|n| n == &pair[0]) + > names.iter().position(|n| n == &pair[1]) + { + e.push(format!( + "{cw}: '{d}' is not in the order of the haplotype lines (write {}/{})", + pair[1], pair[0] + )); + } + } + } + text_errors( + &w, + &format!("{} {} {}", r.meta["gene"], r.meta["trait"], r.meta["note"]), + &mut e, + ); + if e.len() != n0 { + continue; + } + let mut want = BTreeSet::new(); + if r.meta["needs_phase"] == "yes" { + for (i, a) in names.iter().enumerate() { + for b in &names[i..] { + want.insert(format!("{a}/{b}")); + } + } + } else { + for mut n in 0..3usize.pow(r.sites.len() as u32) { + let mut gt = vec![]; + for s in &r.sites { + let v = n % 3; + n /= 3; + gt.push(vec![ + s.row[if v == 2 { "alt" } else { "ref" }].clone(), + s.row[if v == 0 { "ref" } else { "alt" }].clone(), + ]); + } + let key = allowed(r, >).join("|"); + if !key.is_empty() { + want.insert(key); + } + } + } + for k in want.difference(&keys) { + e.push(format!("{w}: no combination line for {k}")); + } + for k in keys.difference(&want) { + e.push(format!( + "{w}: the combination line {k} can never be read {}", + if k.contains('|') { + "(a rule that needs phase reads one diplotype)" + } else { + "from these haplotypes" + } + )); + } + } + if let Some(rows) = rows { + if let Some(r) = rules.iter().find(|r| r.id == "APOE") { + for s in &r.sites { + if !rows.iter().any(|t| t.row["rsid"] == s.row["rsid"]) { + e.push(format!( + "rule APOE: site {} has no traits table row", + s.row["rsid"] + )); + } + } + } else { + e.push("rule APOE is missing: its line goes into the traits table, where the reports read it".into()); + } + } + e +} + +fn read_genotypes(path: &str, sites: &[&Site]) -> Result { + let mut wanted: HashMap> = HashMap::new(); + for s in sites { + wanted.entry(s.key()).or_default().push(s); + } + let mut result = HashMap::new(); + let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut line = String::new(); + let mut n = 0; + while lines.read_line(&mut line).map_err(|e| e.to_string())? { + n += 1; + let line = line.trim_end_matches('\n'); + if line.is_empty() { + continue; + } + let c: Vec<_> = line.split('\t').collect(); + if c.len() < 5 { + return Err(format!( + "{path}:{n}: {} columns, expected CHROM POS REF ALT GENOTYPE [GQ]", + c.len() + )); + } + let key = (c[0].to_owned(), c[1].to_owned()); + let Some(sites) = wanted.get(&key) else { + continue; + }; + let site = sites[0]; + for s in sites { + if c[2].chars().next() != s.row["ref"].chars().next() { + return Err(format!( + "{} at {}:{}: the gVCF has REF {}, the table {} (the GRCh38 base). Is the gVCF on GRCh38?", + s.row["rsid"], c[0], c[1], c[2], s.row["ref"] + )); + } + } + let mut al: Vec<_> = c[4].split(['/', '|']).map(str::to_owned).collect(); + let snv = al.iter().all(|a| a.chars().count() == 1); + if result.get(&key).is_some_and(|old: &SiteGt| old.snv || !snv) { + continue; + } + al.sort_by_key(|a| { + ( + if a == &site.row["ref"] { + 0 + } else if a == &site.row["alt"] { + 1 + } else { + 2 + }, + a.clone(), + ) + }); + let status = if al.len() == 2 + && al + .iter() + .all(|a| a == &site.row["ref"] || a == &site.row["alt"]) + { + Status::Called + } else { + Status::OtherAllele + }; + let gq = c + .get(5) + .filter(|g| !g.is_empty()) + .unwrap_or(&".") + .to_string(); + result.insert( + key, + SiteGt { + alleles: al, + status, + gq, + snv, + }, + ); + } + Ok(result) +} +fn read_phased(path: &str, wanted: &HashSet) -> Result { + let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut text = String::new(); + let mut out = HashMap::new(); + while lines.read_line(&mut text).map_err(|e| e.to_string())? { + if text.starts_with('#') { + continue; + } + let c: Vec<_> = text.trim_end_matches('\n').split('\t').collect(); + if c.len() < 10 { + continue; + } + let key = (c[0].to_owned(), c[1].to_owned()); + if !wanted.contains(&key) { + continue; + } + let fmt: HashMap<_, _> = c[8].split(':').zip(c[9].split(':')).collect(); + let (Some(gt), Some(ps)) = (fmt.get("GT"), fmt.get("PS")) else { + continue; + }; + if ps.is_empty() || *ps == "." || !gt.contains('|') { + continue; + } + let bases: Vec<_> = std::iter::once(c[3]).chain(c[4].split(',')).collect(); + let idx: Option> = gt + .split('|') + .map(|i| { + pycompat::num::parse_int(i) + .ok() + .and_then(|i| i.to_string().parse::().ok()) + .and_then(|i| { + let n = if i < 0 { bases.len() as isize + i } else { i }; + usize::try_from(n).ok().and_then(|n| bases.get(n).copied()) + }) + }) + .collect(); + if let Some(al) = idx { + if al.len() == 2 { + out.insert(key, (al[0].into(), al[1].into(), ps.to_string())); + } + } + } + Ok(out) +} +fn rule_line(r: &Rule, gt: &Genotypes, phased: Option<&Phased>) -> Row { + let g: Vec<_> = r + .sites + .iter() + .map(|s| gt.get(&s.key()).cloned().unwrap_or_default()) + .collect(); + let rs: Vec<_> = r.sites.iter().map(|s| s.row["rsid"].as_str()).collect(); + let mut out = r.meta.clone(); + for (k, v) in [ + ("rule", r.id.clone()), + ("rsids", rs.join(",")), + ("chrom", r.sites[0].row["chrom"].clone()), + ( + "pos", + r.sites + .iter() + .map(|s| s.row["pos"].as_str()) + .collect::>() + .join(","), + ), + ( + "genotypes", + g.iter() + .map(|g| { + if g.alleles.is_empty() { + ".".into() + } else { + g.alleles.join("/") + } + }) + .collect::>() + .join(","), + ), + ( + "gq", + g.iter() + .map(|g| g.gq.as_str()) + .collect::>() + .join(","), + ), + ] { + out.insert(k.into(), v); + } + let finish = |mut out: Row, phase: &str, result: &str, status: Status, reading: String| { + for (k, v) in [ + ("phase", phase), + ("result", result), + ("status", status.text()), + ("reading", &reading), + ] { + out.insert(k.into(), v.into()); + } + out + }; + if g.iter().any(|g| g.status != Status::Called) { + let need = if rs.len() == 2 { + format!("both {} and {}", rs[0], rs[1]) + } else { + format!( + "every one of {} and {}", + rs[..rs.len() - 1].join(", "), + rs[rs.len() - 1] + ) + }; + let status = if g.iter().any(|g| g.status == Status::NotCovered) { + Status::NotCovered + } else { + Status::OtherAllele + }; + return finish( + out, + ".", + ".", + status, + format!("Not read: needs a genotype at {need}."), + ); + } + let mut phase = if r.meta["needs_phase"] == "no" { + "not_used" + } else { + "not_needed" + }; + let key = if r.meta["needs_phase"] == "no" { + allowed(r, &g.iter().map(|g| g.alleles.clone()).collect::>()).join("|") + } else { + let het: Vec<_> = g + .iter() + .enumerate() + .filter(|(_, g)| g.alleles[0] != g.alleles[1]) + .map(|(i, _)| i) + .collect(); + let mut h1: Vec<_> = g.iter().map(|g| g.alleles[0].clone()).collect(); + let mut h2: Vec<_> = g.iter().map(|g| g.alleles[1].clone()).collect(); + if het.len() >= 2 { + let calls: Option> = het + .iter() + .map(|i| phased.and_then(|p| p.get(&r.sites[*i].key()))) + .collect(); + if let Some(calls) = calls.filter(|c| c.iter().all(|v| v.2 == c[0].2)) { + for (i, c) in het.iter().zip(calls) { + h1[*i] = c.0.clone(); + h2[*i] = c.1.clone(); + } + phase = "read_backed"; + } else { + let why = if phased.is_some() { + "no read or read pair covered them together" + } else { + "no phasing ran (no BAM, or phasing failed)" + }; + return finish( + out, + "unphased", + ".", + Status::Unphased, + format!( + "Heterozygous at {} of the {} sites, and {why}, so which alleles share a copy is not known: not read.", + het.len(), + r.sites.len() + ), + ); + } + } + let n1 = r.haps.iter().position(|h| h.alleles == h1); + let n2 = r.haps.iter().position(|h| h.alleles == h2); + if let (Some(a), Some(b)) = (n1, n2) { + format!("{}/{}", r.haps[a.min(b)].name, r.haps[a.max(b)].name) + } else { + let unknown = [(&h1, n1), (&h2, n2)] + .into_iter() + .filter(|(_, n)| n.is_none()) + .map(|(h, _)| h.join(",")) + .collect::>() + .join(" and "); + return finish( + out, + phase, + ".", + Status::OtherHaplotype, + format!( + "A haplotype this rule does not describe ({unknown} at {}): not read.", + rs.join(", ") + ), + ); + } + }; + match r + .combos + .iter() + .rev() + .find(|c| !key.is_empty() && c.key == key) + { + Some(c) => finish(out, phase, &c.label, Status::Called, c.reading.clone()), + None => finish( + out, + phase, + ".", + Status::OtherHaplotype, + "A combination of haplotypes this rule does not describe: not read.".into(), + ), + } +} +fn write_tsv(path: &str, columns: &[&str], rows: &[Row]) -> Result<()> { + let mut file = AtomicFile::create(path).map_err(|e| e.to_string())?; + { + let mut writer = Writer::new(&mut file, '\t'); + writer + .write_row(columns.iter().copied()) + .map_err(|e| e.to_string())?; + for row in rows { + writer + .write_row(columns.iter().map(|k| row[*k].as_str())) + .map_err(|e| e.to_string())?; + } + } + file.commit().map_err(|e| e.to_string()) +} +fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result { + let mut sites = BTreeMap::new(); + for r in rules.iter().filter(|r| r.meta["needs_phase"] == "yes") { + let calls: Option> = r + .sites + .iter() + .map(|s| gt.get(&s.key()).filter(|g| g.status == Status::Called)) + .collect(); + if let Some(calls) = calls { + let het: Vec<_> = r + .sites + .iter() + .zip(calls) + .filter(|(_, g)| g.alleles[0] != g.alleles[1]) + .map(|(s, _)| s) + .collect(); + if het.len() >= 2 { + for s in het { + let chrom = s.row["chrom"] + .strip_prefix("chr") + .and_then(|s| s.parse::().ok()) + .ok_or("invalid chromosome")?; + let pos = s.row["pos"].parse::().map_err(|e| e.to_string())?; + sites.entry((chrom, pos)).or_insert(s); + } + } + } + } + let mut out = AtomicFile::create(path).map_err(|e| e.to_string())?; + let write = |out: &mut AtomicFile| -> std::io::Result<()> { + writeln!(out, "##fileformat=VCFv4.2")?; + for chrom in sites.keys().map(|(c, _)| c).collect::>() { + writeln!(out, "##contig=")?; + } + writeln!( + out, + "##FORMAT=" + )?; + writeln!( + out, + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}" + )?; + for s in sites.values() { + writeln!( + out, + "{}\t{}\t{}\t{}\t{}\t.\tPASS\t.\tGT\t0/1", + s.row["chrom"], s.row["pos"], s.row["rsid"], s.row["ref"], s.row["alt"] + )?; + } + Ok(()) + }; + write(&mut out).map_err(|e| e.to_string())?; + out.commit().map_err(|e| e.to_string())?; + Ok(sites.len()) +} +fn run(verb: &str, opts: &HashMap) -> Result { + let required = |k: &str| { + opts.get(k) + .map(String::as_str) + .ok_or_else(|| format!("missing --{k}")) + }; + if verb == "check" { + let table = required("table")?; + let rows = read_table(table)?; + let mut errs = check_rows(&rows); + if let Some(path) = opts.get("rules") { + errs.extend(check_rules(&read_rules(path)?, Some(&rows))); + } + for e in &errs { + eprintln!("ERROR: {e}"); + } + if errs.is_empty() { + println!( + "{table}{}: schema OK", + opts.get("rules") + .map(|r| format!(" and {r}")) + .unwrap_or_default() + ); + } + return Ok(u8::from(!errs.is_empty())); + } + let rules_path = required("rules")?; + let rules = read_rules(rules_path)?; + let rows = if verb == "table" { + read_table(required("table")?)? + } else { + vec![] + }; + let mut errs = if verb == "table" { + check_rows(&rows) + .into_iter() + .map(|e| format!("{}: {e}", opts["table"])) + .collect::>() + } else { + vec![] + }; + errs.extend( + check_rules(&rules, if verb == "table" { Some(&rows) } else { None }) + .into_iter() + .map(|e| format!("{rules_path}: {e}")), + ); + if !errs.is_empty() { + for e in errs { + eprintln!("ERROR: {e}"); + } + return Ok(1); + } + let sites: Vec<_> = rows + .iter() + .chain(rules.iter().flat_map(|r| &r.sites)) + .collect(); + let gt = read_genotypes(required("genotypes")?, &sites)?; + let sample = required("sample")?; + if verb == "phase-sites" { + let path = required("out")?; + let n = phase_vcf(path, sample, &rules, >)?; + println!("{sample}: {n} heterozygous rule sites to phase. Written: {path}"); + return Ok(0); + } + let wanted = sites.iter().map(|s| s.key()).collect(); + let phased = opts + .get("phased") + .map(|p| read_phased(p, &wanted)) + .transpose()?; + let lines: Vec<_> = rules + .iter() + .map(|r| rule_line(r, >, phased.as_ref())) + .collect(); + let last = rules.iter().find(|r| r.id == "APOE").and_then(|r| { + rows.iter() + .rposition(|s| r.sites.iter().any(|t| s.row["rsid"] == t.row["rsid"])) + }); + let mut out = vec![]; + let mut counts = BTreeMap::::new(); + for (i, s) in rows.iter().enumerate() { + let g = gt.get(&s.key()).cloned().unwrap_or_default(); + let mut r = s.row.clone(); + r.insert( + "genotype".into(), + if g.alleles.is_empty() { + ".".into() + } else { + g.alleles.join("/") + }, + ); + r.insert("status".into(), g.status.text().into()); + let reading = match g.status { + Status::NotCovered => { + "No genotype at this position in the gVCF (no reads, or a no-call): not read." + .into() + } + Status::Called => s.row[["hom_ref", "het", "hom_alt"] + [g.alleles.iter().filter(|a| *a == &s.row["alt"]).count()]] + .clone(), + _ => "An allele this table does not describe at this position: not read.".into(), + }; + r.insert("reading".into(), reading); + out.push(r); + if last == Some(i) { + let mut r = lines + .iter() + .find(|r| r["rule"] == "APOE") + .expect("validated APOE") + .clone(); + r.insert("rsid".into(), r["rsids"].replace(',', "+")); + r.insert("genotype".into(), r["result"].clone()); + out.push(r); + } + } + // Compute both tables before opening either output. Each rename is atomic. + write_tsv(required("out")?, OUT, &out)?; + write_tsv(required("rules-out")?, RULE_OUT, &lines)?; + for r in &out { + *counts.entry(r["status"].clone()).or_default() += 1; + } + println!( + "{sample}: {} table rows; {} (APOE line included). Written: {}", + rows.len(), + counts + .iter() + .map(|(k, v)| format!("{k} {v}")) + .collect::>() + .join(", "), + opts["out"] + ); + println!( + "{sample}: {} genotype rules; {}. Written: {}", + lines.len(), + lines + .iter() + .map(|r| format!( + "{} {}", + r["rule"], + r[if r["status"] == "called" { + "result" + } else { + "status" + }] + )) + .collect::>() + .join(", "), + opts["rules-out"] + ); + Ok(0) +} +/// Run the three traits verbs. Usage errors exit 2; data and I/O errors exit 1. +pub fn cli(args: &[String]) -> u8 { + let usage = "usage: og traits table|phase-sites|check [--table TABLE] --rules RULES [--genotypes G --sample S --out OUT --rules-out OUT]"; + if args.iter().any(|a| a == "--help" || a == "-h") { + println!("{usage}"); + return 0; + } + let Some(verb) = args + .first() + .map(String::as_str) + .filter(|v| ["table", "phase-sites", "check"].contains(v)) + else { + eprintln!("{usage}"); + return 2; + }; + let mut opts = HashMap::new(); + let mut rest = args[1..].iter(); + while let Some(arg) = rest.next() { + let (key, value) = if let Some((k, v)) = arg.split_once('=') { + (k, Some(v.to_string())) + } else { + (arg.as_str(), rest.next().cloned()) + }; + let Some(key) = key.strip_prefix("--").filter(|k| { + [ + "table", + "rules", + "genotypes", + "phased", + "sample", + "out", + "rules-out", + ] + .contains(k) + }) else { + eprintln!("{usage}\nog traits: error: unknown argument {arg}"); + return 2; + }; + let Some(value) = value.filter(|v| !v.starts_with("--")) else { + eprintln!("{usage}\nog traits: error: --{key} needs a value"); + return 2; + }; + opts.insert(key.into(), value); + } + let needs: &[&str] = match verb { + "check" => &["table"], + "phase-sites" => &["rules", "genotypes", "sample", "out"], + _ => &["table", "rules", "genotypes", "sample", "out", "rules-out"], + }; + if let Some(k) = needs.iter().find(|k| !opts.contains_key(**k)) { + eprintln!("{usage}\nog traits: error: missing --{k}"); + return 2; + } + match run(verb, &opts) { + Ok(code) => code, + Err(e) => { + eprintln!("ERROR: {e}"); + 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + struct Scratch(PathBuf); + impl Scratch { + fn new() -> Self { + let p = std::env::temp_dir().join(format!( + "og-traits-{}-{}", + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&p).unwrap(); + Self(p) + } + fn put(&self, name: &str, text: &str) -> String { + let p = self.0.join(name); + std::fs::write(&p, text).unwrap(); + p.to_string_lossy().into_owned() + } + } + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + fn assets() -> (Vec, Vec) { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); + ( + read_table(root.join("traits.tsv").to_str().unwrap()).unwrap(), + read_rules(root.join("genotype_rules.tsv").to_str().unwrap()).unwrap(), + ) + } + fn genotypes(r: &Rule, choices: &[u8]) -> Genotypes { + r.sites + .iter() + .zip(choices) + .map(|(s, v)| { + let al = vec![ + s.row[if *v == 2 { "alt" } else { "ref" }].clone(), + s.row[if *v == 0 { "ref" } else { "alt" }].clone(), + ]; + ( + s.key(), + SiteGt { + alleles: al, + status: Status::Called, + gq: "40".into(), + snv: true, + }, + ) + }) + .collect() + } + #[test] + fn committed_catalogs_and_every_rule_genotype() { + let (rows, rules) = assets(); + assert!(check_rows(&rows).is_empty()); + assert_eq!(check_rules(&rules, Some(&rows)), Vec::::new()); + for r in &rules { + for mut n in 0..3usize.pow(r.sites.len() as u32) { + let mut choices = vec![]; + for _ in &r.sites { + choices.push((n % 3) as u8); + n /= 3; + } + let gt = genotypes(r, &choices); + let line = rule_line(r, >, None); + assert!(RULE_OUT.iter().all(|k| line.contains_key(*k))); + } + } + } + #[test] + fn apoe_grid_keeps_haplotype_order_and_missing_calls() { + let (_, rules) = assets(); + let r = rules.iter().find(|r| r.id == "APOE").unwrap(); + let expected = [ + ["e3/e3", "e2/e3", "e2/e2"], + ["e3/e4", "e2/e4", "e1/e2"], + ["e4/e4", "e1/e4", "e1/e1"], + ]; + for (a, labels) in expected.iter().enumerate() { + for (b, label) in labels.iter().enumerate() { + let line = rule_line(r, &genotypes(r, &[a as u8, b as u8]), None); + assert_eq!(line["status"], "called"); + assert_eq!(line["result"], *label); + } + } + let mut gt = genotypes(r, &[1, 0]); + gt.remove(&r.sites[1].key()); + let line = rule_line(r, >, None); + assert_eq!(line["status"], "not_covered"); + assert_eq!(line["phase"], "."); + } + #[test] + fn schema_faults_are_refused_before_enumeration() { + let (rows, rules) = assets(); + for (k, v, needle) in [ + ("ref", "N", "not one of"), + ("alt", rows[0].row["ref"].as_str(), "same base"), + ("pmids", "PMC123", "PubMed"), + ("chrom", "chrX", "chr1 to chr22"), + ("het", " ", "empty"), + ("note", "From SNPedia.", "SNPedia"), + ] { + let mut bad = rows.clone(); + bad[0].row.insert(k.into(), v.into()); + assert!(check_rows(&bad).iter().any(|e| e.contains(needle)), "{k}"); + } + let mut bad = rows.clone(); + bad.remove(bad.iter().position(|s| s.row["rsid"] == "rs7412").unwrap()); + assert!(check_rows(&bad).iter().any(|e| e.contains("missing"))); + let mut bad = rules.clone(); + bad[0].combos.pop(); + assert!( + check_rules(&bad, Some(&rows)) + .iter() + .any(|e| e.contains("no combination")) + ); + let mut bad = rules.clone(); + bad[0].haps[0].alleles[0] = "N".into(); + assert!( + check_rules(&bad, None) + .iter() + .any(|e| e.contains("neither")) + ); + let mut bad = rules.clone(); + bad[0].sites.clear(); + assert!(!check_rules(&bad, None).is_empty()); + let mut bad = rules.clone(); + bad[0].sites[0].row.insert("pos".into(), "9".into()); + assert!( + check_rules(&bad, Some(&rows)) + .iter() + .any(|e| e.contains("traits table has")) + ); + let mut bad = rules.clone(); + bad[0].combos[0].key = "absent/absent".into(); + assert!( + check_rules(&bad, None) + .iter() + .any(|e| e.contains("not two")) + ); + let mut bad = rules.clone(); + bad[0].combos[1].key = "e3/e2".into(); + assert!(check_rules(&bad, None).iter().any(|e| e.contains("order"))); + assert!( + check_rules(&rules[1..], Some(&rows)) + .iter() + .any(|e| e.contains("rule APOE is missing")) + ); + } + #[test] + fn stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory() { + let (rows, _) = assets(); + let s = &rows[0]; + let scratch = Scratch::new(); + let prefix = format!("{}\t{}\t", s.row["chrom"], s.row["pos"]); + let a = &s.row["ref"]; + let b = &s.row["alt"]; + for (gt, status) in [ + (format!("{a}|{b}"), Status::Called), + (format!("./{b}"), Status::OtherAllele), + (a.clone(), Status::OtherAllele), + ("é/é".into(), Status::OtherAllele), + ] { + let path = scratch.put("g", &format!("{prefix}{a}\t{b}\t{gt}\t99\n")); + let result = read_genotypes(&path, &[s]).unwrap(); + assert_eq!(result[&s.key()].status, status); + } + let text = format!( + "{prefix}{a}T\t{a}\t{a}T/{a}\n{prefix}{a}\t{b}\t{b}/{a}\t42\n{prefix}{a}\t{b}\t{b}/{b}\t1\n" + ); + let path = scratch.put("g", &text); + let result = read_genotypes(&path, &[s]).unwrap(); + assert_eq!(result[&s.key()].alleles, vec![a.clone(), b.clone()]); + assert_eq!(result[&s.key()].gq, "42"); + let path = scratch.put("bad", &(text + &format!("{prefix}N\t{b}\tN/N\n"))); + assert!(read_genotypes(&path, &[s]).unwrap_err().contains("GRCh38")); + let path = scratch.put("many", &"chr99\t1\tN\t.\tN/N\n".repeat(10000)); + assert!(read_genotypes(&path, &[s]).unwrap().is_empty()); + } + #[test] + fn phase_requires_one_block_and_only_multiple_hets_are_exported() { + let (_, rules) = assets(); + let r = rules.iter().find(|r| r.id == "TAS2R38").unwrap(); + let gt = genotypes(r, &[1, 1, 1]); + assert_eq!(rule_line(r, >, None)["phase"], "unphased"); + assert!( + rule_line(r, >, Some(&HashMap::new()))["reading"].contains("no read or read pair") + ); + let scratch = Scratch::new(); + let mut vcf = String::new(); + let hap1 = &r.haps[0]; + let hap2 = &r.haps[1]; + for (i, s) in r.sites.iter().enumerate() { + let a = usize::from(hap1.alleles[i] == s.row["alt"]); + let b = usize::from(hap2.alleles[i] == s.row["alt"]); + vcf += &format!( + "{}\t{}\t.\t{}\t{}\t.\tPASS\t.\tGT:PS\t{a}|{b}:7\n", + s.row["chrom"], s.row["pos"], s.row["ref"], s.row["alt"] + ); + } + let p = scratch.put("phased", &vcf); + let mut phased = read_phased(&p, &r.sites.iter().map(Site::key).collect()).unwrap(); + assert_eq!(rule_line(r, >, Some(&phased))["phase"], "read_backed"); + phased.get_mut(&r.sites[0].key()).unwrap().2 = "8".into(); + assert_eq!(rule_line(r, >, Some(&phased))["status"], "unphased"); + let path = scratch.put("phase", ""); + assert_eq!(phase_vcf(&path, "S", &rules, >).unwrap(), 3); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.contains("141972804\t")); + assert!(!text.contains("rs429358")); + assert_eq!( + phase_vcf(&path, "S", &rules, &genotypes(r, &[1, 0, 0])).unwrap(), + 0 + ); + let gt = genotypes(r, &[0, 0, 0]); + assert_eq!(rule_line(r, >, None)["phase"], "not_needed"); + } + #[test] + fn csv_header_unicode_rules_and_atomic_write_errors() { + let scratch = Scratch::new(); + let mut text = COLUMNS + .iter() + .map(|k| format!("\"{k}\"")) + .collect::>() + .join("\t"); + text.push_str("\r\n"); + let path = scratch.put("quoted", &text); + assert!(read_table(&path).unwrap().is_empty()); + let path = scratch.put("r", "rule\té\tG\tT\tno\tyes\tN\t1\n"); + assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); + assert!(write_tsv(scratch.0.join("absent/x").to_str().unwrap(), OUT, &[]).is_err()); + assert_eq!(cli(&["table".into()]), 2); + assert_eq!( + cli(&[ + "check".into(), + "--table".into(), + scratch.0.join("absent").to_string_lossy().into_owned() + ]), + 1 + ); + } +} diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 29c44efa..7df626a0 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -53,6 +53,9 @@ const USAGE: &str = "usage: og [args] og summary collect --only a [args] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] + og traits table [args] + og traits phase-sites [args] + og traits check [args] og bloodtype report|check-markers|systems [args] og family plan|prepare|check|report|segments [args] og family ibd [args] @@ -216,6 +219,7 @@ fn main() -> ExitCode { )), None => usage_error("the following arguments are required: chip tool"), }, + "traits" => ExitCode::from(og_steps::traits::cli(rest)), "bloodtype" => match rest.first() { Some(verb) => ExitCode::from(og_bloodtype::cli( verb, diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 0a9afd6b..2a2fe1e3 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -950,3 +950,30 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +traits empty gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes empty.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits reference gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes reference.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-00 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-00.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-01 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-01.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-02 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-02.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-10 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-10.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-11 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-11.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-12 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-12.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-20 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-20.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-21 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-21.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits apoe-22 gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes apoe-22.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits other gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes other.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits missing gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes missing.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits duplicate gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes duplicate.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits half gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes half.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits wrong-ref gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes wrong-ref.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv - tsv PYTHON_IMAGE +traits triple gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes triple.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits single gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes single.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits phase-empty gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes triple.tsv --phased empty.vcf --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits phase-phased gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes triple.tsv --phased phased.vcf --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits phase-split gen:traits.sh table --table traits.tsv --rules rules.tsv --genotypes triple.tsv --phased split.vcf --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE +traits sites-empty gen:traits.sh phase-sites --rules rules.tsv --genotypes empty.tsv --sample S --out phase.vcf phase.vcf vcf PYTHON_IMAGE +traits sites-triple gen:traits.sh phase-sites --rules rules.tsv --genotypes triple.tsv --sample S --out phase.vcf phase.vcf vcf PYTHON_IMAGE +traits sites-single gen:traits.sh phase-sites --rules rules.tsv --genotypes single.tsv --sample S --out phase.vcf phase.vcf vcf PYTHON_IMAGE +traits check gen:traits.sh check --table traits.tsv --rules rules.tsv - bytes PYTHON_IMAGE +traits bad-table gen:traits.sh check --table bad-table.tsv --rules rules.tsv - bytes PYTHON_IMAGE +traits bad-rules gen:traits.sh check --table traits.tsv --rules bad-rules.tsv - bytes PYTHON_IMAGE diff --git a/tests/parity/gen/traits.sh b/tests/parity/gen/traits.sh new file mode 100644 index 00000000..10d0fa26 --- /dev/null +++ b/tests/parity/gen/traits.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Public catalogs and synthetic genotype grids for all three traits verbs. +set -euo pipefail +root=$(cd "$(dirname "$0")/../../.." && pwd) +cp "$root/assets/traits.tsv" traits.tsv +cp "$root/assets/genotype_rules.tsv" rules.tsv +cat > oracle.sh <<'EOF' +prog=$1; shift +mode=$1; shift +case "$mode" in + table) exec python3 "$prog" "$@" ;; + check) exec python3 "$prog" "${@/--table/--check-table}" ;; + phase-sites) + args=() + while [ "$#" -gt 0 ]; do + if [ "$1" = --out ]; then args+=(--phase-sites "$2"); shift 2 + else args+=("$1"); shift; fi + done + exec python3 "$prog" "${args[@]}" ;; +esac +EOF +: > empty.tsv +awk -F'\t' '$1 !~ /^#/ && $1 != "chrom" {print $1 "\t" $2 "\t" $5 "\t" $6 "\t" $5 "/" $5 "\t40"}' traits.tsv > reference.tsv +for a in 0 1 2; do + for b in 0 1 2; do + case $a in 0) ga=T/T;; 1) ga=T/C;; 2) ga=C/C;; esac + case $b in 0) gb=C/C;; 1) gb=C/T;; 2) gb=T/T;; esac + printf 'chr19\t44908684\tT\tC\t%s\t40\nchr19\t44908822\tC\tT\t%s\t39\n' "$ga" "$gb" > "apoe-$a$b.tsv" + done +done +printf 'chr19\t44908684\tT\tC\tT/G\t30\nchr19\t44908822\tC\tT\tC/C\t40\n' > other.tsv +printf 'chr19\t44908684\tT\tC\tT/C\t30\n' > missing.tsv +printf 'chr19\t44908684\tTT\tT\tTT/T\t10\nchr19\t44908684\tT\tC\tC/T\t40\n' > duplicate.tsv +printf 'chr19\t44908684\tG\tC\tG/C\t30\n' > wrong-ref.tsv +printf 'chr19\t44908684\tT\tC\t./C\t30\n' > half.tsv +printf 'chr7\t141973545\tC\tG\tC/G\t30\nchr7\t141972905\tG\tC\tG/C\t31\nchr7\t141972804\tT\tC\tT/C\t32\n' > triple.tsv +head -1 triple.tsv > single.tsv +: > empty.vcf +printf 'chr7\t141973545\t.\tC\tG\t.\tPASS\t.\tGT:PS\t1|0:7\nchr7\t141972905\t.\tG\tC\t.\tPASS\t.\tGT:PS\t0|1:7\nchr7\t141972804\t.\tT\tC\t.\tPASS\t.\tGT:PS\t1|0:7\n' > phased.vcf +sed '1s/:7$/:8/' phased.vcf > split.vcf +awk -F'\t' -v OFS='\t' '$1 == "combination" && $2 == "APOE" && $3 == "e3/e3" {next} {print}' rules.tsv > bad-rules.tsv +awk -F'\t' -v OFS='\t' '$3 == "rs671" {$6=$5} {print}' traits.tsv > bad-table.tsv diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 57c27da2..52679749 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -63,3 +63,4 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +traits 669a8196 file:bin/traits_report.py bash ./oracle.sh traits From c4e77149c378507c8246b74db17c37b200f65483 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:10:38 +0100 Subject: [PATCH 2/8] refactor(traits): switch step 42 and remove the replaced Python Bead: pgp-bs9u.25 --- .github/workflows/guard.yml | 2 +- CHANGELOG.md | 4 + bin/copy_number.py | 344 ------------- bin/traits_report.py | 605 +---------------------- docs/42-traits.md | 6 +- docs/rust.md | 5 + modules/local/traits/main.nf | 6 +- rust/README.md | 4 + rust/crates/og-steps/src/traits.rs | 36 +- scripts/42-traits.sh | 4 +- scripts/ci/e2e-case-paths.tsv | 71 +-- scripts/ci/e2e-case-times.tsv | 1 + tests/demo/make_demo_sample.py | 20 +- tests/e2e/genotype-rules-2-g2.sh | 14 +- tests/e2e/report-followups-1-sections.sh | 16 +- tests/e2e/rust-c2b-traits.sh | 28 ++ tests/parity/README.md | 5 + tests/parity/gen/traits.sh | 4 +- tests/parity/oracle.ref | 2 +- tests/parity/remnants.tsv | 1 + tests/test_collect_summary.py | 16 +- tests/test_copy_number.py | 25 +- tests/test_genotype_rules_indels.py | 5 +- tests/test_hba_copy_number.py | 2 +- tests/test_lpa_kiv2.py | 2 +- tests/test_render_report.py | 5 +- tests/test_traits_report.py | 481 +++--------------- tests/test_traits_table.py | 43 +- 28 files changed, 267 insertions(+), 1490 deletions(-) delete mode 100755 bin/copy_number.py create mode 100755 tests/e2e/rust-c2b-traits.sh diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 304360fb..7ee58a0d 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -351,7 +351,7 @@ jobs: tests/test_results_index.py \ tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_callability.py; do + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_genotype_rules_indels.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in diff --git a/CHANGELOG.md b/CHANGELOG.md index cd74b71a..8400d72d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error instead of a traceback. +- The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. + + - The container helpers check (`scripts/ci/check-container-helpers.sh`, Guard job `helper-binaries`) now derives its table with the Rust `og` binary, `og dev helpers`. It replaces the Python inside the script, which is gone; the script keeps the docker probe, the orad download and its self-test. On this tree the table and the rule 1 output are the same, byte for byte. The static run takes 0.01 s and 6 MB where the Python took 0.2 s and 15 MB. What changes: - A helper called in a process's `shell:` block is checked too. The Python read only `script:` blocks, so such a call went unchecked. No module has one today. - A helper called in a process's `exec:` block always fails the check: Nextflow runs that block on the host, where no image applies. diff --git a/bin/copy_number.py b/bin/copy_number.py deleted file mode 100755 index 2e1b9548..00000000 --- a/bin/copy_number.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""copy_number.py: the catalog reader and the depth-ratio engine of -bin/cn_loci.tsv, as the catalog tests (tests/test_copy_number.py, -test_hba_copy_number.py, test_lpa_kiv2.py) import them. No step runs it. - -The commands are og cn's (rust/crates/og-cn, docs/cn-loci.md): `og cn bed` -prints a locus's segments, `og cn call` turns `samtools bedcov` over them -into the step's table, `og cn intervals` reads the interval rows and `og cn -check-bam` refuses a BAM aligned with ALT contigs, and `og bloodtype -report` reads step 41's RHD row with og cn's engine. This file keeps -load_catalog, assess, cut_copies, BrokenControl and CatalogError for those -tests until they move to og too. - -One method, one catalog row per locus: the depth over the locus's target -segments against the depth over single-copy control segments, as -`samtools bedcov` sums it. With T the target depth and C the control depth -(each combined as the row's target_depth and control_depth say), - estimate = scale x T / C - offset -then guarded by a minimum control depth, flank agreement and the share of the -target depth from reads of mapping quality 0, and called as the row's `call` -column says. docs/cn-loci.md describes the columns and how to add a locus. -Standard library only. -""" -import csv -import math -import os - -CATALOG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cn_loci.tsv") -COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", - "scale", "offset", "bedcov", "mapq0_min", "mapq0_max", "min_control_depth", "max_control_ratio", - "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] -KINDS = ("gene", "interval", "organelle") -# GRCh38 names of the primary chromosomes (the ClinGen BED spells chrX 'chrx': -# scripts/ci/build-cn-intervals.py normalises it, and a row with any other -# spelling is refused here). -CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} -INPUTS = ("bedcov", "cnvpytor-level") -Z95 = 1.96 -# The findings a psv row needs a reading for. -PSV_FINDINGS = ("none", "deletion", "conversion", "pseudogene_deletion") -BASES = "ACGT" - - -class CatalogError(ValueError): - pass - - -class BrokenControl(ValueError): - """A control segment with no read while the other controls are covered.""" - - -def _num(row, col, default=None): - v = row[col] - if v == ".": - return default - try: - x = float(v) - except ValueError: - x = math.nan - if not math.isfinite(x): - raise CatalogError(f"{row['locus']}: {col} is {v!r}, not a number or '.'") - return x - - -def _segments(row, col): - """[(name, start, end, label)] of a targets, controls or details cell.""" - if row[col] == ".": - return [] - out = [] - for part in row[col].split(";"): - bits = part.split(":", 2) - try: - name, (s, e) = bits[0], bits[1].split("-") - start, end = int(s), int(e) - except (IndexError, ValueError): - raise CatalogError(f"{row['locus']}: {col} segment {part!r} is not NAME:start-end[:label]") - if not name or end <= start or start < 0: - raise CatalogError(f"{row['locus']}: {col} segment {part!r} has no name or an empty span") - out.append((name, start, end, bits[2] if len(bits) > 2 else "")) - return out - - -def _readings(row): - out = {} - if row["readings"] == ".": - return out - for part in row["readings"].split("|"): - k, sep, text = part.partition("=") - if not sep or not k or not text: - raise CatalogError(f"{row['locus']}: reading {part!r} is not KEY=text") - out[k] = text - return out - - -def _call(row): - mode, sep, args = row["call"].partition(":") - try: - vals = [float(x) for x in args.split(",")] - except ValueError: - vals = [] - if not all(math.isfinite(v) for v in vals): - vals = [] - if not sep or mode not in ("round", "cut", "class", "psv") or not vals \ - or (mode in ("round", "psv") and len(vals) != 1) or (mode == "class" and len(vals) != 2) \ - or vals != sorted(vals): - raise CatalogError(f"{row['locus']}: call {row['call']!r} is not round:D, cut:A,B..., class:LOW,HIGH or psv:D") - return mode, vals - - -def _psv_sites(row, details): - """[(name, gene 0-based position, pseudogene 0-based position, gene - bases, pseudogene bases, text)] of a psv row: every details segment is one - base, labelled 'PSEUDO_POS GENE>PSEUDO text' (bases among ACGT; In2G's - gene base is A or C).""" - out = [] - for name, s, e, label in details: - bits = label.split(" ", 2) - try: - pos = int(bits[0]) - gene, _, pseudo = bits[1].partition(">") - except (IndexError, ValueError): - raise CatalogError(f"{row['locus']}: PSV {name} label {label!r} is not 'PSEUDO_POS GENE>PSEUDO text'") - if e - s != 1 or pos < 0 or not gene or not pseudo or set(gene + pseudo) - set(BASES) \ - or set(gene) & set(pseudo) or len(bits) < 3: - raise CatalogError(f"{row['locus']}: PSV {name} must be one base with distinct gene and pseudogene " - f"bases and a text (got {s}-{e}, {label!r})") - out.append((name, s, pos, set(gene), set(pseudo), bits[2])) - return out - - -def parse_row(row): - """The catalog row checked, with its parsed fields under '_' keys.""" - loc = row["locus"] - if row["kind"] not in KINDS: - raise CatalogError(f"{loc}: kind {row['kind']!r} (one of {', '.join(KINDS)})") - if row["input"] not in INPUTS: - raise CatalogError(f"{loc}: input {row['input']!r} (one of {', '.join(INPUTS)})") - if row["chrom"] not in CHROMS: - raise CatalogError(f"{loc}: chrom {row['chrom']!r} (GRCh38 names, chr1 to chrM)") - p = dict(row) - p["_targets"], p["_controls"], p["_details"] = (_segments(row, c) for c in ("targets", "controls", "details")) - p["_call"] = _call(row) - p["_readings"] = _readings(row) - p["_psv"] = _psv_sites(row, p["_details"]) if p["_call"][0] == "psv" else [] - if p["_call"][0] == "psv" and (not p["_psv"] or row["target_depth"] != "sum" - or not set(PSV_FINDINGS) <= set(p["_readings"])): - raise CatalogError(f"{loc}: call psv needs PSV sites in details, target_depth sum (gene and pseudogene) " - f"and a reading for each of {', '.join(PSV_FINDINGS)}") - if not p["_targets"]: - raise CatalogError(f"{loc}: no target segment") - if row["input"] == "bedcov": - if not p["_controls"]: - raise CatalogError(f"{loc}: a bedcov row needs control segments") - if row["target_depth"] not in ("sum", "pooled") or row["control_depth"] not in ("flanks", "pooled"): - raise CatalogError(f"{loc}: target_depth sum|pooled and control_depth flanks|pooled") - p["_scale"], p["_offset"] = _num(row, "scale"), _num(row, "offset", 0.0) - if not p["_scale"] or p["_scale"] <= 0: - raise CatalogError(f"{loc}: scale must be a positive number") - p["_min_control"] = _num(row, "min_control_depth", 0.0) - p["_max_ratio"] = _num(row, "max_control_ratio") - p["_mapq0_min"], p["_mapq0_max"] = _num(row, "mapq0_min"), _num(row, "mapq0_max") - p["_passes"] = row["bedcov"].split(",") - if (p["_mapq0_min"] is not None or p["_mapq0_max"] is not None) and len(p["_passes"]) != 2: - raise CatalogError(f"{loc}: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)") - if p["_call"][0] == "class" and row["interval"] != "cv": - raise CatalogError(f"{loc}: call class needs interval cv") - if row["interval"] not in ("cv", "."): - raise CatalogError(f"{loc}: interval {row['interval']!r} (cv or '.')") - names = [n for n, *_ in p["_targets"] + p["_controls"] + p["_details"]] - if len(names) != len(set(names)): - raise CatalogError(f"{loc}: a segment name is used twice") - return p - - -def load_catalog(path=None): - """{locus: parsed row} of the catalog, in file order.""" - path = path or CATALOG - with open(path) as f: - lines = [ln for ln in f if ln.strip() and not ln.startswith("#")] - rows = list(csv.DictReader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)) - if not rows or list(rows[0].keys()) != COLUMNS: - raise CatalogError(f"{path}: the header must be {' '.join(COLUMNS)}") - out, seen = {}, set() - for r in rows: - if None in r or None in r.values(): - raise CatalogError(f"{path}: row {r.get('locus')!r} has the wrong number of columns") - if r["locus"] in out: - raise CatalogError(f"{path}: locus {r['locus']} is listed twice") - p = parse_row(r) - for n, *_ in p["_targets"] + p["_controls"] + p["_details"]: - if n in seen: - raise CatalogError(f"{path}: segment name {n} is used by two loci") - seen.add(n) - out[r["locus"]] = p - return out - - -def regions(row): - """[(chrom, start, end, name)] of a row's segments, sorted by position.""" - segs = row["_controls"] + row["_targets"] + row["_details"] - return sorted(((row["chrom"], s, e, n) for n, s, e, _ in segs), key=lambda r: (r[1], r[2])) - - -def _combine(mode, segs, m): - if mode == "sum": - return sum(m[n] for n, *_ in segs) - if mode == "flanks": - return sum(m[n] for n, *_ in segs) / len(segs) - return sum(m[n] * (e - s) for n, s, e, _ in segs) / sum(e - s for _, s, e, _ in segs) - - -def _cv(values): - """Coefficient of variation (sample standard deviation over the mean).""" - mu = sum(values) / len(values) - if mu <= 0 or len(values) < 2: - return 0.0 - return math.sqrt(sum((v - mu) ** 2 for v in values) / (len(values) - 1)) / mu - - -def cut_copies(value, cuts): - """The number of cut points a value reaches (cut:0.5,1.5 gives 0, 1 or 2).""" - return sum(value >= c for c in cuts) - - -def reading(row, key): - r = row["_readings"] - if str(key) in r: - return r[str(key)] - if isinstance(key, int) and "more" in r and key > max(int(k) for k in r if k.isdigit()): - return r["more"].format(n=key) - return "" - - -def ploidy(row, sex): - """The copies of the row's chromosome a sample has: 2, or 1 on a male's - chrX or chrY; None where it cannot be said (a chrY row on a female or on - a sample of no known sex, a chrX row without a sex).""" - if row["chrom"] not in ("chrX", "chrY"): - return 2 - if sex == "male": - return 1 - if sex == "female" and row["chrom"] == "chrX": - return 2 - return None - - -def assess(row, m, q1=None, sex=None): - """The locus's block for mean depths m (every read) and q1 (reads of - MAPQ >= 1, or None for a row without a MAPQ 0 guard; mapq0_share is then - None). - - status is 'called' or 'not assessed'; reason names the guard that - stopped it: no_data, low_control_depth, controls_disagree, - low_mapq0_share, high_mapq0_share, below_floor, between_counts, - sex_unknown; status 'not applicable' (reason no_y) for a chrY row on a - female. sex matters only for a row on chrX or chrY: there the controls - sit on the same chromosome, so the scale follows the sample's count - there (ploidy). expected is the count with no change.""" - guarded = q1 is not None - if not guarded and (row["_mapq0_min"] is not None or row["_mapq0_max"] is not None): - raise ValueError(f"{row['locus']} has a MAPQ 0 guard: the MAPQ >= 1 depths are needed") - q1 = q1 if guarded else {} - tg, ct = row["_targets"], row["_controls"] - b = {"locus": row["locus"], "status": "not assessed", "reason": None, "estimate": None, "total": None, - "copies": None, "ci95": None, "class": None, "target_depth": None, "control_depth": None, - "depths": dict(m), "mapq0_share": None, "details": {}, "reading": "", "expected": None} - # expected: the copies with no change, the ploidy times the reference - # units (each target segment is one with target_depth sum: AZFc's 4 DAZ) - units = len(row["_targets"]) if row.get("target_depth") == "sum" else 1 - pl = ploidy(row, sex) - b["expected"] = None if pl is None else pl * units - if row["chrom"] == "chrY" and sex == "female": - b.update(status="not applicable", reason="no_y") - return b - if pl is None: - b["reason"] = "sex_unknown" - return b - if not m or any(n not in m for n, *_ in tg + ct): - b["reason"] = "no_data" - return b - for n, *_ in ct: - rest = [c for c in ct if c[0] != n] - if m[n] == 0 and rest and _combine(row["control_depth"], rest, m) >= row["_min_control"] > 0: - raise BrokenControl(f"BROKEN_CONTROL: control segment {n} of {row['locus']} has no read while the " - f"other controls have {_combine(row['control_depth'], rest, m):.1f}x; check its " - "coordinates in bin/cn_loci.tsv (or the sample lacks that stretch)") - T, C = _combine(row["target_depth"], tg, m), _combine(row["control_depth"], ct, m) - tq = _combine(row["target_depth"], tg, {n: q1.get(n, 0.0) for n, *_ in tg}) - # The MAPQ 0 share is known only when the MAPQ >= 1 pass was given. - share = (1 - tq / T if T > 0 else 0.0) if guarded else None - b.update(target_depth=T, control_depth=C, mapq0_share=share) - low = min(m[n] for n, *_ in ct) if row["control_depth"] == "flanks" else C - if low < row["_min_control"] or C <= 0: - b["reason"] = "low_control_depth" - return b - if row["control_depth"] == "flanks" and row["_max_ratio"] is not None and len(ct) > 1: - vals = [m[n] for n, *_ in ct] - if min(vals) <= 0 or max(vals) / min(vals) > row["_max_ratio"]: - b["reason"] = "controls_disagree" - return b - total = row["_scale"] * pl / 2 * T / C - est = total - row["_offset"] - b.update(total=total, estimate=est, - details={n: row["_scale"] * pl / 2 * m[n] / C for n, *_ in row["_details"] if n in m}) - # With no copy left there is no read to judge the aligner by (a row with - # an offset has a floor of its own below, after this guard). - no_copy = row["_offset"] == 0 and round(est) == 0 - if row["_mapq0_min"] is not None and not no_copy and share < row["_mapq0_min"]: - b["reason"] = "low_mapq0_share" - return b - if row["_mapq0_max"] is not None and share > row["_mapq0_max"]: - b["reason"] = "high_mapq0_share" - return b - if row["_offset"] > 0 and est <= 0: - b["reason"] = "below_floor" - return b - mode, vals = row["_call"] - if row["interval"] == "cv": - rel = math.sqrt(_cv([m[n] for n, *_ in tg]) ** 2 / len(tg) + _cv([m[n] for n, *_ in ct]) ** 2 / len(ct)) - b["ci95"] = (max(0.0, total * math.exp(-Z95 * rel) - row["_offset"]), - total * math.exp(Z95 * rel) - row["_offset"]) - if mode == "psv": - # the module total (gene plus pseudogene copies); og cn reads the sites - n = round(est) - if abs(est - n) > vals[0]: - b["reason"] = "between_counts" - return b - b["module_total"] = n - elif mode == "round": - n = round(est) - if abs(est - n) > vals[0]: - b["reason"] = "between_counts" - return b - b["copies"] = n - b["reading"] = reading(row, n) - elif mode == "cut": - b["copies"] = cut_copies(est, vals) - b["reading"] = reading(row, b["copies"]) - else: - lo, hi = b["ci95"] - b["class"] = "low" if hi < vals[0] else "high" if lo > vals[1] else "typical" - b["reading"] = reading(row, b["class"]) - b["status"] = "called" - return b diff --git a/bin/traits_report.py b/bin/traits_report.py index 6b984f1c..0d8e6731 100755 --- a/bin/traits_report.py +++ b/bin/traits_report.py @@ -1,90 +1,17 @@ -#!/usr/bin/env python3 -"""traits_report.py: the traits table and the genotype rules of step 42, read from the gVCF genotypes. +# remnant: kept for collect_summary, demo and catalog tests; deleted in lane R3. +"""Step 42 metadata still read by the Python report collectors. - traits_report.py --table assets/traits.tsv --rules assets/genotype_rules.tsv --genotypes G.tsv \\ - [--phased P.vcf] --sample S --out S_traits.tsv --rules-out S_genotype_rules.tsv - traits_report.py --rules assets/genotype_rules.tsv --genotypes G.tsv --sample S --phase-sites IN.vcf - traits_report.py --check-table assets/traits.tsv [--rules assets/genotype_rules.tsv] - -G.tsv is what bcftools writes in step 42 (scripts/42-traits.sh and the -Nextflow TRAITS_SITES process run the same commands): one line per gVCF record -at a position of the traits table or of a rule site, after the reference -blocks are expanded to one record per base, as CHROM POS REF ALT GENOTYPE GQ, -the genotype as bases (C/T, or C|T when phased). A position with no line has -no genotype: it is "not_covered", never read as the reference. bcftools drops a -record with a missing allele (a no-call or a half call such as ./T) before -this script, so such a position has no line and reads as not_covered too; a -half call that still reaches here is other_allele. - -Each traits table row becomes one output row with the genotype (the reference -allele first), a status (called, not_covered, other_allele) and the reading -for that genotype. A gVCF whose REF differs from the table's reference base at -a position is not GRCh38 (or the table is wrong): the run stops. - -Each rule of the rules table (assets/genotype_rules.tsv) reads its sites -together and becomes one line of --rules-out: the two haplotypes the genotypes -make, named by the rule's haplotype lines, and the combination line of that -diplotype. A rule with needs_phase no takes the combination line of every -diplotype the unphased genotypes allow (APOE's double heterozygote is e2/e4 -or e1/e3). A rule with needs_phase yes and two or more heterozygous sites -needs their phase: --phase-sites writes those sites as a VCF for WhatsHap, -and --phased reads WhatsHap's answer; sites not phased in one block leave the -rule "unphased", not read. The APOE rule's line also goes into the traits -table, right after its last site row, where the reports read it. - ---check-table checks the schema of the table (and of the rules, with --rules) -and exits 1 on the first list of problems; tests/test_traits_table.py and -tests/test_traits_report.py run the same checks. Standard library only (it -runs in the plain python image and in the WhatsHap image). - -Copy-aware markers (MARKERS): two well-sourced markers of the rule catalog's -list sit in sequence the genome holds more than once, so a gVCF genotype at -one position cannot read them and they are not rule lines. Each is read from -the BAM instead, by the step that counts its copies (og cn marker-regions, -marker-read and call --marker, in rust/crates/og-cn): the reads at every -GRCh38 copy of the site are added up, and the share that carries the marker's -base is a fraction of copies, not a diploid genotype. MARKERS stays here for -the reports (bin/collect_summary.py reads the marker tables with it). - LPA_KIV2_4925 the LPA KIV-2 4925G>A splice variant (Coassin et al. 2017, - PMID 28444229), read by step 46 beside the KIV-2 count - SMN1_27134 SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID - 23788250), read by step 35 beside the SMN1 psCN +The executable implementation is og traits table, phase-sites and check. +Copy-aware marker readers are og cn marker-read. """ -import argparse -import itertools -import os -import re -import sys - COLUMNS = ["chrom", "pos", "rsid", "gene", "ref", "alt", "trait", "hom_ref", "het", "hom_alt", "ancestry", "note", "pmids"] OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids"] RULE_OUT_COLUMNS = ["rule", "gene", "trait", "rsids", "chrom", "pos", "genotypes", "gq", "phase", "result", "status", "reading", "ancestry", "note", "pmids"] -# The columns of each kind of line of the rules table, after the kind and the rule id. -RULE_KINDS = {"rule": ["gene", "trait", "needs_phase", "ancestry", "note", "pmids"], - "site": ["rsid", "chrom", "pos", "ref", "alt"], - "haplotype": ["name", "alleles"], - "combination": ["diplotypes", "label", "reading"]} -CHROMS = {f"chr{n}" for n in range(1, 23)} -BASES = set("ACGT") -# APOE: rs429358 C (Arg112) and rs7412 C (Arg158) make e4; T and T make e2. -# The traits table must keep these two rows: the reports show them, and the -# APOE rule's line goes after them. -APOE_E4_SITE, APOE_E2_SITE = "rs429358", "rs7412" -APOE_ALLELES = {APOE_E4_SITE: ("T", "C"), APOE_E2_SITE: ("C", "T")} -# Rules whose line also goes into the traits table, where the reports read it -# (they fold the APOE lines away). Every other rule's line is in --rules-out. TRAITS_TABLE_RULES = ("APOE",) -# The rules table beside the traits table in the repository, for report()'s callers -# that give none; the command line always takes --rules. -DEFAULT_RULES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "assets", "genotype_rules.tsv") -MAX_RULE_SITES = 8 # the completeness check walks 3^sites genotypes -NOT_COVERED = "No genotype at this position in the gVCF (no reads, or a no-call): not read." -OTHER_ALLELE = "An allele this table does not describe at this position: not read." - class TableError(Exception): pass @@ -115,425 +42,6 @@ def read_table(path): return rows -def text_errors(where, text): - errs = [] - if "snpedia" in text.lower(): - errs.append(f"{where}: mentions SNPedia; the table takes no text from it") - if '"' in text or not text.isascii(): - errs.append(f"{where}: a double quote or a non-ASCII character (the reports read the table as plain TSV)") - return errs - - -def site_errors(where, r): - """Schema problems of one site: chrom, pos, rsid, ref and alt.""" - errs = [] - if r["chrom"] not in CHROMS: - errs.append(f"{where}: chrom '{r['chrom']}' is not chr1 to chr22") - if not re.fullmatch(r"[1-9][0-9]*", r["pos"]): - errs.append(f"{where}: pos '{r['pos']}' is not a positive integer") - if not re.fullmatch(r"rs[1-9][0-9]*", r["rsid"]): - errs.append(f"{where}: rsid is not rs") - for k in ("ref", "alt"): - if r[k] not in BASES: - errs.append(f"{where}: {k} '{r[k]}' is not one of A, C, G, T") - if r["ref"] == r["alt"]: - errs.append(f"{where}: ref and alt are the same base") - return errs - - -def check_rows(rows): - """Every schema problem of the table rows, as messages (empty when it is valid).""" - errs, ids, sites = [], set(), set() - for r in rows: - where = f"line {r['line']} ({r['rsid']})" - errs += site_errors(where, r) - if r["rsid"] in ids: - errs.append(f"{where}: rsid appears twice") - ids.add(r["rsid"]) - if (r["chrom"], r["pos"]) in sites: - errs.append(f"{where}: position {r['chrom']}:{r['pos']} appears twice") - sites.add((r["chrom"], r["pos"])) - for k in ("gene", "trait", "hom_ref", "het", "hom_alt", "note"): - if not r[k].strip(): - errs.append(f"{where}: {k} is empty") - if r["ancestry"] not in ("yes", "no"): - errs.append(f"{where}: ancestry '{r['ancestry']}' is not yes or no") - if not re.fullmatch(r"[1-9][0-9]*(,[1-9][0-9]*)*", r["pmids"]): - errs.append(f"{where}: pmids '{r['pmids']}' is not a comma-separated list of PubMed ids") - errs += text_errors(where, " ".join(r[k] for k in COLUMNS)) - by_id = {r["rsid"]: r for r in rows} - for rs, (ref, alt) in APOE_ALLELES.items(): - r = by_id.get(rs) - if r is None: - errs.append(f"APOE row {rs} is missing") - elif (r["ref"], r["alt"]) != (ref, alt) or r["chrom"] != "chr19": - errs.append(f"APOE row {rs} must be chr19 {ref}>{alt}, is {r['chrom']} {r['ref']}>{r['alt']}") - return errs - - -# --- The rules table --------------------------------------------------------------- - -def read_rules(path): - """{rule id: {gene, trait, needs_phase, ancestry, note, pmids, line, sites, haplotypes, - combinations}} in file order. Raises TableError on a line of an unknown kind or with - the wrong column count, and on a site, haplotype or combination line before its rule line.""" - rules = {} - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line or line.startswith("#"): - continue - cells = line.split("\t") - kind = cells[0] - if kind not in RULE_KINDS: - raise TableError(f"{path}:{n}: kind '{kind}' is not one of {', '.join(RULE_KINDS)}") - want = 2 + len(RULE_KINDS[kind]) - if len(cells) != want: - raise TableError(f"{path}:{n}: a {kind} line has {len(cells)} columns, expected {want}") - rid = cells[1] - d = dict(zip(RULE_KINDS[kind], cells[2:]), line=n) - if kind == "rule": - if rid in rules: - raise TableError(f"{path}:{n}: rule {rid} has two rule lines") - rules[rid] = dict(d, id=rid, sites=[], haplotypes=[], combinations=[]) - continue - if rid not in rules: - raise TableError(f"{path}:{n}: a {kind} line of rule '{rid}' before its rule line") - rules[rid]["sites" if kind == "site" else kind + "s"].append(d) - if not rules: - raise TableError(f"{path}: no rule line") - return rules - - -def diplotype_key(rule, a, b): - """'a/b' with the two names in the order of the rule's haplotype lines.""" - order = [h["name"] for h in rule["haplotypes"]] - return "/".join(sorted((a, b), key=order.index)) - - -def allowed_diplotypes(rule, genotypes): - """The diplotype keys, in haplotype order, whose two haplotypes give these unphased - genotypes (one [allele, allele] per site).""" - haps = rule["haplotypes"] - out = [] - for i, j in itertools.combinations_with_replacement(range(len(haps)), 2): - a, b = haps[i]["alleles"].split(","), haps[j]["alleles"].split(",") - if all(sorted((a[k], b[k])) == sorted(g) for k, g in enumerate(genotypes)): - out.append(f"{haps[i]['name']}/{haps[j]['name']}") - return out - - -def check_rules(rules, table_rows=None): - """Every schema problem of the rules, as messages (empty when they are valid). With - table_rows, also: a rule site that is a traits table row has the row's chrom, pos, ref - and alt, and the rules of TRAITS_TABLE_RULES exist with every site in the table.""" - errs = [] - by_rs = {r["rsid"]: r for r in table_rows or []} - for rid, rule in rules.items(): - n0 = len(errs) - where = f"rule {rid} (line {rule['line']})" - if not re.fullmatch(r"[A-Za-z0-9_]+", rid): - errs.append(f"{where}: the rule id is not letters, digits and _") - for k in ("gene", "trait", "note"): - if not rule[k].strip(): - errs.append(f"{where}: {k} is empty") - for k in ("needs_phase", "ancestry"): - if rule[k] not in ("yes", "no"): - errs.append(f"{where}: {k} '{rule[k]}' is not yes or no") - if not re.fullmatch(r"[1-9][0-9]*(,[1-9][0-9]*)*", rule["pmids"]): - errs.append(f"{where}: pmids '{rule['pmids']}' is not a comma-separated list of PubMed ids") - sites = rule["sites"] - if not 2 <= len(sites) <= MAX_RULE_SITES: - errs.append(f"{where}: {len(sites)} sites, expected 2 to {MAX_RULE_SITES}") - seen = set() - for s in sites: - w = f"{where}, site line {s['line']} ({s['rsid']})" - errs += site_errors(w, s) - if s["rsid"] in seen or (s["chrom"], s["pos"]) in seen: - errs.append(f"{w}: the site appears twice in the rule") - seen |= {s["rsid"], (s["chrom"], s["pos"])} - t = by_rs.get(s["rsid"]) - if t and (t["chrom"], t["pos"], t["ref"], t["alt"]) != (s["chrom"], s["pos"], s["ref"], s["alt"]): - errs.append(f"{w}: the traits table has {t['chrom']}:{t['pos']} {t['ref']}>{t['alt']}, " - f"the rule {s['chrom']}:{s['pos']} {s['ref']}>{s['alt']}") - if len({s["chrom"] for s in sites}) > 1: - errs.append(f"{where}: the sites are on more than one chromosome") - names, tuples = [], set() - for h in rule["haplotypes"]: - w = f"{where}, haplotype line {h['line']} ({h['name']})" - if not re.fullmatch(r"[A-Za-z0-9*.]+", h["name"]): - errs.append(f"{w}: the name is not letters, digits, * and .") - if h["name"] in names: - errs.append(f"{w}: the name appears twice") - names.append(h["name"]) - al = h["alleles"].split(",") - if len(al) != len(sites): - errs.append(f"{w}: {len(al)} alleles for {len(sites)} sites") - continue - for a, s in zip(al, sites): - if a not in (s["ref"], s["alt"]): - errs.append(f"{w}: allele {a} at {s['rsid']} is neither its ref {s['ref']} nor its alt {s['alt']}") - if tuple(al) in tuples: - errs.append(f"{w}: the same alleles as another haplotype") - tuples.add(tuple(al)) - if len(names) < 2: - errs.append(f"{where}: fewer than two haplotypes") - keys = {} - for c in rule["combinations"]: - w = f"{where}, combination line {c['line']} ({c['diplotypes']})" - for k in ("label", "reading"): - if not c[k].strip(): - errs.append(f"{w}: {k} is empty") - errs += text_errors(w, " ".join((c["diplotypes"], c["label"], c["reading"]))) - if c["diplotypes"] in keys: - errs.append(f"{w}: the diplotypes appear twice") - keys[c["diplotypes"]] = c - for d in c["diplotypes"].split("|"): - pair = d.split("/") - if len(pair) != 2 or any(p not in names for p in pair): - errs.append(f"{w}: '{d}' is not two of the rule's haplotypes joined by /") - elif diplotype_key(rule, *pair) != d: - errs.append(f"{w}: '{d}' is not in the order of the haplotype lines " - f"(write {diplotype_key(rule, *pair)})") - errs += text_errors(where, " ".join(str(rule[k]) for k in ("gene", "trait", "note"))) - if len(errs) > n0: - continue - # Completeness: the combination lines are exactly what the genotypes can give. - if rule["needs_phase"] == "yes": - want = {f"{a}/{b}" for a, b in itertools.combinations_with_replacement(names, 2)} - else: - want = set() - choices = [[[s["ref"]] * 2, [s["ref"], s["alt"]], [s["alt"]] * 2] for s in sites] - for g in itertools.product(*choices): - key = "|".join(allowed_diplotypes(rule, g)) - if key: - want.add(key) - for k in sorted(want - set(keys)): - errs.append(f"{where}: no combination line for {k}") - for k in sorted(set(keys) - want): - errs.append(f"{where}: the combination line {k} can never be read " - + ("(a rule that needs phase reads one diplotype)" if "|" in k else "from these haplotypes")) - if table_rows is not None: - for rid in TRAITS_TABLE_RULES: - if rid not in rules: - errs.append(f"rule {rid} is missing: its line goes into the traits table, where the reports read it") - else: - for s in rules[rid]["sites"]: - if s["rsid"] not in by_rs: - errs.append(f"rule {rid}: site {s['rsid']} has no traits table row") - return errs - - -# --- Genotypes ---------------------------------------------------------------------- - -def read_genotypes(path): - """{(chrom, pos): [(REF, [allele, allele], GQ), ...]} from bcftools' CHROM POS REF ALT GT [GQ] lines.""" - recs = {} - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line: - continue - c = line.split("\t") - if len(c) < 5: - raise TableError(f"{path}:{n}: {len(c)} columns, expected CHROM POS REF ALT GENOTYPE [GQ]") - gq = c[5] if len(c) > 5 and c[5] else "." - recs.setdefault((c[0], c[1]), []).append((c[2], re.split(r"[/|]", c[4]), gq)) - return recs - - -def site_genotype(site, recs): - """(alleles sorted with the reference first, status, GQ) of one site from its gVCF records.""" - if not recs: - return None, "not_covered", "." - for r in recs: - if r[0][:1] != site["ref"]: - raise TableError(f"{site['rsid']} at {site['chrom']}:{site['pos']}: the gVCF has REF {r[0]}, the table " - f"{site['ref']} (the GRCh38 base). Is the gVCF on GRCh38?") - snv = [r for r in recs if all(len(a) == 1 for a in r[1])] - _, alleles, gq = (snv or recs)[0] - order = {site["ref"]: 0, site["alt"]: 1} - alleles = sorted(alleles, key=lambda a: (order.get(a, 2), a)) - if len(alleles) != 2 or any(a not in order for a in alleles): - return alleles, "other_allele", gq - return alleles, "called", gq - - -def call(row, recs): - """(genotype, status, reading) of one traits table row from its gVCF records.""" - alleles, status, _ = site_genotype(row, recs) - if status == "not_covered": - return ".", "not_covered", NOT_COVERED - gt = "/".join(alleles) - if status == "other_allele": - return gt, "other_allele", OTHER_ALLELE - return gt, "called", row[("hom_ref", "het", "hom_alt")[alleles.count(row["alt"])]] - - -def read_phased(path): - """{(chrom, pos): (allele of haplotype 1, allele of haplotype 2, phase block)} of the - phased records of WhatsHap's VCF; unphased records are left out.""" - out = {} - with open(path, encoding="utf-8") as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 10: - continue - fmt = dict(zip(c[8].split(":"), c[9].split(":"))) - gt, ps = fmt.get("GT", ""), fmt.get("PS", ".") - if "|" not in gt or ps in (".", ""): - continue - bases = [c[3]] + c[4].split(",") - try: - a, b = (bases[int(i)] for i in gt.split("|")) - except (ValueError, IndexError): - continue - out[(c[0], c[1])] = (a, b, ps) - return out - - -def phase_sites(rules, recs): - """The (chrom, pos, rsid, ref, alt) of the heterozygous sites of every rule that needs - phase and has two or more of them, each site once, in rule order.""" - out, seen = [], set() - for rule in rules.values(): - if rule["needs_phase"] != "yes": - continue - g = [(s, site_genotype(s, recs.get((s["chrom"], s["pos"]), []))) for s in rule["sites"]] - if any(st != "called" for _, (_, st, _) in g): - continue - het = [s for s, (al, _, _) in g if al[0] != al[1]] - if len(het) < 2: - continue - for s in het: - k = (s["chrom"], s["pos"]) - if k not in seen: - seen.add(k) - out.append((s["chrom"], s["pos"], s["rsid"], s["ref"], s["alt"])) - return out - - -def write_phase_vcf(path, sample, sites): - """The sites as a one-sample VCF of 0/1 genotypes, sorted, for whatshap phase.""" - contigs = sorted({c for c, *_ in sites}, key=lambda c: int(c[3:])) - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("##fileformat=VCFv4.2\n") - for c in contigs: - f.write(f"##contig=\n") - f.write('##FORMAT=\n') - f.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + sample + "\n") - for c, p, rs, ref, alt in sorted(sites, key=lambda s: (int(s[0][3:]), int(s[1]))): - f.write(f"{c}\t{p}\t{rs}\t{ref}\t{alt}\t.\tPASS\t.\tGT\t0/1\n") - os.replace(tmp, path) - - -def rule_line(rule, recs, phased): - """One output line of a rule. phased: read_phased's dict, or None when no phasing ran.""" - sites = rule["sites"] - g = [site_genotype(s, recs.get((s["chrom"], s["pos"]), [])) for s in sites] - rsids = [s["rsid"] for s in sites] - line = {"rule": rule["id"], "gene": rule["gene"], "trait": rule["trait"], "rsids": ",".join(rsids), - "chrom": sites[0]["chrom"], "pos": ",".join(s["pos"] for s in sites), - "genotypes": ",".join("/".join(al) if al else "." for al, _, _ in g), - "gq": ",".join(gq for _, _, gq in g), "ancestry": rule["ancestry"], "note": rule["note"], - "pmids": rule["pmids"]} - phase = "not_used" if rule["needs_phase"] == "no" else "not_needed" - statuses = [st for _, st, _ in g] - if any(st != "called" for st in statuses): - need = (f"both {rsids[0]} and {rsids[1]}" if len(rsids) == 2 - else "every one of " + ", ".join(rsids[:-1]) + " and " + rsids[-1]) - return dict(line, phase=".", result=".", - status="not_covered" if "not_covered" in statuses else "other_allele", - reading=f"Not read: needs a genotype at {need}.") - alleles = [al for al, _, _ in g] - combos = {c["diplotypes"]: c for c in rule["combinations"]} - if rule["needs_phase"] == "no": - key = "|".join(allowed_diplotypes(rule, alleles)) - else: - het = [i for i, al in enumerate(alleles) if al[0] != al[1]] - h1, h2 = [al[0] for al in alleles], [al[1] for al in alleles] - if len(het) >= 2: - blocks = {(phased or {}).get((sites[i]["chrom"], sites[i]["pos"]), (None, None, None))[2] for i in het} - if None in blocks or len(blocks) != 1: - why = ("no read or read pair covered them together" - if phased is not None else "no phasing ran (no BAM, or phasing failed)") - return dict(line, phase="unphased", result=".", status="unphased", - reading=f"Heterozygous at {len(het)} of the {len(sites)} sites, and {why}, so which " - "alleles share a copy is not known: not read.") - for i in het: - h1[i], h2[i] = phased[(sites[i]["chrom"], sites[i]["pos"])][:2] - phase = "read_backed" - by_alleles = {tuple(h["alleles"].split(",")): h["name"] for h in rule["haplotypes"]} - n1, n2 = by_alleles.get(tuple(h1)), by_alleles.get(tuple(h2)) - unknown = [",".join(h) for h, n in ((h1, n1), (h2, n2)) if n is None] - if unknown: - return dict(line, phase=phase, result=".", status="other_haplotype", - reading=f"A haplotype this rule does not describe ({' and '.join(unknown)} at " - f"{', '.join(rsids)}): not read.") - key = diplotype_key(rule, n1, n2) - if not key or key not in combos: - return dict(line, phase=phase, result=".", status="other_haplotype", - reading="A combination of haplotypes this rule does not describe: not read.") - c = combos[key] - return dict(line, phase=phase, result=c["label"], status="called", reading=c["reading"]) - - -def rule_lines(rules, recs, phased=None): - """One output line per rule, in the order of the rules table.""" - return [rule_line(r, recs, phased) for r in rules.values()] - - -def report(rows, recs, rules=None, phased=None, lines=None): - """The traits output rows: one per table row, with the line of each rule of - TRAITS_TABLE_RULES right after its last site row. rules defaults to - DEFAULT_RULES; lines, rule_lines' output, is computed when not given.""" - if rules is None: - rules = read_rules(DEFAULT_RULES) - if lines is None: - lines = rule_lines(rules, recs, phased) - after = {} - for rid in TRAITS_TABLE_RULES: - rule = rules.get(rid) - if rule: - members = {s["rsid"] for s in rule["sites"]} - last = max((i for i, r in enumerate(rows) if r["rsid"] in members), default=None) - if last is not None: - after[last] = next(ln for ln in lines if ln["rule"] == rid) - out = [] - for i, r in enumerate(rows): - gt, status, reading = call(r, recs.get((r["chrom"], r["pos"]), [])) - out.append({"rsid": r["rsid"], "gene": r["gene"], "chrom": r["chrom"], "pos": r["pos"], - "trait": r["trait"], "genotype": gt, "status": status, "reading": reading, - "ancestry": r["ancestry"], "note": r["note"], "pmids": r["pmids"]}) - if i in after: - ln = after[i] - out.append({"rsid": ln["rsids"].replace(",", "+"), "gene": ln["gene"], "chrom": ln["chrom"], - "pos": ln["pos"], "trait": ln["trait"], "genotype": ln["result"], "status": ln["status"], - "reading": ln["reading"], "ancestry": ln["ancestry"], "note": ln["note"], - "pmids": ln["pmids"]}) - return out - - -# --- Copy-aware markers -------------------------------------------------------------- - -# Each marker: the dbSNP record of one copy (rsID, chrom, POS, REF, ALT of -# dbSNP b157's GRCh38 VCF, read from NCBI's Variation API on 2026-11-07), every -# GRCh38 copy of the site (1-based), the plus-strand base pair read there, and -# the GRCh38 sequence of 10 bases on each side of every copy (UCSC hg38 and -# Ensembl GRCh38 agree, 2026-11-07): the same 21 bases at each copy, which is -# why the reads cannot be placed on one copy. tests/test_traits_report.py -# checks every copy against the context and the record against the marker, and -# that og cn reads the same copies; tests/e2e/genotype-rules-2-smn1-marker.sh -# compares the records with dbSNP. -# min_depth fewer reads over all copies together: too little depth -# min_alt reads with the marker's base for 'present' ... -# min_fraction ... and a share of the reads at least this -# max_absent this many reads with the marker's base or fewer: 'not seen' -# (one or two reads can be sequencing errors); between the two -# the marker is 'uncertain' MARKERS = { # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds @@ -560,108 +68,3 @@ def report(rows, recs, rules=None, phased=None, lines=None): }, } MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") - - -def marker_errors(name, m=None, dbsnp=None): - """Problems of one marker (empty when it is consistent): the dbSNP record - is one of its copies with its ref and alt, on the plus strand and not - swapped; every copy sits in the middle of the same context, whose middle - base is the ref.""" - m = MARKERS[name] if m is None else m - rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp - comp = {"A": "T", "C": "G", "G": "C", "T": "A"} - errs = [] - if chrom != m["chrom"] or pos not in m["copies"]: - errs.append(f"{name}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies") - if (m["ref"], m["alt"]) == (alt, ref): - errs.append(f"{name}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt})") - elif (m["ref"], m["alt"]) == (comp.get(ref), comp.get(alt)): - errs.append(f"{name}: the alleles are the minus-strand bases (marker {m['ref']}>{m['alt']}, " - f"dbSNP {ref}>{alt}): write the plus-strand bases") - elif (m["ref"], m["alt"]) != (ref, alt): - errs.append(f"{name}: marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt}") - ctx = m["context"] - if len(ctx) % 2 != 1 or ctx[len(ctx) // 2] != m["ref"]: - errs.append(f"{name}: the middle base of the context is not the ref {m['ref']}") - if len(set(m["copies"])) != len(m["copies"]): - errs.append(f"{name}: a copy is listed twice") - return errs - - -def write_tsv(path, out, columns=OUT_COLUMNS): - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("\t".join(columns) + "\n") - for o in out: - f.write("\t".join(str(o[k]) for k in columns) + "\n") - os.replace(tmp, path) - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--table", help="assets/traits.tsv") - ap.add_argument("--rules", help="assets/genotype_rules.tsv") - ap.add_argument("--genotypes", help="bcftools' CHROM POS REF ALT GENOTYPE GQ lines at the table and rule positions") - ap.add_argument("--phased", help="WhatsHap's VCF of the --phase-sites sites (absent: no phasing ran)") - ap.add_argument("--sample") - ap.add_argument("--out", help="the sample's traits table (TSV)") - ap.add_argument("--rules-out", help="the sample's genotype rules (TSV)") - ap.add_argument("--phase-sites", metavar="VCF", help="write the sites WhatsHap must phase and exit") - ap.add_argument("--check-table", metavar="TABLE", help="check the table's schema (and --rules') and exit") - a = ap.parse_args(argv) - try: - if a.check_table: - rows = read_table(a.check_table) - errs = check_rows(rows) - if a.rules: - errs += check_rules(read_rules(a.rules), rows) - for e in errs: - print(f"ERROR: {e}", file=sys.stderr) - if not errs: - print(f"{a.check_table}{' and ' + a.rules if a.rules else ''}: schema OK") - return 1 if errs else 0 - if a.phase_sites: - if not (a.rules and a.genotypes and a.sample): - ap.error("--phase-sites needs --rules, --genotypes and --sample") - rules = read_rules(a.rules) - errs = check_rules(rules) - if errs: - for e in errs: - print(f"ERROR: {a.rules}: {e}", file=sys.stderr) - return 1 - sites = phase_sites(rules, read_genotypes(a.genotypes)) - write_phase_vcf(a.phase_sites, a.sample, sites) - print(f"{a.sample}: {len(sites)} heterozygous rule sites to phase. Written: {a.phase_sites}") - return 0 - if not (a.table and a.rules and a.genotypes and a.sample and a.out and a.rules_out): - ap.error("give --table, --rules, --genotypes, --sample, --out and --rules-out " - "(or --phase-sites, or --check-table)") - rows = read_table(a.table) - rules = read_rules(a.rules) - errs = [f"{a.table}: {e}" for e in check_rows(rows)] + [f"{a.rules}: {e}" for e in check_rules(rules, rows)] - if errs: - for e in errs: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - phased = read_phased(a.phased) if a.phased else None - recs = read_genotypes(a.genotypes) - lines = rule_lines(rules, recs, phased) - out = report(rows, recs, rules, phased, lines) - except (TableError, OSError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - write_tsv(a.out, out) - write_tsv(a.rules_out, lines, RULE_OUT_COLUMNS) - counts = {} - for o in out: - counts[o["status"]] = counts.get(o["status"], 0) + 1 - print(f"{a.sample}: {len(rows)} table rows; " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) - + f" (APOE line included). Written: {a.out}") - print(f"{a.sample}: {len(lines)} genotype rules; " - + ", ".join(f"{ln['rule']} {ln['result'] if ln['status'] == 'called' else ln['status']}" for ln in lines) - + f". Written: {a.rules_out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/42-traits.md b/docs/42-traits.md index 7c6affb7..ad3aa25c 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -12,7 +12,7 @@ These are the results most people find easiest to read, and the pipeline had no - **bcftools** (`bcftools convert --gvcf2vcf`, `view`, `query`) reads the genotypes at the table's positions from the gVCF. - **WhatsHap** (`whatshap phase`) phases the heterozygous sites of a rule that needs phase from the BAM's reads. -- **`bin/traits_report.py`** (Python standard library) reads each genotype against the table, and each rule against the rule table. +- **`og traits table`** (Rust) reads each genotype against the table, and each rule against the rule table. ## Docker Image @@ -122,12 +122,12 @@ Kim et al. (2003) found five TAS2R38 haplotypes worldwide. The rule reads the tw ### Checks on the rule table -- `bin/traits_report.py --check-table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `tests/test_traits_report.py`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. +- `bin/og traits check --table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `tests/test_traits_report.py`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. - `scripts/ci/check-rule-alleles.sh` checks every site against dbSNP: the rsID at that GRCh38 position, ref as dbSNP's REF and alt among its ALT alleles. A pair written the wrong way round, or as the minus-strand bases (TAS2R38 lies on the minus strand, where the papers name the codon bases), fails with the reason. The dbSNP records it compares with sit in the script, taken from dbSNP's GRCh38 VCF of the build `CONSUMER_DBSNP_BUILD` pins in `versions.env` (b157); `--online` reads them again over HTTPS and fails when they differ, and the e2e case `genotype-rules-1-phase` runs it. A C/G or A/T site is its own complement, so there a strand mistake keeps valid alleles: the check prints a note for each such site (rs713598 is one), and its haplotype alleles rest on the codon: Pro49 has C at the first base of codon 49 on the coding strand (NM_176817.5 c.145C, where Ala49 has G), so the PAV base on GRCh38 is its complement, G. The other such sites: rs17580 (SERPINA1 S, minus strand: Glu264 GAA, Val264 GTA, so the S base on GRCh38 is the complement of T, A), rs1799945 (HFE H63D, plus strand: His63 CAT, Asp63 GAT, so D is G) and rs1333049 (no gene; the risk allele C is the plus-strand base of the WTCCC 2007 record). ### Adding a rule -Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `python3 tests/test_traits_report.py`. +Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `OG=bin/og python3 tests/test_traits_report.py`. ## APOE diff --git a/docs/rust.md b/docs/rust.md index c2ed48f6..99637ea7 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -111,3 +111,8 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to - **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. [Testing](testing.md) has the rest of CI. + +Step 42 uses `og traits table`, `og traits phase-sites` and `og traits check`. +The readers retain only catalog sites. The trait and genotype-rule output +columns and readings stay the same. `bin/traits_report.py` keeps report +metadata until the Python summary and report collectors move. diff --git a/modules/local/traits/main.nf b/modules/local/traits/main.nf index b7be4b1c..b6087ab7 100644 --- a/modules/local/traits/main.nf +++ b/modules/local/traits/main.nf @@ -13,7 +13,7 @@ an empty file when no rule has two such sites TRAITS _traits.tsv: the reading of each genotype from the table, and the APOE line; _genotype_rules.tsv: one - line per rule (bin/traits_report.py) + line per rule (og traits table) Runs for the samples with a gVCF only (DEEPVARIANT's, or one the samplesheet gives); main.nf logs the samples it skips. A sample without @@ -89,7 +89,7 @@ process TRAITS_PHASE { script: // The same commands as scripts/42-traits.sh """ - traits_report.py --rules ${rules} --genotypes ${genotypes} --sample ${meta.id} --phase-sites phase_sites.vcf + og traits phase-sites --rules ${rules} --genotypes ${genotypes} --sample ${meta.id} --out phase_sites.vcf if grep -qv '^#' phase_sites.vcf; then whatshap phase --reference ${reference} --ignore-read-groups -o ${meta.id}_traits_phased.vcf phase_sites.vcf ${bam} else @@ -129,7 +129,7 @@ process TRAITS { script: def phased_arg = phased ? "--phased ${phased}" : '' """ - traits_report.py --table ${table} --rules ${rules} --genotypes ${genotypes} ${phased_arg} --sample ${meta.id} \\ + og traits table --table ${table} --rules ${rules} --genotypes ${genotypes} ${phased_arg} --sample ${meta.id} \\ --out ${meta.id}_traits.tsv --rules-out ${meta.id}_genotype_rules.tsv printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml diff --git a/rust/README.md b/rust/README.md index 394e83e1..505ff4fa 100644 --- a/rust/README.md +++ b/rust/README.md @@ -34,3 +34,7 @@ cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # py ``` A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. + +`og traits table`, `og traits phase-sites` and `og traits check` read step 42's +curated catalogs and synthetic or bcftools genotype input. Their tests live +in `og-steps::traits` and the `traits` parity cases. diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index 145e6915..40c3db95 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -378,23 +378,21 @@ fn check_rules(rules: &[Rule], rows: Option<&[Site]>) -> Vec { } if let Some(t) = rows.and_then(|rows| rows.iter().rev().find(|t| t.row["rsid"] == s.row["rsid"])) - { - if ["chrom", "pos", "ref", "alt"] + && ["chrom", "pos", "ref", "alt"] .iter() .any(|k| t.row[*k] != s.row[*k]) - { - e.push(format!( - "{sw}: the traits table has {}:{} {}>{}, the rule {}:{} {}>{}", - t.row["chrom"], - t.row["pos"], - t.row["ref"], - t.row["alt"], - s.row["chrom"], - s.row["pos"], - s.row["ref"], - s.row["alt"] - )); - } + { + e.push(format!( + "{sw}: the traits table has {}:{} {}>{}, the rule {}:{} {}>{}", + t.row["chrom"], + t.row["pos"], + t.row["ref"], + t.row["alt"], + s.row["chrom"], + s.row["pos"], + s.row["ref"], + s.row["alt"] + )); } } if r.sites @@ -650,10 +648,10 @@ fn read_phased(path: &str, wanted: &HashSet) -> Result { }) }) .collect(); - if let Some(al) = idx { - if al.len() == 2 { - out.insert(key, (al[0].into(), al[1].into(), ps.to_string())); - } + if let Some(al) = idx + && al.len() == 2 + { + out.insert(key, (al[0].into(), al[1].into(), ps.to_string())); } } Ok(out) diff --git a/scripts/42-traits.sh b/scripts/42-traits.sh index 8994e027..ea302332 100755 --- a/scripts/42-traits.sh +++ b/scripts/42-traits.sh @@ -93,7 +93,7 @@ if [ -f "$BAM" ] && [ -f "${BAM}.bai" ]; then # shellcheck disable=SC2016 # $1 to $6 belong to the inner bash run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$WHATSHAP_IMAGE" \ bash -euo pipefail -c ' - python3 /pgp-bin/traits_report.py --rules /pgp-assets/genotype_rules.tsv --genotypes "$1" --sample "$2" --phase-sites "$3" + /pgp-bin/og traits phase-sites --rules /pgp-assets/genotype_rules.tsv --genotypes "$1" --sample "$2" --out "$3" if grep -qv "^#" "$3"; then whatshap phase --reference "$4" --ignore-read-groups -o "$5" "$3" "$6" else @@ -118,7 +118,7 @@ fi echo "[3/3] Reading each genotype and each rule against the tables..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/traits_report.py --table /pgp-assets/traits.tsv --rules /pgp-assets/genotype_rules.tsv \ + /pgp-bin/og traits table --table /pgp-assets/traits.tsv --rules /pgp-assets/genotype_rules.tsv \ --genotypes "$(cpath "$GENOTYPES")" ${PHASE_ARGS[@]+"${PHASE_ARGS[@]}"} --sample "$SAMPLE" \ --out "$(cpath "$OUT")" --rules-out "$(cpath "$RULES_OUT")" rm -f "$PHASED" diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dc..fc0a5239 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -73,22 +73,22 @@ clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/ clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-guard-1-mapq0 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh -cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh @@ -99,40 +99,40 @@ genome-lookup-1 bin/og scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv bin/traits_report.py modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/genotype_rules_indels.py bin/traits_report.py -genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh -genotype-rules-2-smn1-marker bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh +genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh +genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh -nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/traits_report.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/clip_check.py bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py bin/traits_report.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh -pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh platelet-antigens-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -142,21 +142,21 @@ prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py bin/traits_repo prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-1-sample-qc bin/collect_summary.py bin/og bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/clip_check.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh -reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/traits_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh reports-and-pgx-correctness-revel scripts/30-vcfanno.sh scripts/ci/fake-docker/lib.sh -run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cpsr_sample_id bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh @@ -169,12 +169,13 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index be90f627..5fd3717e 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -177,3 +177,4 @@ vntyper-1 shard 92 - y-private-report-1-private-and-tree shard 150 nextflow-hardening-2-nf y-str-1 shard 42 - y-tree-current-1-placement shard 21 nextflow-hardening-2-nf +rust-c2b-traits shard 2 - diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 49bf9e2a..ec57b581 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -1282,13 +1282,21 @@ def traits(d): unknown = sorted(r["rsid"] for r in rows if r["rsid"] not in TRAIT_ALT_COPIES | {"rs4410790": 0, "rs4481887": 0}) if unknown: raise SystemExit(f"assets/traits.tsv has rows TRAIT_ALT_COPIES does not give: {', '.join(unknown)}") - recs = {} - for r in rows: - if r["rsid"] in TRAIT_ALT_COPIES: - n = TRAIT_ALT_COPIES[r["rsid"]] - recs[(r["chrom"], r["pos"])] = [(r["ref"], [r["ref"]] * (2 - n) + [r["alt"]] * n, "60")] os.makedirs(os.path.join(d, "traits"), exist_ok=True) - traits_report.write_tsv(os.path.join(d, "traits", f"{SAMPLE}_traits.tsv"), traits_report.report(rows, recs)) + with tempfile.TemporaryDirectory() as work: + genotypes = os.path.join(work, "g.tsv") + with open(genotypes, "w") as f: + for r in rows: + if r["rsid"] in TRAIT_ALT_COPIES: + n = TRAIT_ALT_COPIES[r["rsid"]] + gt = "/".join([r["ref"]] * (2 - n) + [r["alt"]] * n) + f.write(f"{r['chrom']}\t{r['pos']}\t{r['ref']}\t{r['alt']}\t{gt}\t60\n") + subprocess.run([os.environ.get("OG", os.path.join(REPO, "bin", "og")), "traits", "table", + "--table", os.path.join(REPO, "assets", "traits.tsv"), + "--rules", os.path.join(REPO, "assets", "genotype_rules.tsv"), + "--genotypes", genotypes, "--sample", SAMPLE, + "--out", os.path.join(d, "traits", f"{SAMPLE}_traits.tsv"), + "--rules-out", os.path.join(work, "rules.tsv")], check=True) def mocha(d): diff --git a/tests/e2e/genotype-rules-2-g2.sh b/tests/e2e/genotype-rules-2-g2.sh index 7216eb96..5932e63c 100644 --- a/tests/e2e/genotype-rules-2-g2.sh +++ b/tests/e2e/genotype-rules-2-g2.sh @@ -40,16 +40,18 @@ mkdir -p "$D" for s in G2HET G2REF; do mkdir -p "${D}/${s}/vcf" "${D}/${s}/traits" python3 - "$REPO" "${D}/${s}" "$s" <<'PY' -import os, sys +import os, subprocess, sys repo, d, s = sys.argv[1:] sys.path.insert(0, os.path.join(repo, "bin")) -import traits_report as tr -rules = tr.read_rules(os.path.join(repo, "assets", "genotype_rules.tsv")) g = os.path.join(d, "g.tsv") with open(g, "w") as f: # G0/G1 at the two G1 sites f.write("chr22\t36265860\tA\tG\tA/G\t40\nchr22\t36265988\tT\tG\tT/G\t40\n") -tr.write_tsv(os.path.join(d, "traits", f"{s}_genotype_rules.tsv"), tr.rule_lines(rules, tr.read_genotypes(g)), - tr.RULE_OUT_COLUMNS) +subprocess.run([os.environ.get("OG", os.path.join(repo, "bin", "og")), "traits", "table", + "--table", os.path.join(repo, "assets", "traits.tsv"), + "--rules", os.path.join(repo, "assets", "genotype_rules.tsv"), + "--genotypes", g, "--sample", s, + "--out", os.path.join(d, "traits", f"{s}_traits.tsv"), + "--rules-out", os.path.join(d, "traits", f"{s}_genotype_rules.tsv")], check=True) os.remove(g) body = ["chr22\t36200000\t.\tA\t<*>\t0\t.\tEND=36265994\tGT:GQ\t0/0:50"] if s == "G2HET": @@ -72,7 +74,7 @@ done read_result() { python3 - "$REPO" "${D}/$1" "$1" <<'PY' -import os, sys +import os, subprocess, sys repo, d, s = sys.argv[1:] sys.path.insert(0, os.path.join(repo, "bin")) import collect_summary as cs diff --git a/tests/e2e/report-followups-1-sections.sh b/tests/e2e/report-followups-1-sections.sh index 4cfcf745..b89ec541 100644 --- a/tests/e2e/report-followups-1-sections.sh +++ b/tests/e2e/report-followups-1-sections.sh @@ -41,7 +41,6 @@ python3 - "$REPO" "$G" "$T" <<'PY' import json, os, shutil, subprocess, sys repo, g, s = sys.argv[1:4] sys.path.insert(0, os.path.join(repo, "bin")) -import traits_report # noqa: E402 def put(path, text): @@ -56,14 +55,15 @@ for kind in ("full", "bare"): # Step 42: TAS2R38 homozygous for PAV (G, G, C on the plus strand). geno = os.path.join(g, kind, "genotypes.tsv") put(geno, "chr7\t141972804\tT\tC\tC/C\t40\nchr7\t141972905\tG\t.\tG/G\t40\nchr7\t141973545\tC\tG\tG/G\t40\n") - rows = traits_report.read_table(os.path.join(repo, "assets", "traits.tsv")) - rules = traits_report.read_rules(os.path.join(repo, "assets", "genotype_rules.tsv")) - recs = traits_report.read_genotypes(geno) - lines = traits_report.rule_lines(rules, recs) os.makedirs(f"{d}/traits", exist_ok=True) - traits_report.write_tsv(f"{d}/traits/{s}_traits.tsv", traits_report.report(rows, recs, rules, None, lines)) - if kind == "full": - traits_report.write_tsv(f"{d}/traits/{s}_genotype_rules.tsv", lines, traits_report.RULE_OUT_COLUMNS) + subprocess.run([os.environ.get("OG", os.path.join(repo, "bin", "og")), "traits", "table", + "--table", os.path.join(repo, "assets", "traits.tsv"), + "--rules", os.path.join(repo, "assets", "genotype_rules.tsv"), + "--genotypes", geno, "--sample", s, + "--out", f"{d}/traits/{s}_traits.tsv", + "--rules-out", f"{d}/traits/{s}_genotype_rules.tsv"], check=True) + if kind == "bare": + os.remove(f"{d}/traits/{s}_genotype_rules.tsv") # Step 05: two class 4 deletions; the rule table puts the second over half inside an IG locus. head = ("AnnotSV_ID\tSV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tID\tINFO\tAnnotation_mode\tGene_name\t" "B_loss_source\tB_loss_AFmax\tACMG_class\tAnnotSV_ranking_score\n") diff --git a/tests/e2e/rust-c2b-traits.sh b/tests/e2e/rust-c2b-traits.sh new file mode 100755 index 00000000..71613e72 --- /dev/null +++ b/tests/e2e/rust-c2b-traits.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Traits, APOE ordering, phase sites and refusal controls on synthetic input. +. "$(dirname "$0")/lib.sh" +OG=${OG:-${REPO}/bin/og} +TABLE="${REPO}/assets/traits.tsv" +RULES="${REPO}/assets/genotype_rules.tsv" +G="${CASE_TMP}/g.tsv" +printf 'chr19\t44908684\tT\tC\tT/C\t40\nchr19\t44908822\tC\tT\tC/T\t40\nchr7\t141973545\tC\tG\tC/G\t30\nchr7\t141972905\tG\tA\tG/A\t31\nchr7\t141972804\tT\tC\tT/C\t32\n' > "$G" +"$OG" traits table --table "$TABLE" --rules "$RULES" --genotypes "$G" --sample S \ + --out "${CASE_TMP}/traits.tsv" --rules-out "${CASE_TMP}/rules.tsv" +check_eq 'traits table exits 0' "$?" 0 +check_eq 'APOE double heterozygote is e2/e4' "$(awk -F'\t' '$1 == "APOE" {print $10}' "${CASE_TMP}/rules.tsv")" e2/e4 +check_eq 'the taster rule needs phase' "$(awk -F'\t' '$1 == "TAS2R38" {print $11}' "${CASE_TMP}/rules.tsv")" unphased +check_eq 'APOE line occurs exactly once' "$(grep -c '^rs429358+rs7412' "${CASE_TMP}/traits.tsv")" 1 +check 'absent LCT has no inferred genotype' awk -F'\t' '$1 == "rs4988235" {ok = ($6 == "." && $7 == "not_covered")} END {exit !ok}' "${CASE_TMP}/traits.tsv" +"$OG" traits phase-sites --rules "$RULES" --genotypes "$G" --sample S --out "${CASE_TMP}/phase.vcf" +check_eq 'phase-sites exits 0' "$?" 0 +check_eq 'only the three taster sites need phase' "$(grep -cv '^#' "${CASE_TMP}/phase.vcf")" 3 +check_eq 'phase sites are in genome order' "$(grep -v '^#' "${CASE_TMP}/phase.vcf" | cut -f2 | paste -sd, -)" 141972804,141972905,141973545 +sed 's/44908684\tT\t/44908684\tG\t/' "$G" > "${CASE_TMP}/bad.tsv" +"$OG" traits table --table "$TABLE" --rules "$RULES" --genotypes "${CASE_TMP}/bad.tsv" --sample S \ + --out "${CASE_TMP}/bad-out.tsv" --rules-out "${CASE_TMP}/bad-rules.tsv" > "${CASE_TMP}/bad.log" 2>&1 +check_eq 'control: wrong reference is refused' "$?" 1 +check 'control: refusal names the site and GRCh38' has 'rs429358.*GRCh38' "$(cat "${CASE_TMP}/bad.log")" +check 'control: no table is written' test ! -e "${CASE_TMP}/bad-out.tsv" +"$OG" traits check --table "$TABLE" --rules "$RULES" +check_eq 'committed catalogs pass' "$?" 0 +finish diff --git a/tests/parity/README.md b/tests/parity/README.md index d83ff208..3d32b997 100644 --- a/tests/parity/README.md +++ b/tests/parity/README.md @@ -42,3 +42,8 @@ A `known.tsv` row whose difference is gone fails. A golden that the oracle also 3. Run `scripts/ci/oracle-diff.sh`. Turn each difference into a fix, a golden or a `known.tsv` row. 4. Switch the callers to `og`, delete the Python, and add the tool to `OG_SHADOW` in `.github/workflows/e2e.yml`. 5. Link a red run in the pull request: the cases with one planted change in the port must fail. + +The `traits` cases cover all APOE genotype pairs, missing and other alleles, +duplicate records, TAS2R38 phase blocks, phase-site VCFs and catalog refusals. +The oracle adapter maps the three Rust verbs to the old flags. The Python +report keeps only the metadata and table reader listed in `remnants.tsv`. diff --git a/tests/parity/gen/traits.sh b/tests/parity/gen/traits.sh index 10d0fa26..a5777138 100644 --- a/tests/parity/gen/traits.sh +++ b/tests/parity/gen/traits.sh @@ -33,10 +33,10 @@ printf 'chr19\t44908684\tT\tC\tT/C\t30\n' > missing.tsv printf 'chr19\t44908684\tTT\tT\tTT/T\t10\nchr19\t44908684\tT\tC\tC/T\t40\n' > duplicate.tsv printf 'chr19\t44908684\tG\tC\tG/C\t30\n' > wrong-ref.tsv printf 'chr19\t44908684\tT\tC\t./C\t30\n' > half.tsv -printf 'chr7\t141973545\tC\tG\tC/G\t30\nchr7\t141972905\tG\tC\tG/C\t31\nchr7\t141972804\tT\tC\tT/C\t32\n' > triple.tsv +printf 'chr7\t141973545\tC\tG\tC/G\t30\nchr7\t141972905\tG\tA\tG/A\t31\nchr7\t141972804\tT\tC\tT/C\t32\n' > triple.tsv head -1 triple.tsv > single.tsv : > empty.vcf -printf 'chr7\t141973545\t.\tC\tG\t.\tPASS\t.\tGT:PS\t1|0:7\nchr7\t141972905\t.\tG\tC\t.\tPASS\t.\tGT:PS\t0|1:7\nchr7\t141972804\t.\tT\tC\t.\tPASS\t.\tGT:PS\t1|0:7\n' > phased.vcf +printf 'chr7\t141973545\t.\tC\tG\t.\tPASS\t.\tGT:PS\t1|0:7\nchr7\t141972905\t.\tG\tA\t.\tPASS\t.\tGT:PS\t0|1:7\nchr7\t141972804\t.\tT\tC\t.\tPASS\t.\tGT:PS\t1|0:7\n' > phased.vcf sed '1s/:7$/:8/' phased.vcf > split.vcf awk -F'\t' -v OFS='\t' '$1 == "combination" && $2 == "APOE" && $3 == "e3/e3" {next} {print}' rules.tsv > bad-rules.tsv awk -F'\t' -v OFS='\t' '$3 == "rs671" {$6=$5} {print}' traits.tsv > bad-table.tsv diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 52679749..7d9e929b 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -63,4 +63,4 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd -traits 669a8196 file:bin/traits_report.py bash ./oracle.sh traits +traits 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:bin/traits_report.py bash ./oracle.sh traits diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 479efa8b..084de768 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -9,3 +9,4 @@ bin/pgx_outside_calls.py HLA_GENES,NO_CALL,PROBABLE,NOT_CALLABLE,LOW_GQ,is_call, bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/collect_summary.py,bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 bin/cnvpytor_to_vcf.py DEPTH_MIN_SIZE,delly_depth_contradicts bin/couple_report.py R4 bin/annotsv_postfilter.py CONSENSUS_BP,KEEP,ADDED,open_text,chrom_key,sv_type,info_value,sample_format,sample_columns,read_consensus,consensus_support bin/collect_summary.py,tests/demo/make_demo_sample.py R3 +bin/traits_report.py COLUMNS,OUT_COLUMNS,RULE_OUT_COLUMNS,TRAITS_TABLE_RULES,TableError,read_table,MARKERS,MARKER_STATUSES bin/collect_summary.py,tests/demo/make_demo_sample.py,tests/test_traits_report.py,tests/test_traits_table.py,tests/test_collect_summary.py,tests/test_lpa_kiv2.py,tests/test_render_report.py R3 diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 62d7d1b9..ee2580d7 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1206,19 +1206,18 @@ def report_followups_sections(work, schema): # (G, G, C on the plus strand; rs1726866's G is the reference base). t = os.path.join(work, "rules", "S") rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) - rules = traits_report.read_rules(os.path.join(REPO, "assets", "genotype_rules.tsv")) g = os.path.join(work, "rules_g.tsv") put(g, "chr7\t141972804\tT\tC\tC/C\t40\nchr7\t141972905\tG\t.\tG/G\t40\nchr7\t141973545\tC\tG\tG/G\t40\n" "chr19\t44908684\tT\t.\tT/T\t40\nchr19\t44908822\tC\t.\tC/C\t40\n") - recs = traits_report.read_genotypes(g) - lines = traits_report.rule_lines(rules, recs) + from test_traits_report import traits_tables os.makedirs(f"{t}/traits", exist_ok=True) - traits_report.write_tsv(f"{t}/traits/S_traits.tsv", traits_report.report(rows, recs, rules, None, lines)) + saved_rules = os.path.join(work, "rules_saved.tsv") + traits_tables(g, f"{t}/traits/S_traits.tsv", saved_rules) summ, txt, html_ = render(t) check("rules: without the rules table the section has no rule lines and the reports none", summ["sections"]["traits"]["data"].get("rules") is None and "Genotype rules (several sites" not in txt + html_, summ["sections"]["traits"]["data"].get("rules")) - traits_report.write_tsv(f"{t}/traits/S_genotype_rules.tsv", lines, traits_report.RULE_OUT_COLUMNS) + shutil.copyfile(saved_rules, f"{t}/traits/S_genotype_rules.tsv") summ, txt, html_ = render(t) html_ = htmlmod.unescape(html_) rl = summ["sections"]["traits"]["data"].get("rules") or [] @@ -1226,7 +1225,9 @@ def report_followups_sections(work, schema): [(r["rule"], r["result"], r["status"], r["phase"]) for r in rl if r["rule"] in ("TAS2R38", "APOE")] == [("TAS2R38", "PAV/PAV", "called", "not_needed")], rl) - trait = rules["TAS2R38"]["trait"] + import csv + with open(saved_rules) as f: + trait = next(r["trait"] for r in csv.DictReader(f, delimiter="\t") if r["rule"] == "TAS2R38") check("rules: the text report prints the rule line with its result and sites", f"{trait} (TAS2R38 rs713598, rs1726866, rs10246939): PAV/PAV (no phase needed). " in txt and "traits/S_genotype_rules.tsv" in txt, txt) @@ -2861,7 +2862,8 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): f"{ear['chrom']}\t{ear['pos']}\t{ear['ref']}\t.\t{ear['ref']}/{ear['ref']}\n" "chr19\t44908684\tT\t.\tT/T\nchr19\t44908822\tC\t.\tC/C\n") os.makedirs(f"{t}/traits", exist_ok=True) - traits_report.write_tsv(f"{t}/traits/S_traits.tsv", traits_report.report(rows, traits_report.read_genotypes(g))) + from test_traits_report import traits_tables + traits_tables(g, f"{t}/traits/S_traits.tsv", os.path.join(work, "traits_rules.tsv")) summ_t, txt_t, html_t = render(t) td = summ_t["sections"]["traits"] check("traits: section ok, counts without the APOE rows", diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index b00a0030..03a4bf34 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -86,7 +86,30 @@ CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") MODELS = os.path.join(REPO, "assets", "parascopy_models_v1.7.txt") sys.path.insert(0, os.path.join(REPO, "bin")) -import copy_number as cn # noqa: E402 +# The old engine is a test oracle pinned to the traits port's parent. It is +# materialised only for these historical differential assertions. No pipeline +# step imports or ships a second engine. +import importlib.util # noqa: E402 + + +def catalog_oracle(): + ref = "669a8196cf4c377200f9b4af9c235ef85fb774a1" + if subprocess.run(["git", "-C", REPO, "cat-file", "-e", ref + "^{commit}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + subprocess.run(["git", "-C", REPO, "fetch", "--depth=1", "origin", ref], check=True) + source = subprocess.check_output(["git", "-C", REPO, "show", ref + ":bin/copy_number.py"]) + with tempfile.TemporaryDirectory(prefix="cn-oracle-") as work: + path = os.path.join(work, "copy_number.py") + with open(path, "wb") as f: + f.write(source) + spec = importlib.util.spec_from_file_location("cn_catalog_oracle", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") + return module + + +cn = catalog_oracle() FAILS = 0 # Frozen from bin/hba_copy_number.py and bin/lpa_kiv2.py as they were before the diff --git a/tests/test_genotype_rules_indels.py b/tests/test_genotype_rules_indels.py index 777aaf4b..35fccfd7 100644 --- a/tests/test_genotype_rules_indels.py +++ b/tests/test_genotype_rules_indels.py @@ -155,8 +155,7 @@ def bgzip_with_tbi(path, lines): def report_checks(): import collect_summary as cs import render_report as rr - import traits_report as tr - rules = tr.read_rules(os.path.join(REPO, "assets", "genotype_rules.tsv")) + from test_traits_report import traits_tables head = "##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" g1 = [("chr22", "36265860", "A", "G", "A/G", "40"), ("chr22", "36265988", "T", "G", "T/G", "40")] work = tempfile.mkdtemp() @@ -169,7 +168,7 @@ def folder(name, gvcf_lines, kind): g = os.path.join(work, name + "_g.tsv") with open(g, "w") as f: f.writelines("\t".join(x) + "\n" for x in g1) - tr.write_tsv(p, tr.rule_lines(rules, tr.read_genotypes(g)), tr.RULE_OUT_COLUMNS) + traits_tables(g, os.path.join(d, "traits", "S_traits.tsv"), p) v = os.path.join(d, "vcf", "S.g.vcf.gz" if kind == "gvcf" else "S.vcf.gz") with gzip.open(v, "wt") as f: f.write(head + "".join("\t".join(r) + "\n" for r in gvcf_lines)) diff --git a/tests/test_hba_copy_number.py b/tests/test_hba_copy_number.py index a31afb5c..4dc92a38 100755 --- a/tests/test_hba_copy_number.py +++ b/tests/test_hba_copy_number.py @@ -35,7 +35,7 @@ CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import copy_number as cn # noqa: E402 +from test_copy_number import cn # noqa: E402 pinned differential oracle REGIONS = cn.regions(cn.load_catalog(CATALOG)["HBA"]) # Step 35's HBA table, in its order (what bin/hba_copy_number.py wrote). diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index c1aeccf1..03ab6f15 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -72,7 +72,7 @@ shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) OG = os.environ["OG"] = os.path.join(_og_dir, "og") sys.path.insert(0, os.path.join(REPO, "bin")) -import copy_number as cn # noqa: E402 +from test_copy_number import cn # noqa: E402 pinned differential oracle FAILS = 0 # The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. diff --git a/tests/test_render_report.py b/tests/test_render_report.py index c5406109..a8377a4d 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -958,7 +958,10 @@ def mosaic_table(rows): d = f"{work}/traits/S" rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) put(f"{d}/traits/S_traits.tsv", "") - traits_report.write_tsv(f"{d}/traits/S_traits.tsv", traits_report.report(rows, {})) + from test_traits_report import traits_tables + g = os.path.join(work, "traits_empty.tsv") + put(g, "") + traits_tables(g, f"{d}/traits/S_traits.tsv", os.path.join(work, "traits_rules.tsv")) summ = collect_summary.collect("S", d) txt, html = render_report.text_report(summ), render_report.html_report(summ) check("traits: both reports say a reading is not a diagnosis", diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index cd9b7547..a10e13f4 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -1,92 +1,26 @@ #!/usr/bin/env python3 -"""bin/traits_report.py (step 42) on synthetic bcftools output, on assets/traits.tsv and on -assets/genotype_rules.tsv; and scripts/ci/check-rule-alleles.sh on the rules. +"""Step 42 CLI goldens, rule-allele controls and og cn marker outputs. -Checks: - 1. a reference block expanded at a position (ALT '.', genotype G/G) reads as - ref/ref with the table's hom_ref text; ref/alt and alt/alt read het and - hom_alt, phased (C|T) as unphased, the reference allele printed first; - 2. a planted alternate allele changes the reading: the same position read - as ref/ref and then as ref/alt gives two different readings, each the - table's own; - 3. a position with no line is not_covered (never read as the reference); an - allele the table does not describe and a deletion starting at the - position are other_allele, not read; a half call that reaches the script - (the bcftools genotype filter of step 42 drops it first, so in a run its - position is not_covered) is other_allele too; - 4. a gVCF REF that is not the table's reference base stops the run (exit 1, - the row named), so a gVCF on another build is not read; - 5. the APOE line, now read from the APOE rule: for every pair of rs429358 and - rs7412 genotypes, one site not covered and an allele the rule does not - describe, the line is byte for byte the line the hand code of main - (6056452) wrote, by its sha256; it comes right after the second APOE row, - once; control: one changed character in a reading changes the hash; - report() without rules reads the repository's rules table; - 6. the rules table passes the schema check, and planted problems are each - caught: a missing and an extra combination line, a haplotype allele that - is neither ref nor alt, diplotypes out of haplotype order, a site that - disagrees with its traits table row, no APOE rule, a line of an unknown - kind; - 7. the rules on synthetic gVCF records, for each rule: reference blocks only - (homozygous reference, with the block's GQ; TAS2R38's reference haplotype - is AAI, which the rule does not read); APOE's double heterozygote - (e2/e4, maybe e1/e3) without phase; TAS2R38 homozygous PAV/PAV and a - single heterozygous site, which need no phase; the triple heterozygote - unphased (WhatsHap ran and no read pair spans the sites, or no phasing - ran), phased in one block (PAV/AVI, read-backed), phased in two blocks - (unphased), phased into a haplotype the rule does not describe; a site - not covered; - 8. --phase-sites writes only the heterozygous sites of a rule that needs - phase and has two or more of them, as a sorted VCF; WhatsHap's phased VCF - is read back (GT with |, PS), its unphased records left out; - 9. the command line writes the traits table (one row per table row plus the - APOE line, the columns in order) and the rules table (one line per rule), - and exits 1 on a traits table or a rules table that fails the schema check; - 10. scripts/ci/check-rule-alleles.sh passes on the rules and fails, naming the - site and the mistake, on planted copies: ref and alt swapped, the - minus-strand bases, an rsID the dbSNP table lacks, a wrong position; and - on a rules file with no site, and on a dbSNP table of another build than - versions.env pins; - 11. the copy-aware markers (MARKERS: LPA KIV-2 4925G>A, SMN1 g.27134T>G): - each pinned dbSNP record is one of the marker's copies with its ref and - alt, and a planted swap, a minus-strand pair and a position off the - copies are each caught; the context's middle base is the ref. With OG - naming the og binary (skipped without it, a failure with OG_REQUIRED - set too): og cn marker-regions prints one region per copy of MARKERS; - og cn marker-read SMN1_27134 on mpileup lines made up - for each case at the SMN1 and SMN2 copies: a quarter of the reads with G - is 'present' (copies = fraction x agCN), T only (the reference) 'not - seen', 2 reads with G 'uncertain', under 10 reads 'too little depth', an - unreadable pileup 'not assessed' (exit 0); the SMN1 psCN and the agCN - come from step 35's table (homologous region included, quality under 20 - not resolved); every message, for every status and every psCN, says the - marker does not prove 2+0 (control: a message without the sentence fails - the same test). - -Run: python3 tests/test_traits_report.py +Internal validation, phase and bounded-reader assertions live in +og-steps::traits::tests. The parity harness checks the old Python at oracle.ref. """ +import csv import hashlib import os import shutil import subprocess import sys import tempfile - REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) import traits_report as tr # noqa: E402 - TABLE = os.path.join(REPO, "assets", "traits.tsv") RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") -SCRIPT = os.path.join(REPO, "bin", "traits_report.py") CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") OG = os.environ.get("OG", "") -# The sentence every SMN1 marker message ends with. +SMN_HEADER = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n" SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." FAILS = 0 - -# The sha256 of the APOE line (its OUT_COLUMNS joined by tabs) the hand code of -# main 6056452 wrote, per rs429358 and rs7412 genotype ("." = no line). APOE_GOLDEN = { ("T/T", "C/C"): "e3d7512b649f1d23f5acbcaa08da19bd6a009d5b39ac57cb6856591c50e7cfda", ("T/T", "C/T"): "62e910d8ec7ac45ab15e9c9ed85c93b128ac41703d1149a332082dc0d7b5f389", @@ -100,10 +34,6 @@ ("T/C", "."): "34624db16a92981c0882edb3d8ce8d291162aa78b7c5208eba2360141e6ccbd8", ("T/G", "C/C"): "1d0aee0b8d241561ebd7477d6f4a8be274a48157a297abecd444c6c6a0a6abac", } -# TAS2R38's three sites in the rule's order (codons 49, 262, 296). -T1, T2, T3 = ("chr7", "141973545"), ("chr7", "141972905"), ("chr7", "141972804") - - def check(desc, ok, detail=""): global FAILS print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") @@ -116,46 +46,6 @@ def write(path, text): f.write(text) -def recs(*lines): - """bcftools' CHROM POS REF ALT GENOTYPE [GQ] lines -> read_genotypes' records.""" - d = tempfile.mkdtemp() - try: - p = os.path.join(d, "g.tsv") - write(p, "".join("\t".join(map(str, l)) + "\n" for l in lines)) - return tr.read_genotypes(p) - finally: - shutil.rmtree(d) - - -def phased_vcf(*records): - """WhatsHap-style VCF records (chrom, pos, ref, alt, GT[:PS], GT as allele indexes) -> read_phased's dict.""" - d = tempfile.mkdtemp() - try: - p = os.path.join(d, "p.vcf") - body = "".join(f"{c}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{'GT:PS' if ':' in g else 'GT'}\t{g}\n" - for c, pos, ref, alt, g in records) - write(p, "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + body) - return tr.read_phased(p) - finally: - shutil.rmtree(d) - - -def apoe_line(out): - return next(o for o in out if o["rsid"] == "rs429358+rs7412") - - -def line_hash(o): - return hashlib.sha256("\t".join(str(o[c]) for c in tr.OUT_COLUMNS).encode()).hexdigest() - - -def run(*args): - return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) - - -def og_cn(*args): - return subprocess.run([OG, "cn", *args], capture_output=True, text=True) - - def dbsnp_checks(rules): """10. scripts/ci/check-rule-alleles.sh on the rules and on planted copies. Runs first, so a rules table that breaks the other checks still shows what dbSNP says.""" @@ -206,9 +96,6 @@ def planted(desc, new, want): shutil.rmtree(work) -# 11. The multi-SNP rules (APOL1, SERPINA1, HFE, GC, CAD9p21, MAPT): per rule, the -# genotypes (ALT allele per site, "." for a reference block) of one positive case and -# its label, and of one negative case, a copy of the positive with one allele flipped. MULTI_SNP = { "APOL1": [(("G/G", "G/G"), "G1/G1"), (("A/G", "T/G"), "G0/G1")], "SERPINA1": [(("T/T", "T/T"), "PiZZ"), (("T/A", "C/T"), "PiSZ"), (("A/A", "C/C"), "PiSS")], @@ -222,64 +109,6 @@ def planted(desc, new, want): "CAD9p21": "no risk copy", "MAPT": "H1/H1"} -def multi_snp_checks(rules): - """11. Each multi-SNP rule on synthetic records: every positive genotype gives its label, - reference blocks alone give the reference label, and the control: the first positive - case with one allele flipped at one site gives another result. Then the lines reach the - HTML report: collect_summary reads them from the rules table and render_report writes - one row per rule.""" - import collect_summary as cs - import render_report as rr - - def line(rid, gts): - sites = rules[rid]["sites"] - rows = [] - for s, gt in zip(sites, gts): - a = [x for x in gt.split("/") if x != s["ref"]] - rows.append((s["chrom"], s["pos"], s["ref"], a[0] if a else ".", gt, "40")) - return next(x for x in tr.rule_lines(rules, recs(*rows)) if x["rule"] == rid) - - check("11. every multi-SNP rule is in the rules table", set(MULTI_SNP) <= set(rules), sorted(rules)) - for rid, cases in MULTI_SNP.items(): - if rid not in rules: - continue - sites = rules[rid]["sites"] - for gts, want in cases: - ln = line(rid, gts) - check(f"{rid} {','.join(gts)}: {want}", (ln["status"], ln["result"]) == ("called", want), ln) - ref = tuple(f"{s['ref']}/{s['ref']}" for s in sites) - ln = line(rid, ref) - check(f"{rid}, reference blocks only: {MULTI_SNP_REF[rid]}", - (ln["status"], ln["result"]) == ("called", MULTI_SNP_REF[rid]), ln) - gts, want = cases[0] - a, b = gts[0].split("/") - s0 = sites[0] - flip = s0["ref"] if b == s0["alt"] else s0["alt"] - flipped = (f"{a}/{flip}",) + tuple(gts[1:]) - ln = line(rid, flipped) - check(f"control: {rid} with one allele flipped at {s0['rsid']} ({','.join(flipped)}) is not {want}", - ln["result"] != want, ln) - apol1 = line("APOL1", ("A/G", "T/G"))["reading"] - check("APOL1 G0/G1 says G2 is not read and the other copy could carry it", - "G2 is not read" in apol1 and "G1/G2" in apol1, apol1) - work = tempfile.mkdtemp() - try: - os.makedirs(os.path.join(work, "traits")) - lines = [line(rid, cases[0][0]) for rid, cases in MULTI_SNP.items() if rid in rules] - tr.write_tsv(os.path.join(work, "traits", "S_genotype_rules.tsv"), lines, tr.RULE_OUT_COLUMNS) - read = cs.genotype_rules(work, "S") or [] - page = "\n".join(rr.rules_html({"rules": read}, "S")) - for rid, cases in MULTI_SNP.items(): - if rid in rules: - check(f"HTML report: the {rid} rule row with its result {cases[0][1]}", - rr.E(rules[rid]["trait"]) in page and rr.E(cases[0][1]) in page, page[:300]) - finally: - shutil.rmtree(work) - - -SMN_HEADER = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n" - - def says_not_proof(message): """The SMN1 marker's message must say the marker does not prove 2+0.""" return "does not prove 2+0" in message @@ -287,22 +116,6 @@ def says_not_proof(message): def marker_checks(): """11. The copy-aware markers.""" - for name, m in tr.MARKERS.items(): - check(f"marker {name}: the pinned dbSNP record agrees with the marker", tr.marker_errors(name) == [], - tr.marker_errors(name)) - rs, chrom, pos, ref, alt = m["dbsnp"] - comp = {"A": "T", "C": "G", "G": "C", "T": "A"} - e = tr.marker_errors(name, dict(m, ref=alt, alt=ref)) - check(f"marker {name} control: ref and alt swapped is caught", any("swapped" in x for x in e), e) - e = tr.marker_errors(name, dict(m, ref=comp[ref], alt=comp[alt], - context=m["context"][:10] + comp[ref] + m["context"][11:])) - check(f"marker {name} control: the minus-strand bases are caught", any("minus-strand" in x for x in e), e) - e = tr.marker_errors(name, dbsnp=(rs, chrom, pos + 1, ref, alt)) - check(f"marker {name} control: a dbSNP position off the copies is caught", - any("not at one of the marker's copies" in x for x in e), e) - e = tr.marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) - check(f"marker {name} control: a context whose middle base is not the ref is caught", - any("middle base" in x for x in e), e) check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) if not OG: print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") @@ -395,235 +208,73 @@ def table(p, smn_path): shutil.rmtree(work) -def main(): - rules = tr.read_rules(RULES) - dbsnp_checks(rules) - rows = tr.read_table(TABLE) - by_id = {r["rsid"]: r for r in rows} - lct = by_id["rs4988235"] # chr2:135851076 G>A - c, p, ref, alt = lct["chrom"], lct["pos"], lct["ref"], lct["alt"] - at = lambda *ls: recs(*ls).get((c, p), []) # noqa: E731 - # 1. genotypes and their readings - g = tr.call(lct, at((c, p, ref, ".", f"{ref}/{ref}"))) - check("expanded reference block: ref/ref, hom_ref reading", g == (f"{ref}/{ref}", "called", lct["hom_ref"]), g) - g = tr.call(lct, at((c, p, ref, alt, f"{alt}/{ref}"))) - check("ref/alt: het reading, reference allele first", g == (f"{ref}/{alt}", "called", lct["het"]), g) - g = tr.call(lct, at((c, p, ref, alt, f"{ref}|{alt}"))) - check("phased ref|alt reads as ref/alt", g == (f"{ref}/{alt}", "called", lct["het"]), g) - g = tr.call(lct, at((c, p, ref, alt, f"{alt}/{alt}"))) - check("alt/alt: hom_alt reading", g == (f"{alt}/{alt}", "called", lct["hom_alt"]), g) +def traits_tables(genotypes, out, rules_out, sample="S"): + """Use og in the Rust job. Unit shards without og use the pinned oracle.""" + args = ["--table", TABLE, "--rules", RULES, "--genotypes", genotypes, + "--sample", sample, "--out", out, "--rules-out", rules_out] + if OG: + subprocess.run([OG, "traits", "table", *args], check=True, capture_output=True, text=True) + else: + with tempfile.TemporaryDirectory(prefix="traits-oracle-") as work: + ref = "669a8196cf4c377200f9b4af9c235ef85fb774a1" + if subprocess.run(["git", "-C", REPO, "cat-file", "-e", ref + "^{commit}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + subprocess.run(["git", "-C", REPO, "fetch", "--depth=1", "origin", ref], check=True) + source = subprocess.check_output(["git", "-C", REPO, "show", ref + ":bin/traits_report.py"]) + path = os.path.join(work, "traits_report.py") + with open(path, "wb") as f: + f.write(source) + subprocess.run([sys.executable, path, *args], check=True, capture_output=True, text=True) - # 2. a planted alternate allele changes the reading - before = tr.call(lct, at((c, p, ref, ".", f"{ref}/{ref}")))[2] - after = tr.call(lct, at((c, p, ref, alt, f"{ref}/{alt}")))[2] - check("planted alt: the reading changes from hom_ref to het", before == lct["hom_ref"] and after == lct["het"] - and before != after, (before, after)) - # 3. not covered and other alleles - check("no line: not_covered", tr.call(lct, [])[1] == "not_covered") - other = next(b for b in "ACGT" if b not in (ref, alt)) - g = tr.call(lct, at((c, p, ref, other, f"{ref}/{other}"))) - check("an allele the table does not describe: other_allele, not read", g[1] == "other_allele" - and g[2] == tr.OTHER_ALLELE, g) - g = tr.call(lct, at((c, p, ref, alt, f"./{alt}"))) - check("a half call: other_allele", g[1] == "other_allele", g) - g = tr.call(lct, at((c, p, ref + "T", ref, f"{ref}T/{ref}"))) - check("a deletion starting at the position: other_allele", g[1] == "other_allele", g) - g = tr.call(lct, at((c, p, ref + "T", ref, f"{ref}T/{ref}"), (c, p, ref, alt, f"{ref}/{alt}"))) - check("a SNV record beside an indel at the position is the one read", g == (f"{ref}/{alt}", "called", lct["het"]), g) - - # 4. a REF that is not the GRCh38 base stops - try: - tr.call(lct, at((c, p, other, alt, f"{other}/{alt}"))) - check("a gVCF REF that differs from the table stops the run", False, "no error") - except tr.TableError as e: - check("a gVCF REF that differs from the table stops the run, naming the row", "rs4988235" in str(e), e) - - # 5. APOE, byte for byte the hand code's line - r4, r2 = by_id["rs429358"], by_id["rs7412"] - for (g4, g2), want in APOE_GOLDEN.items(): - lines = [(r4["chrom"], r4["pos"], "T", g4.split("/")[1], g4)] - if g2 != ".": - lines.append((r2["chrom"], r2["pos"], "C", "T", g2)) - out = tr.report(rows, recs(*lines), rules) - line = apoe_line(out) - check(f"APOE line byte for byte as before: rs429358 {g4}, rs7412 {g2} -> {line['genotype']} {line['status']}", - line_hash(line) == want, line) - out = tr.report(rows, recs((r4["chrom"], r4["pos"], "T", "C", "T/C"), (r2["chrom"], r2["pos"], "C", "T", "C/T")), - rules) - check("APOE e2/e4: says it could also be the rare e1/e3", "e1/e3" in apoe_line(out)["reading"], apoe_line(out)) - ids = [o["rsid"] for o in out] - check("APOE: the line comes right after the second APOE row", - ids.index("rs429358+rs7412") == max(ids.index("rs429358"), ids.index("rs7412")) + 1, ids) - check("APOE: one line only", ids.count("rs429358+rs7412") == 1, ids) - changed = dict(rules["APOE"], combinations=[dict(x, reading=x["reading"].replace("e3/e3.", "e3/e3!")) - for x in rules["APOE"]["combinations"]]) - out = tr.report(rows, recs((r4["chrom"], r4["pos"], "T", "C", "T/C"), (r2["chrom"], r2["pos"], "C", "T", "C/C")), - dict(rules, APOE=changed)) - check("control: one changed character in an APOE reading changes the hash", - line_hash(apoe_line(out)) != APOE_GOLDEN[("T/C", "C/C")], apoe_line(out)) - e4 = apoe_line(tr.report(rows, recs((r4["chrom"], r4["pos"], "T", "C", "C/C"), - (r2["chrom"], r2["pos"], "C", "T", "C/C")), rules)) - check("APOE e4/e4: a risk factor, not a diagnosis", "not a diagnosis" in e4["reading"], e4) - check("report() without rules reads assets/genotype_rules.tsv (other tests call it so)", - line_hash(apoe_line(tr.report(rows, {}))) == line_hash(apoe_line(tr.report(rows, {}, rules))), rows[:0]) - - # 6. the rules table's schema - errs = tr.check_rules(rules, rows) - check(f"assets/genotype_rules.tsv passes the schema check ({len(rules)} rules)", not errs, errs) - check("the APOE and TAS2R38 rules are there", {"APOE", "TAS2R38"} <= set(rules), list(rules)) - - def broken(desc, mutate, want, table=rows): - rs = {k: dict(v, sites=[dict(s) for s in v["sites"]], haplotypes=[dict(h) for h in v["haplotypes"]], - combinations=[dict(x) for x in v["combinations"]]) for k, v in rules.items()} - mutate(rs) - e = tr.check_rules(rs, table) - check(f"control: {desc} is caught", any(want in x for x in e), e) - - broken("a missing combination line", lambda rs: rs["TAS2R38"]["combinations"].pop(), "no combination line for AVI/AVI") - broken("an extra combination line", lambda rs: rs["TAS2R38"]["combinations"].append( - dict(rs["TAS2R38"]["combinations"][0], diplotypes="PAV/PAV|AVI/AVI")), "can never be read") - broken("a haplotype allele that is neither ref nor alt", - lambda rs: rs["TAS2R38"]["haplotypes"][0].update(alleles="C,C,C"), "neither its ref") - broken("diplotypes out of haplotype order", - lambda rs: rs["TAS2R38"]["combinations"][1].update(diplotypes="AVI/PAV"), "write PAV/AVI") - broken("a rule site that disagrees with its traits table row", - lambda rs: rs["TAS2R38"]["sites"][1].update(ref="A", alt="G"), "the traits table has chr7:141972905 G>A") - broken("no APOE rule", lambda rs: rs.pop("APOE"), "rule APOE is missing") - broken("a needs_phase that is not yes or no", lambda rs: rs["APOE"].update(needs_phase="maybe"), "needs_phase") - work = tempfile.mkdtemp() - try: - bad = os.path.join(work, "r.tsv") - write(bad, open(RULES).read() + "allele\tAPOE\tx\n") - try: - tr.read_rules(bad) - check("control: a line of an unknown kind is caught", False, "read without error") - except tr.TableError as e: - check("control: a line of an unknown kind is caught", "kind 'allele'" in str(e), e) - finally: - shutil.rmtree(work) - - # 7. the rules on synthetic gVCF records - def rule(rid, *lines, phased=None): - return next(x for x in tr.rule_lines(rules, recs(*lines), phased) if x["rule"] == rid) - - a4, a2 = (r4["chrom"], r4["pos"]), (r2["chrom"], r2["pos"]) - l = rule("APOE", (*a4, "T", ".", "T/T", "37"), (*a2, "C", ".", "C/C", "41")) - check("APOE, reference blocks only: e3/e3 called, with each block's GQ", - (l["result"], l["status"], l["genotypes"], l["gq"], l["phase"]) == ("e3/e3", "called", "T/T,C/C", "37,41", - "not_used"), l) - l = rule("APOE", (*a4, "T", "C", "T/C", "30"), (*a2, "C", "T", "C/T", "30")) - check("APOE, double heterozygote without phase: e2/e4 (the rule needs no phase)", - (l["result"], l["status"]) == ("e2/e4", "called"), l) - ref_blocks = [(*T1, "C", ".", "C/C", "50"), (*T2, "G", ".", "G/G", "50"), (*T3, "T", ".", "T/T", "48")] - l = rule("TAS2R38", *ref_blocks) - check("TAS2R38, reference blocks only: homozygous reference with the GQ, AAI (C,G,T) not read", - (l["status"], l["genotypes"], l["gq"], l["result"]) == ("other_haplotype", "C/C,G/G,T/T", "50,50,48", ".") - and "C,G,T" in l["reading"], l) - pav = [(*T1, "C", "G", "G/G", "40"), (*T2, "G", ".", "G/G", "40"), (*T3, "T", "C", "C/C", "40")] - l = rule("TAS2R38", *pav) - check("TAS2R38 PAV/PAV: called, no phase needed", - (l["result"], l["status"], l["phase"]) == ("PAV/PAV", "called", "not_needed"), l) - one_het = [(*T1, "C", "G", "C/G", "40"), (*T2, "G", "A", "A/A", "40"), (*T3, "T", ".", "T/T", "40")] - l = rule("TAS2R38", *one_het) - check("TAS2R38, one heterozygous site: AVI with a G,A,T copy, which the rule does not read (no phase needed)", - l["status"] == "other_haplotype" and l["phase"] == "not_needed" and "G,A,T" in l["reading"], l) - tri = [(*T1, "C", "G", "C/G", "35"), (*T2, "G", "A", "G/A", "35"), (*T3, "T", "C", "T/C", "35")] - l = rule("TAS2R38", *tri, phased={}) - check("TAS2R38 triple heterozygote, no read pair spans the sites: unphased, not read", - (l["status"], l["phase"], l["result"]) == ("unphased", "unphased", ".") - and "no read or read pair" in l["reading"], l) - l = rule("TAS2R38", *tri) - check("TAS2R38 triple heterozygote, no phasing ran: unphased, says why", - l["status"] == "unphased" and "no phasing ran" in l["reading"], l) - one_block = phased_vcf(("chr7", T3[1], "T", "C", "1|0:141972804"), ("chr7", T2[1], "G", "A", "0|1:141972804"), - ("chr7", T1[1], "C", "G", "1|0:141972804")) - l = rule("TAS2R38", *tri, phased=one_block) - check("TAS2R38 triple heterozygote phased in one block: PAV/AVI, read-backed", - (l["result"], l["status"], l["phase"]) == ("PAV/AVI", "called", "read_backed"), l) - two_blocks = phased_vcf(("chr7", T3[1], "T", "C", "1|0:141972804"), ("chr7", T2[1], "G", "A", "0|1:141972804"), - ("chr7", T1[1], "C", "G", "1|0:141973545")) - l = rule("TAS2R38", *tri, phased=two_blocks) - check("TAS2R38 triple heterozygote phased in two blocks: unphased", l["status"] == "unphased", l) - odd = phased_vcf(("chr7", T3[1], "T", "C", "0|1:141972804"), ("chr7", T2[1], "G", "A", "0|1:141972804"), - ("chr7", T1[1], "C", "G", "1|0:141972804")) - l = rule("TAS2R38", *tri, phased=odd) - check("TAS2R38 phased into G,G,T and C,A,C: a haplotype the rule does not describe", - l["status"] == "other_haplotype" and "G,G,T and C,A,C" in l["reading"], l) - l = rule("TAS2R38", *tri[:2]) - check("TAS2R38, a site not covered: not_covered, names the three sites", - l["status"] == "not_covered" and "every one of rs713598, rs1726866 and rs10246939" in l["reading"], l) - - # 8. the sites to phase, and WhatsHap's VCF read back - s = tr.phase_sites(rules, recs(*tri, (*a4, "T", "C", "T/C"), (*a2, "C", "T", "C/T"))) - check("--phase-sites: the three TAS2R38 sites, not APOE's (needs_phase no)", - sorted(x[2] for x in s) == ["rs10246939", "rs1726866", "rs713598"], s) - check("--phase-sites: nothing for one heterozygous site", tr.phase_sites(rules, recs(*one_het)) == []) - check("--phase-sites: nothing when a site is not covered", tr.phase_sites(rules, recs(*tri[:2])) == []) - check("WhatsHap's VCF: unphased records are left out", - phased_vcf(("chr7", "1", "C", "G", "0/1"), ("chr7", "2", "G", "A", "0|1:2")) == {("chr7", "2"): ("G", "A", "2")}) - work = tempfile.mkdtemp() - try: - g = os.path.join(work, "g.tsv") - write(g, "".join("\t".join(x) + "\n" for x in tri)) - vcf = os.path.join(work, "in.vcf") - res = run("--rules", RULES, "--genotypes", g, "--sample", "S", "--phase-sites", vcf) - body = [x.split("\t") for x in open(vcf).read().splitlines() if not x.startswith("#")] if res.returncode == 0 else [] - check("--phase-sites CLI: a sorted one-sample VCF of the three 0/1 sites", - [(x[1], x[2], x[9]) for x in body] == [(T3[1], "rs10246939", "0/1"), (T2[1], "rs1726866", "0/1"), - (T1[1], "rs713598", "0/1")] - and "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS" in open(vcf).read(), (res.stderr, body)) - - # 9. the command line - write(g, f"{c}\t{p}\t{ref}\t{alt}\t{ref}/{alt}\t30\n" + "".join("\t".join(x) + "\n" for x in tri)) - out, rout = os.path.join(work, "S_traits.tsv"), os.path.join(work, "S_genotype_rules.tsv") - ph = os.path.join(work, "phased.vcf") - write(ph, "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - f"chr7\t{T3[1]}\t.\tT\tC\t.\tPASS\t.\tGT:PS\t1|0:9\nchr7\t{T2[1]}\t.\tG\tA\t.\tPASS\t.\tGT:PS\t0|1:9\n" - f"chr7\t{T1[1]}\t.\tC\tG\t.\tPASS\t.\tGT:PS\t1|0:9\n") - res = run("--table", TABLE, "--rules", RULES, "--genotypes", g, "--phased", ph, "--sample", "S", - "--out", out, "--rules-out", rout) - check("CLI: exits 0", res.returncode == 0, res.stderr) - lines = open(out).read().splitlines() if res.returncode == 0 else [""] - check("CLI: the header is the output columns", lines[0].split("\t") == tr.OUT_COLUMNS, lines[0]) - check("CLI: one row per table row plus the APOE line", len(lines) - 1 == len(rows) + 1, len(lines)) - row = next((l.split("\t") for l in lines[1:] if l.startswith("rs4988235\t")), []) - check("CLI: the planted het reads het", row[5:8] == [f"{ref}/{alt}", "called", lct["het"]], row) - check("CLI: a row with no line is not_covered", - next((l for l in lines if l.startswith("rs671\t")), "").split("\t")[6:7] == ["not_covered"]) - rl = open(rout).read().splitlines() if res.returncode == 0 else [""] - check("CLI: the rules table has its header and one line per rule", - rl[0].split("\t") == tr.RULE_OUT_COLUMNS and len(rl) - 1 == len(rules), rl[:1]) - t = next((x.split("\t") for x in rl if x.startswith("TAS2R38\t")), []) - check("CLI: TAS2R38 read from the phased VCF: PAV/AVI, read-backed, with the GQ", - t[6:11] == ["C/G,G/A,T/C", "35,35,35", "read_backed", "PAV/AVI", "called"], t) - bad = os.path.join(work, "bad.tsv") - write(bad, open(TABLE).read().replace("\trs671\tALDH2\tG\tA\t", "\trs671\tALDH2\tG\tG\t")) - res = run("--table", bad, "--rules", RULES, "--genotypes", g, "--sample", "S", - "--out", os.path.join(work, "x.tsv"), "--rules-out", os.path.join(work, "y.tsv")) - check("CLI: a traits table that fails the schema check exits 1, naming the row", - res.returncode == 1 and "rs671" in res.stderr and not os.path.exists(os.path.join(work, "x.tsv")), - (res.returncode, res.stderr)) - badr = os.path.join(work, "badr.tsv") - write(badr, open(RULES).read().replace("haplotype\tTAS2R38\tAVI\tC,A,T", "haplotype\tTAS2R38\tAVI\tC,A,A")) - res = run("--table", TABLE, "--rules", badr, "--genotypes", g, "--sample", "S", - "--out", os.path.join(work, "x.tsv"), "--rules-out", os.path.join(work, "y.tsv")) - check("CLI: a rules table that fails the schema check exits 1, naming the haplotype", - res.returncode == 1 and "AVI" in res.stderr and not os.path.exists(os.path.join(work, "y.tsv")), - (res.returncode, res.stderr)) - res = run("--check-table", TABLE, "--rules", RULES) - check("CLI --check-table with --rules: schema OK", res.returncode == 0 and "schema OK" in res.stdout, - (res.returncode, res.stderr)) +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) - finally: - shutil.rmtree(work) - multi_snp_checks(rules) +def main(): + rules = {} + with open(RULES) as f: + for cells in csv.reader((line for line in f if not line.startswith("#")), delimiter="\t"): + if cells and cells[0] == "site": + rules.setdefault(cells[1], {"sites": []})["sites"].append(cells[2:]) + dbsnp_checks(rules) + if not OG: + check("OG_REQUIRED needs the og binary", not os.environ.get("OG_REQUIRED")) + print("skip: OG is not set: traits CLI and cn marker checks did not run") + return int(bool(FAILS)) + with tempfile.TemporaryDirectory() as work: + def table(records): + g = os.path.join(work, "g.tsv") + write(g, "".join("\t".join(rec) + "\n" for rec in records)) + out, rout = os.path.join(work, "traits.tsv"), os.path.join(work, "rules.tsv") + res = subprocess.run([OG, "traits", "table", "--table", TABLE, "--rules", RULES, + "--genotypes", g, "--sample", "S", "--out", out, + "--rules-out", rout], capture_output=True, text=True) + check("traits table exits 0", res.returncode == 0, res.stderr) + if res.returncode: + return [], [] + with open(out) as f: + traits = list(csv.DictReader(f, delimiter="\t")) + with open(rout) as f: + lines = list(csv.DictReader(f, delimiter="\t")) + return traits, lines + for (a, b), want in APOE_GOLDEN.items(): + records = [("chr19", pos, ref, alt, gt) for pos, ref, alt, gt in + [("44908684", "T", "C", a), ("44908822", "C", "T", b)] if gt != "."] + traits, lines = table(records) + apoe = next(r for r in traits if "+" in r["rsid"]) + got = hashlib.sha256("\t".join(apoe[k] for k in tr.OUT_COLUMNS).encode()).hexdigest() + check(f"APOE golden {a}, {b}", got == want, got) + for rid, cases in MULTI_SNP.items(): + for gts, label in cases: + records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(rules[rid]["sites"], gts)] + _, lines = table(records) + got = next(r for r in lines if r["rule"] == rid) + check(f"{rid} {gts}: {label}", got["status"] == "called" and got["result"] == label, got) marker_checks() - - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 + print("RESULT:", "ALL PASS" if not FAILS else f"{FAILS} FAILED") + return int(bool(FAILS)) if __name__ == "__main__": diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 0996c911..84cf6020 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -145,39 +145,19 @@ def online_errors(rows): return errs +def schema(path): + return subprocess.run([os.environ.get("OG", os.path.join(REPO, "bin", "og")), + "traits", "check", "--table", path], capture_output=True, text=True) + + def schema_controls(): good = tr.read_table(TABLE) - errs = tr.check_rows(good) - check(f"assets/traits.tsv passes the schema check ({len(good)} rows)", not errs, errs) check("the table has at least 30 rows", len(good) >= 30, len(good)) - - def broken(desc, mutate, want): - rows = [dict(r) for r in good] - mutate(rows) - e = tr.check_rows(rows) - check(f"control: {desc} is caught", any(want in x for x in e), e) - - broken("ref = alt", lambda rs: rs[0].update(alt=rs[0]["ref"]), "same base") - broken("a ref that is not a base", lambda rs: rs[0].update(ref="N"), "is not one of A, C, G, T") - broken("a duplicate rsID", lambda rs: rs[1].update(rsid=rs[0]["rsid"]), "appears twice") - broken("a PMID that is not a number", lambda rs: rs[0].update(pmids="PMC123"), "PubMed ids") - broken("a contig outside chr1 to chr22", lambda rs: rs[0].update(chrom="chrX"), "is not chr1 to chr22") - broken("a missing APOE row", lambda rs: rs.remove(next(r for r in rs if r["rsid"] == "rs7412")), "rs7412 is missing") - broken("an empty reading", lambda rs: rs[0].update(het=" "), "het is empty") - broken("SNPedia text", lambda rs: rs[0].update(note="From SNPedia."), "SNPedia") work = tempfile.mkdtemp() try: - bad = os.path.join(work, "t.tsv") - with open(TABLE) as f: - text = f.read() - with open(bad, "w") as f: - f.write(text.replace("\trs671\t", "\trs671\textra\t", 1)) - try: - tr.read_table(bad) - check("control: a row with an extra column is caught", False, "read without error") - except tr.TableError as e: - check("control: a row with an extra column is caught", "columns" in str(e), e) - + # Schema controls are Rust unit tests and traits parity cases. + res = schema(TABLE) + check("assets/traits.tsv passes og traits check", res.returncode == 0, res.stderr) # FASTA check on a synthetic FASTA: chr2 holds the lactase row's base. lct = next(r for r in good if r["rsid"] == "rs4988235") pos = int(lct["pos"]) @@ -212,6 +192,9 @@ def main(argv=None): ap.add_argument("--online", action="store_true", help="look the PMIDs up in PubMed and the rsIDs in dbSNP") a = ap.parse_args(argv) if a.table == TABLE and not a.fasta and not a.online: + if not os.environ.get("OG") and not os.environ.get("OG_REQUIRED"): + print("skip: OG is not set: traits schema controls run in the Rust job") + return 0 schema_controls() else: try: @@ -219,8 +202,8 @@ def main(argv=None): except tr.TableError as e: check("the table can be read", False, e) rows = [] - errs = tr.check_rows(rows) - check(f"{a.table} passes the schema check", not errs, errs) + res = schema(a.table) + check(f"{a.table} passes the schema check", res.returncode == 0, res.stderr) if a.fasta and rows: e, n = fasta_errors(a.fasta, rows) print(f" ref checked against {a.fasta}: {n} rows (the others are on contigs it lacks)") From 339b64733676314d8b5d75bc98553d9e9aea5254 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:11:07 +0100 Subject: [PATCH 3/8] test(traits): plant diplotype, reference and phase controls Bead: pgp-bs9u.25 --- rust/crates/og-steps/src/traits.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index 40c3db95..1c4d6169 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -339,6 +339,7 @@ fn allowed(rule: &Rule, gt: &[Vec]) -> Vec { } } } + out.reverse(); // planted control: diplotype key order out } fn check_rules(rules: &[Rule], rows: Option<&[Site]>) -> Vec { @@ -562,7 +563,7 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { }; let site = sites[0]; for s in sites { - if c[2].chars().next() != s.row["ref"].chars().next() { + if c[2].is_empty() { // planted control: REF check removed return Err(format!( "{} at {}:{}: the gVCF has REF {}, the table {} (the GRCh38 base). Is the gVCF on GRCh38?", s.row["rsid"], c[0], c[1], c[2], s.row["ref"] @@ -861,6 +862,7 @@ fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result } } } + sites.clear(); // planted control: missing phase sites let mut out = AtomicFile::create(path).map_err(|e| e.to_string())?; let write = |out: &mut AtomicFile| -> std::io::Result<()> { writeln!(out, "##fileformat=VCFv4.2")?; From ab5088dc7e0ec80665fba0b074a4d756c7b37b65 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:14:48 +0100 Subject: [PATCH 4/8] test(traits): restore evaluation after planted controls Reverts 339b64733676314d8b5d75bc98553d9e9aea5254. Bead: pgp-bs9u.25 --- rust/crates/og-steps/src/traits.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index 1c4d6169..40c3db95 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -339,7 +339,6 @@ fn allowed(rule: &Rule, gt: &[Vec]) -> Vec { } } } - out.reverse(); // planted control: diplotype key order out } fn check_rules(rules: &[Rule], rows: Option<&[Site]>) -> Vec { @@ -563,7 +562,7 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { }; let site = sites[0]; for s in sites { - if c[2].is_empty() { // planted control: REF check removed + if c[2].chars().next() != s.row["ref"].chars().next() { return Err(format!( "{} at {}:{}: the gVCF has REF {}, the table {} (the GRCh38 base). Is the gVCF on GRCh38?", s.row["rsid"], c[0], c[1], c[2], s.row["ref"] @@ -862,7 +861,6 @@ fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result } } } - sites.clear(); // planted control: missing phase sites let mut out = AtomicFile::create(path).map_err(|e| e.to_string())?; let write = |out: &mut AtomicFile| -> std::io::Result<()> { writeln!(out, "##fileformat=VCFv4.2")?; From 0e0ba11bd114bb1c417feb8a099ab8d2ff9682d3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:33:10 +0100 Subject: [PATCH 5/8] fix(traits): keep shared sites independent and stage both outputs Exercise the three named fault controls in Guard before testing the restored source. Retain the local red log as reviewable evidence. Bead: pgp-bs9u.25 --- .github/workflows/guard.yml | 23 +++ CHANGELOG.md | 2 +- rust/crates/og-steps/src/traits.rs | 155 +++++++++++++++++--- tests/e2e/rust-c2b-traits.sh | 1 + tests/parity/cases.tsv | 1 + tests/parity/evidence/rust-c2b-controls.txt | 69 +++++++++ tests/parity/gen/traits.sh | 16 +- tests/parity/remnants.tsv | 2 +- tests/test_copy_number.py | 3 +- tests/test_render_report.py | 2 - tests/test_traits_report.py | 3 +- tests/test_traits_table.py | 35 +---- 12 files changed, 252 insertions(+), 60 deletions(-) create mode 100644 tests/parity/evidence/rust-c2b-controls.txt diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 7ee58a0d..66800c3a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -279,6 +279,29 @@ jobs: grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." + - name: Traits controls (diplotype order, REF check and missing phase rows) + env: + CARGO_BUILD_JOBS: 3 + run: | + src=crates/og-steps/src/traits.rs + saved="${RUNNER_TEMP}/traits.rs" + log="${RUNNER_TEMP}/traits-controls.log" + cp -f "$src" "$saved" + trap 'cp -f "$saved" "$src"' EXIT + perl -0pi -e 's/(fn allowed\b.*?)(\n out\n)/$1\n out.reverse();$2/s; s/if c\[2\]\.chars\(\)\.next\(\) != s\.row\["ref"\]\.chars\(\)\.next\(\)/if c[2].is_empty()/; s/(fn phase_vcf\b.*?)( let mut out = AtomicFile::create\(path\))/$1 sites.clear();\n$2/s' "$src" + cmp -s "$saved" "$src" && { echo "ERROR: no traits control was planted"; exit 1; } + if cargo test --locked -p og-steps traits::tests --no-fail-fast > "$log" 2>&1; then + cat "$log" + echo "ERROR: traits accepted the planted faults" + exit 1 + fi + cat "$log" + for test in apoe_grid_keeps_haplotype_order_and_missing_calls stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory phase_requires_one_block_and_only_multiple_hets_are_exported; do + grep -E "traits::tests::${test} .*FAILED" "$log" || { echo "ERROR: $test did not catch its fault"; exit 1; } + done + cp -f "$saved" "$src" + trap - EXIT + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/CHANGELOG.md b/CHANGELOG.md index 8400d72d..b6245e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error instead of a traceback. +- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published, so an unwritable second output leaves the previous traits table intact. - The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index 40c3db95..d537e672 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -112,7 +112,10 @@ fn row(keys: &[&str], values: &[String]) -> Row { } fn records(path: &str) -> Result)>> { // Parse even the header as a CSV record. Comments do not enter the parser. - let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut lines = Lines::new( + og_io::open(path).map_err(|e| og_filters::os_error(&e, path))?, + Utf8::Strict, + ); let mut text = String::new(); let mut number = 0; let mut out = Vec::new(); @@ -534,13 +537,48 @@ fn check_rules(rules: &[Rule], rows: Option<&[Site]>) -> Vec { e } +// A locus may be reused by rules with different alternate alleles. The +// selected record is shared, but each site's allele vocabulary is its own. +fn genotype(site: &Site, records: &Genotypes) -> SiteGt { + let mut gt = records.get(&site.key()).cloned().unwrap_or_default(); + if gt.status == Status::NotCovered { + return gt; + } + gt.alleles.sort_by_key(|a| { + ( + if a == &site.row["ref"] { + 0 + } else if a == &site.row["alt"] { + 1 + } else { + 2 + }, + a.clone(), + ) + }); + gt.status = if gt.alleles.len() == 2 + && gt + .alleles + .iter() + .all(|a| a == &site.row["ref"] || a == &site.row["alt"]) + { + Status::Called + } else { + Status::OtherAllele + }; + gt +} + fn read_genotypes(path: &str, sites: &[&Site]) -> Result { let mut wanted: HashMap> = HashMap::new(); for s in sites { wanted.entry(s.key()).or_default().push(s); } let mut result = HashMap::new(); - let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut lines = Lines::new( + og_io::open(path).map_err(|e| og_filters::os_error(&e, path))?, + Utf8::Strict, + ); let mut line = String::new(); let mut n = 0; while lines.read_line(&mut line).map_err(|e| e.to_string())? { @@ -613,7 +651,10 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { Ok(result) } fn read_phased(path: &str, wanted: &HashSet) -> Result { - let mut lines = Lines::new(og_io::open(path).map_err(|e| e.to_string())?, Utf8::Strict); + let mut lines = Lines::new( + og_io::open(path).map_err(|e| og_filters::os_error(&e, path))?, + Utf8::Strict, + ); let mut text = String::new(); let mut out = HashMap::new(); while lines.read_line(&mut text).map_err(|e| e.to_string())? { @@ -657,11 +698,7 @@ fn read_phased(path: &str, wanted: &HashSet) -> Result { Ok(out) } fn rule_line(r: &Rule, gt: &Genotypes, phased: Option<&Phased>) -> Row { - let g: Vec<_> = r - .sites - .iter() - .map(|s| gt.get(&s.key()).cloned().unwrap_or_default()) - .collect(); + let g: Vec<_> = r.sites.iter().map(|s| genotype(s, gt)).collect(); let rs: Vec<_> = r.sites.iter().map(|s| s.row["rsid"].as_str()).collect(); let mut out = r.meta.clone(); for (k, v) in [ @@ -818,20 +855,22 @@ fn rule_line(r: &Rule, gt: &Genotypes, phased: Option<&Phased>) -> Row { ), } } -fn write_tsv(path: &str, columns: &[&str], rows: &[Row]) -> Result<()> { - let mut file = AtomicFile::create(path).map_err(|e| e.to_string())?; +fn stage_tsv(path: &str, columns: &[&str], rows: &[Row]) -> Result { + let mut file = AtomicFile::create(path).map_err(|e| format!("cannot write {path}: {e}"))?; { let mut writer = Writer::new(&mut file, '\t'); writer .write_row(columns.iter().copied()) - .map_err(|e| e.to_string())?; + .map_err(|e| format!("cannot write {path}: {e}"))?; for row in rows { writer .write_row(columns.iter().map(|k| row[*k].as_str())) - .map_err(|e| e.to_string())?; + .map_err(|e| format!("cannot write {path}: {e}"))?; } } - file.commit().map_err(|e| e.to_string()) + file.flush() + .map_err(|e| format!("cannot write {path}: {e}"))?; + Ok(file) } fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result { let mut sites = BTreeMap::new(); @@ -839,7 +878,10 @@ fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result let calls: Option> = r .sites .iter() - .map(|s| gt.get(&s.key()).filter(|g| g.status == Status::Called)) + .map(|s| { + let g = genotype(s, gt); + (g.status == Status::Called).then_some(g) + }) .collect(); if let Some(calls) = calls { let het: Vec<_> = r @@ -861,7 +903,7 @@ fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result } } } - let mut out = AtomicFile::create(path).map_err(|e| e.to_string())?; + let mut out = AtomicFile::create(path).map_err(|e| format!("cannot write {path}: {e}"))?; let write = |out: &mut AtomicFile| -> std::io::Result<()> { writeln!(out, "##fileformat=VCFv4.2")?; for chrom in sites.keys().map(|(c, _)| c).collect::>() { @@ -884,8 +926,9 @@ fn phase_vcf(path: &str, sample: &str, rules: &[Rule], gt: &Genotypes) -> Result } Ok(()) }; - write(&mut out).map_err(|e| e.to_string())?; - out.commit().map_err(|e| e.to_string())?; + write(&mut out).map_err(|e| format!("cannot write {path}: {e}"))?; + out.commit() + .map_err(|e| format!("cannot write {path}: {e}"))?; Ok(sites.len()) } fn run(verb: &str, opts: &HashMap) -> Result { @@ -968,7 +1011,7 @@ fn run(verb: &str, opts: &HashMap) -> Result { let mut out = vec![]; let mut counts = BTreeMap::::new(); for (i, s) in rows.iter().enumerate() { - let g = gt.get(&s.key()).cloned().unwrap_or_default(); + let g = genotype(s, >); let mut r = s.row.clone(); r.insert( "genotype".into(), @@ -1003,8 +1046,16 @@ fn run(verb: &str, opts: &HashMap) -> Result { } } // Compute both tables before opening either output. Each rename is atomic. - write_tsv(required("out")?, OUT, &out)?; - write_tsv(required("rules-out")?, RULE_OUT, &lines)?; + let out_path = required("out")?; + let rules_path = required("rules-out")?; + let traits_file = stage_tsv(out_path, OUT, &out)?; + let rules_file = stage_tsv(rules_path, RULE_OUT, &lines)?; + traits_file + .commit() + .map_err(|e| format!("cannot write {out_path}: {e}"))?; + rules_file + .commit() + .map_err(|e| format!("cannot write {rules_path}: {e}"))?; for r in &out { *counts.entry(r["status"].clone()).or_default() += 1; } @@ -1284,6 +1335,29 @@ mod tests { let path = scratch.put("many", &"chr99\t1\tN\t.\tN/N\n".repeat(10000)); assert!(read_genotypes(&path, &[s]).unwrap().is_empty()); } + #[test] + fn shared_position_keeps_each_rules_allele_vocabulary() { + let (rows, _) = assets(); + let first = &rows[0]; + let mut second = first.clone(); + let alt = ["A", "C", "G", "T"] + .into_iter() + .find(|a| *a != first.row["ref"] && *a != first.row["alt"]) + .unwrap(); + second.row.insert("alt".into(), alt.into()); + let scratch = Scratch::new(); + let path = scratch.put( + "g", + &format!( + "{}\t{}\t{}\t{}\t{}/{}\n", + first.row["chrom"], first.row["pos"], first.row["ref"], alt, first.row["ref"], alt + ), + ); + let gt = read_genotypes(&path, &[first, &second]).unwrap(); + assert_eq!(genotype(first, >).status, Status::OtherAllele); + assert_eq!(genotype(&second, >).status, Status::Called); + } + #[test] fn phase_requires_one_block_and_only_multiple_hets_are_exported() { let (_, rules) = assets(); @@ -1322,6 +1396,45 @@ mod tests { let gt = genotypes(r, &[0, 0, 0]); assert_eq!(rule_line(r, >, None)["phase"], "not_needed"); } + #[test] + fn unwritable_rules_output_keeps_existing_traits_table() { + let scratch = Scratch::new(); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); + let target = scratch.put("out", "previous complete table\n"); + let gt = scratch.put("g", ""); + let opts = [ + ( + "table", + root.join("traits.tsv").to_string_lossy().into_owned(), + ), + ( + "rules", + root.join("genotype_rules.tsv") + .to_string_lossy() + .into_owned(), + ), + ("genotypes", gt), + ("sample", "S".into()), + ("out", target.clone()), + ( + "rules-out", + scratch + .0 + .join("missing/rules.tsv") + .to_string_lossy() + .into_owned(), + ), + ] + .into_iter() + .map(|(k, v)| (k.into(), v)) + .collect(); + assert!(run("table", &opts).unwrap_err().contains("cannot write")); + assert_eq!( + std::fs::read_to_string(target).unwrap(), + "previous complete table\n" + ); + } + #[test] fn csv_header_unicode_rules_and_atomic_write_errors() { let scratch = Scratch::new(); @@ -1335,7 +1448,7 @@ mod tests { assert!(read_table(&path).unwrap().is_empty()); let path = scratch.put("r", "rule\té\tG\tT\tno\tyes\tN\t1\n"); assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); - assert!(write_tsv(scratch.0.join("absent/x").to_str().unwrap(), OUT, &[]).is_err()); + assert!(stage_tsv(scratch.0.join("absent/x").to_str().unwrap(), OUT, &[]).is_err()); assert_eq!(cli(&["table".into()]), 2); assert_eq!( cli(&[ diff --git a/tests/e2e/rust-c2b-traits.sh b/tests/e2e/rust-c2b-traits.sh index 71613e72..e5e2e385 100755 --- a/tests/e2e/rust-c2b-traits.sh +++ b/tests/e2e/rust-c2b-traits.sh @@ -12,6 +12,7 @@ check_eq 'traits table exits 0' "$?" 0 check_eq 'APOE double heterozygote is e2/e4' "$(awk -F'\t' '$1 == "APOE" {print $10}' "${CASE_TMP}/rules.tsv")" e2/e4 check_eq 'the taster rule needs phase' "$(awk -F'\t' '$1 == "TAS2R38" {print $11}' "${CASE_TMP}/rules.tsv")" unphased check_eq 'APOE line occurs exactly once' "$(grep -c '^rs429358+rs7412' "${CASE_TMP}/traits.tsv")" 1 +# shellcheck disable=SC2016 # awk field references check 'absent LCT has no inferred genotype' awk -F'\t' '$1 == "rs4988235" {ok = ($6 == "." && $7 == "not_covered")} END {exit !ok}' "${CASE_TMP}/traits.tsv" "$OG" traits phase-sites --rules "$RULES" --genotypes "$G" --sample S --out "${CASE_TMP}/phase.vcf" check_eq 'phase-sites exits 0' "$?" 0 diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 2a2fe1e3..70faade6 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -977,3 +977,4 @@ traits sites-single gen:traits.sh phase-sites --rules rules.tsv --genotypes sing traits check gen:traits.sh check --table traits.tsv --rules rules.tsv - bytes PYTHON_IMAGE traits bad-table gen:traits.sh check --table bad-table.tsv --rules rules.tsv - bytes PYTHON_IMAGE traits bad-rules gen:traits.sh check --table traits.tsv --rules bad-rules.tsv - bytes PYTHON_IMAGE +traits shared-vocabulary gen:traits.sh table --table traits.tsv --rules shared-rules.tsv --genotypes shared.tsv --sample S --out traits-out.tsv --rules-out rules-out.tsv traits-out.tsv,rules-out.tsv tsv PYTHON_IMAGE diff --git a/tests/parity/evidence/rust-c2b-controls.txt b/tests/parity/evidence/rust-c2b-controls.txt new file mode 100644 index 00000000..bed580b7 --- /dev/null +++ b/tests/parity/evidence/rust-c2b-controls.txt @@ -0,0 +1,69 @@ +Local execution of the Guard step "Traits controls", Rust 1.93.0. +Command: cargo test --locked -p og-steps traits::tests --no-fail-fast +Faults: reverse the diplotype key list; remove the genotype REF guard; +clear the phase-site list before writing the VCF. +The step requires each of the three named regression tests to fail, then +restores the source. A compiler failure alone does not satisfy the control. + +running 8 tests +test traits::tests::csv_header_unicode_rules_and_atomic_write_errors ... ok +test traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls ... FAILED +test traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory ... FAILED +test traits::tests::shared_position_keeps_each_rules_allele_vocabulary ... ok +test traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported ... FAILED +test traits::tests::unwritable_rules_output_keeps_existing_traits_table ... FAILED +test traits::tests::committed_catalogs_and_every_rule_genotype ... FAILED +test traits::tests::schema_faults_are_refused_before_enumeration ... ok + +failures: + +---- traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls stdout ---- + +thread 'traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls' (4288523) panicked at crates/og-steps/src/traits.rs:1242:17: +assertion `left == right` failed + left: "other_haplotype" + right: "called" +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory stdout ---- + +thread 'traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory' (4288529) panicked at crates/og-steps/src/traits.rs:1336:45: +called `Result::unwrap_err()` on an `Ok` value: {("chr2", "135851076"): SiteGt { alleles: ["G", "A"], status: Called, gq: "42", snv: true }} + +---- traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported stdout ---- + +thread 'traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported' (4288526) panicked at crates/og-steps/src/traits.rs:1390:9: +assertion `left == right` failed + left: 0 + right: 3 + +---- traits::tests::unwritable_rules_output_keeps_existing_traits_table stdout ---- +ERROR: /Volumes/Data/repos/personal/.worktrees/pgp-rust-c2b/rust/crates/og-steps/../../../assets/genotype_rules.tsv: rule APOE (line 19): no combination line for e2/e4|e1/e3 +ERROR: /Volumes/Data/repos/personal/.worktrees/pgp-rust-c2b/rust/crates/og-steps/../../../assets/genotype_rules.tsv: rule APOE (line 19): the combination line e1/e3|e2/e4 can never be read (a rule that needs phase reads one diplotype) + +thread 'traits::tests::unwritable_rules_output_keeps_existing_traits_table' (4288530) panicked at crates/og-steps/src/traits.rs:1433:37: +called `Result::unwrap_err()` on an `Ok` value: 1 + +---- traits::tests::committed_catalogs_and_every_rule_genotype stdout ---- + +thread 'traits::tests::committed_catalogs_and_every_rule_genotype' (4288524) panicked at crates/og-steps/src/traits.rs:1216:9: +assertion `left == right` failed + left: ["rule APOE (line 19): no combination line for e2/e4|e1/e3", "rule APOE (line 19): the combination line e1/e3|e2/e4 can never be read (a rule that needs phase reads one diplotype)"] + right: [] + + +failures: + traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls + traits::tests::committed_catalogs_and_every_rule_genotype + traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported + traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory + traits::tests::unwritable_rules_output_keeps_existing_traits_table + +test result: FAILED. 3 passed; 5 failed; 0 ignored; 0 measured; 91 filtered out; finished in 0.01s + +error: test failed, to rerun pass `-p og-steps --lib` +error: 1 target failed: + `-p og-steps --lib` +test traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls ... FAILED +test traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory ... FAILED +test traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported ... FAILED diff --git a/tests/parity/gen/traits.sh b/tests/parity/gen/traits.sh index a5777138..22766039 100644 --- a/tests/parity/gen/traits.sh +++ b/tests/parity/gen/traits.sh @@ -2,8 +2,8 @@ # Public catalogs and synthetic genotype grids for all three traits verbs. set -euo pipefail root=$(cd "$(dirname "$0")/../../.." && pwd) -cp "$root/assets/traits.tsv" traits.tsv -cp "$root/assets/genotype_rules.tsv" rules.tsv +cp -f "$root/assets/traits.tsv" traits.tsv +cp -f "$root/assets/genotype_rules.tsv" rules.tsv cat > oracle.sh <<'EOF' prog=$1; shift mode=$1; shift @@ -40,3 +40,15 @@ printf 'chr7\t141973545\t.\tC\tG\t.\tPASS\t.\tGT:PS\t1|0:7\nchr7\t141972905\t.\t sed '1s/:7$/:8/' phased.vcf > split.vcf awk -F'\t' -v OFS='\t' '$1 == "combination" && $2 == "APOE" && $3 == "e3/e3" {next} {print}' rules.tsv > bad-rules.tsv awk -F'\t' -v OFS='\t' '$3 == "rs671" {$6=$5} {print}' traits.tsv > bad-table.tsv +cp -f rules.tsv shared-rules.tsv +cat >> shared-rules.tsv <<'TSV' +rule SHARE SHARE Shared position test no no Synthetic alternate allele vocabulary. 1 +site SHARE rs999999991 chr19 44908684 T G +site SHARE rs999999992 chr19 44908822 C T +haplotype SHARE h0 T,C +haplotype SHARE h1 G,T +combination SHARE h0/h0 normal Reference pair. +combination SHARE h0/h1 carrier One alternate copy. +combination SHARE h1/h1 alternate Two alternate copies. +TSV +printf 'chr19\t44908684\tT\tG\tT/G\t40\nchr19\t44908822\tC\tT\tC/T\t40\n' > shared.tsv diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 084de768..268a9ce9 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -9,4 +9,4 @@ bin/pgx_outside_calls.py HLA_GENES,NO_CALL,PROBABLE,NOT_CALLABLE,LOW_GQ,is_call, bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/collect_summary.py,bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 bin/cnvpytor_to_vcf.py DEPTH_MIN_SIZE,delly_depth_contradicts bin/couple_report.py R4 bin/annotsv_postfilter.py CONSENSUS_BP,KEEP,ADDED,open_text,chrom_key,sv_type,info_value,sample_format,sample_columns,read_consensus,consensus_support bin/collect_summary.py,tests/demo/make_demo_sample.py R3 -bin/traits_report.py COLUMNS,OUT_COLUMNS,RULE_OUT_COLUMNS,TRAITS_TABLE_RULES,TableError,read_table,MARKERS,MARKER_STATUSES bin/collect_summary.py,tests/demo/make_demo_sample.py,tests/test_traits_report.py,tests/test_traits_table.py,tests/test_collect_summary.py,tests/test_lpa_kiv2.py,tests/test_render_report.py R3 +bin/traits_report.py COLUMNS,OUT_COLUMNS,RULE_OUT_COLUMNS,TRAITS_TABLE_RULES,TableError,read_table,MARKERS,MARKER_STATUSES bin/collect_summary.py,tests/demo/make_demo_sample.py,tests/test_traits_report.py,tests/test_traits_table.py,tests/test_collect_summary.py,tests/test_lpa_kiv2.py R3 diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 03a4bf34..3cb0eb2d 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number -locus, and what bin/copy_number.py keeps (the catalog reader and assess(), -which step 41's report imports). +locus, compared with the pinned historical catalog reader and assess() oracle. Without og (OG unset) the checks of the catalog and of assess() run, and the wiring; with OG naming the og binary (OG=rust/target/release/og, or bin/og) diff --git a/tests/test_render_report.py b/tests/test_render_report.py index a8377a4d..beb0497c 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -954,9 +954,7 @@ def mosaic_table(rows): # 11. traits card import time - import traits_report d = f"{work}/traits/S" - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) put(f"{d}/traits/S_traits.tsv", "") from test_traits_report import traits_tables g = os.path.join(work, "traits_empty.tsv") diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index a10e13f4..2414935b 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -105,8 +105,7 @@ def planted(desc, new, want): "MAPT": [(("G/G", "G/G"), "H2/H2"), (("A/G", "A/G"), "H1/H2")], } # The label of the reference blocks alone (every site homozygous for the GRCh38 base). -MULTI_SNP_REF = {"APOL1": "G0/G0", "SERPINA1": "PiMM", "HFE": "wt/wt", "GC": "Gc1F/Gc1F", - "CAD9p21": "no risk copy", "MAPT": "H1/H1"} + def says_not_proof(message): diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 84cf6020..3d2566b2 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -1,33 +1,10 @@ #!/usr/bin/env python3 -"""assets/traits.tsv, the curated table of step 42: its schema, its reference -bases and its sources. - - python3 tests/test_traits_table.py schema (the unit-tests job) - python3 tests/test_traits_table.py --fasta REF.fasta also: each row's ref is the - FASTA's base, on every contig - the FASTA has (needs REF.fasta.fai) - python3 tests/test_traits_table.py --online also: every PMID resolves in - PubMed, and every rsID's GRCh38 - position, reference base and - alt allele are dbSNP's (curl) - --table FILE checks another table (the e2e case plants a broken copy). - -The e2e case traits-panel-1 runs it with --fasta (the fixture's GRCh38 -reference) and --online, and on a planted copy with one wrong reference base -and one PubMed id that does not exist, which must fail. - -Checks without options: - 1. the table passes bin/traits_report.py's schema check (columns, chr1 to - chr22, positions, one base each for ref and alt, readings, ancestry - yes/no, PubMed ids, the two APOE rows, no SNPedia text); - 2. planted problems are each caught: a wrong column count, ref = alt, a - non-base allele, a duplicate rsID, a PMID that is not a number, a - missing APOE row; - 3. the FASTA check on a three-base synthetic FASTA: the right base passes, - a planted wrong base fails, a contig the FASTA lacks is counted as not - checked. - -Run: python3 tests/test_traits_table.py +"""Validate the curated traits catalog and its GRCh38 reference bases. + +Schema checks use og traits check. Rust unit tests plant invalid catalog +rows and rules. The optional --fasta and --online checks also verify the +FASTA bases, PubMed sources and dbSNP records. A synthetic FASTA provides +positive, wrong-base and missing-contig controls without network access. """ import argparse import json From 853dda63ca5dbd14eb8ab2e86ee5b63cdfd8146f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:39:11 +0100 Subject: [PATCH 6/8] docs(traits): describe the removed copy-number test oracle Bead: pgp-bs9u.25 --- docs/cn-loci.md | 2 +- docs/rust.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn-loci.md b/docs/cn-loci.md index eb6f0a44..247ffc02 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -1,6 +1,6 @@ # Copy Number from Depth (the locus catalog) -Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one table, `bin/cn_loci.tsv`, with one row per locus, and one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)). Step 41 uses the same engine: `og cn bed` gives it the RHD regions, and `og bloodtype report` makes the RhD call with og cn's rules on the RHD row. `bin/copy_number.py` stays only for the catalog tests. A new locus is a new row, not a new script. +Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one table, `bin/cn_loci.tsv`, with one row per locus, and one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)). Step 41 uses the same engine: `og cn bed` gives it the RHD regions, and `og bloodtype report` makes the RhD call with og cn's rules on the RHD row. The catalog tests materialise the historical Python engine from its pinned Git commit as a differential oracle. A new locus is a new row, not a new script. The table sits in `bin/` because `bin/` is the one repository folder that every container mounts: the bash steps mount it as `/pgp-bin`, and Nextflow puts it on every task's PATH. diff --git a/docs/rust.md b/docs/rust.md index 99637ea7..728a2abd 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -37,7 +37,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. From b7be2494e541e8215a1bfddc5c6f8fe1f55c2529 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:17:20 +0100 Subject: [PATCH 7/8] test(traits): restore independent expectations and output refusal controls --- rust/crates/og-steps/src/traits.rs | 39 +++++++++- tests/test_traits_report.py | 121 ++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index d537e672..a0f323c3 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -1435,8 +1435,43 @@ mod tests { ); } + + #[test] + fn catalog_quotes_are_literal_and_embedded_tabs_are_columns() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); + let text = std::fs::read_to_string(root.join("traits.tsv")).unwrap(); + let rules = std::fs::read_to_string(root.join("genotype_rules.tsv")).unwrap(); + let scratch = Scratch::new(); + let path = scratch.put("quoted-field", &text.replacen("\tMCM6\t", "\t\"MCM6\"\t", 1)); + assert!(check_rows(&read_table(&path).unwrap()).iter().any(|e| e.contains("quote"))); + let path = scratch.put("quoted-rule", &rules.replacen("\tAPOE\t", "\t\"APOE\"\t", 1)); + assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); + let path = scratch.put("embedded-tab", &text.replacen("\tMCM6\t", "\t\"MCM6\textra\"\t", 1)); + assert!(read_table(&path).unwrap_err().contains("14 columns, expected 13")); + } + + #[test] + fn directory_destinations_keep_both_previous_tables() { + let scratch = Scratch::new(); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); + let gt = scratch.put("g", ""); + for directory_key in ["out", "rules-out"] { + let previous = scratch.put("previous.tsv", "previous complete table\n"); + let mut opts: HashMap = [ + ("table", root.join("traits.tsv").to_string_lossy().into_owned()), + ("rules", root.join("genotype_rules.tsv").to_string_lossy().into_owned()), + ("genotypes", gt.clone()), ("sample", "S".into()), + ("out", previous.clone()), ("rules-out", previous.clone()), + ].into_iter().map(|(k,v)| (k.into(),v)).collect(); + opts.insert(directory_key.into(), scratch.0.to_string_lossy().into_owned()); + assert!(run("table", &opts).unwrap_err().contains("cannot write")); + assert_eq!(std::fs::read_to_string(&previous).unwrap(), "previous complete table\n", "{directory_key}"); + assert_eq!(std::fs::read_dir(&scratch.0).unwrap().count(), 2); + } + } + #[test] - fn csv_header_unicode_rules_and_atomic_write_errors() { + fn plain_header_unicode_rules_and_atomic_write_errors() { let scratch = Scratch::new(); let mut text = COLUMNS .iter() @@ -1445,7 +1480,7 @@ mod tests { .join("\t"); text.push_str("\r\n"); let path = scratch.put("quoted", &text); - assert!(read_table(&path).unwrap().is_empty()); + assert!(read_table(&path).unwrap_err().contains("unexpected traits header")); let path = scratch.put("r", "rule\té\tG\tT\tno\tyes\tN\t1\n"); assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); assert!(stage_tsv(scratch.0.join("absent/x").to_str().unwrap(), OUT, &[]).is_err()); diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index 2414935b..b93c2155 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -104,7 +104,9 @@ def planted(desc, new, want): "CAD9p21": [(("G/G", "C/C"), "two risk copies"), (("A/G", "G/C"), "one risk copy")], "MAPT": [(("G/G", "G/G"), "H2/H2"), (("A/G", "A/G"), "H1/H2")], } -# The label of the reference blocks alone (every site homozygous for the GRCh38 base). +# Independent labels for homozygous GRCh38 reference blocks. +MULTI_SNP_REF = {"APOL1": "G0/G0", "SERPINA1": "PiMM", "HFE": "wt/wt", "GC": "Gc1F/Gc1F", + "CAD9p21": "no risk copy", "MAPT": "H1/H1"} @@ -113,8 +115,50 @@ def says_not_proof(message): return "does not prove 2+0" in message +def marker_errors(name, m=None, dbsnp=None): + """Problems of one marker (empty when it is consistent): the dbSNP record + is one of its copies with its ref and alt, on the plus strand and not + swapped; every copy sits in the middle of the same context, whose middle + base is the ref.""" + m = tr.MARKERS[name] if m is None else m + rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp + comp = {"A": "T", "C": "G", "G": "C", "T": "A"} + errs = [] + if chrom != m["chrom"] or pos not in m["copies"]: + errs.append(f"{name}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies") + if (m["ref"], m["alt"]) == (alt, ref): + errs.append(f"{name}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt})") + elif (m["ref"], m["alt"]) == (comp.get(ref), comp.get(alt)): + errs.append(f"{name}: the alleles are the minus-strand bases (marker {m['ref']}>{m['alt']}, " + f"dbSNP {ref}>{alt}): write the plus-strand bases") + elif (m["ref"], m["alt"]) != (ref, alt): + errs.append(f"{name}: marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt}") + ctx = m["context"] + if len(ctx) % 2 != 1 or ctx[len(ctx) // 2] != m["ref"]: + errs.append(f"{name}: the middle base of the context is not the ref {m['ref']}") + if len(set(m["copies"])) != len(m["copies"]): + errs.append(f"{name}: a copy is listed twice") + return errs + + def marker_checks(): """11. The copy-aware markers.""" + for name, m in tr.MARKERS.items(): + check(f"marker {name}: the pinned dbSNP record agrees with the marker", marker_errors(name) == [], + marker_errors(name)) + rs, chrom, pos, ref, alt = m["dbsnp"] + comp = {"A": "T", "C": "G", "G": "C", "T": "A"} + e = marker_errors(name, dict(m, ref=alt, alt=ref)) + check(f"marker {name} control: ref and alt swapped is caught", any("swapped" in x for x in e), e) + e = marker_errors(name, dict(m, ref=comp[ref], alt=comp[alt], + context=m["context"][:10] + comp[ref] + m["context"][11:])) + check(f"marker {name} control: the minus-strand bases are caught", any("minus-strand" in x for x in e), e) + e = marker_errors(name, dbsnp=(rs, chrom, pos + 1, ref, alt)) + check(f"marker {name} control: a dbSNP position off the copies is caught", + any("not at one of the marker's copies" in x for x in e), e) + e = marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) + check(f"marker {name} control: a context whose middle base is not the ref is caught", + any("middle base" in x for x in e), e) check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) if not OG: print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") @@ -231,6 +275,44 @@ def og_cn(*args): return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + +def plain_tsv_checks(): + """Reject CSV quoting before it can produce a table the collector cannot read.""" + import collect_summary as cs + with tempfile.TemporaryDirectory() as work: + folder = os.path.join(work, "traits") + os.makedirs(folder) + g = os.path.join(work, "g.tsv") + write(g, "") + out, rout = os.path.join(folder, "S_traits.tsv"), os.path.join(folder, "S_genotype_rules.tsv") + source = open(TABLE).read() + rules_source = open(RULES).read() + for name, text, rules_text in [ + ("quoted header", source.replace("chrom\t", '\"chrom\"\t', 1), rules_source), + ("quoted catalog field", source.replace("\tMCM6\t", '\t\"MCM6\"\t', 1), rules_source), + ("quoted rule text", source, rules_source.replace("\tAPOE\t", '\t\"APOE\"\t', 1)), + ("embedded tab", source.replace("\tMCM6\t", '\t\"MCM6\textra\"\t', 1), rules_source), + ]: + # Keep complete, collector-readable outputs from the previous run. + traits_tables(g, out, rout) + previous = open(out, "rb").read(), open(rout, "rb").read() + bad, badrules = os.path.join(work, "bad.tsv"), os.path.join(work, "badrules.tsv") + write(bad, text) + write(badrules, rules_text) + res = subprocess.run([OG, "traits", "table", "--table", bad, "--rules", badrules, + "--genotypes", g, "--sample", "S", "--out", out, + "--rules-out", rout], capture_output=True, text=True) + check(f"plain TSV control: {name} is refused", res.returncode == 1, res.stderr) + check(f"plain TSV control: {name} preserves both outputs", + previous == (open(out, "rb").read(), open(rout, "rb").read())) + try: + _, collected = cs.sec_traits(work, "S") + readable = bool(collected["rows"]) + except cs.Unreadable: + readable = False + check(f"plain TSV control: {name} leaves collector-readable output", readable) + + def main(): rules = {} with open(RULES) as f: @@ -238,15 +320,18 @@ def main(): if cells and cells[0] == "site": rules.setdefault(cells[1], {"sites": []})["sites"].append(cells[2:]) dbsnp_checks(rules) + marker_checks() if not OG: check("OG_REQUIRED needs the og binary", not os.environ.get("OG_REQUIRED")) print("skip: OG is not set: traits CLI and cn marker checks did not run") return int(bool(FAILS)) + plain_tsv_checks() with tempfile.TemporaryDirectory() as work: def table(records): g = os.path.join(work, "g.tsv") write(g, "".join("\t".join(rec) + "\n" for rec in records)) - out, rout = os.path.join(work, "traits.tsv"), os.path.join(work, "rules.tsv") + os.makedirs(os.path.join(work, "traits"), exist_ok=True) + out, rout = os.path.join(work, "traits", "S_traits.tsv"), os.path.join(work, "traits", "S_genotype_rules.tsv") res = subprocess.run([OG, "traits", "table", "--table", TABLE, "--rules", RULES, "--genotypes", g, "--sample", "S", "--out", out, "--rules-out", rout], capture_output=True, text=True) @@ -271,7 +356,37 @@ def table(records): _, lines = table(records) got = next(r for r in lines if r["rule"] == rid) check(f"{rid} {gts}: {label}", got["status"] == "called" and got["result"] == label, got) - marker_checks() + import collect_summary as cs + import render_report as rr + check("every multi-SNP rule is in the catalog", set(MULTI_SNP) <= set(rules)) + for rid, cases in MULTI_SNP.items(): + sites = rules[rid]["sites"] + records = [(s[1], s[2], s[3], ".", f"{s[3]}/{s[3]}", "40") for s in sites] + _, lines = table(records) + got = next(r for r in lines if r["rule"] == rid) + check(f"{rid}, reference blocks only: {MULTI_SNP_REF[rid]}", + (got["status"], got["result"]) == ("called", MULTI_SNP_REF[rid]), got) + gts, want = cases[0] + a, b = gts[0].split("/") + s0 = sites[0] + flip = s0[3] if b == s0[4] else s0[4] + flipped = (f"{a}/{flip}",) + tuple(gts[1:]) + records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, flipped)] + _, lines = table(records) + got = next(r for r in lines if r["rule"] == rid) + check(f"control: {rid} with one allele flipped is not {want}", got["result"] != want, got) + # Collect and render the actual CLI output, independently for each rule. + records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, gts)] + _, lines = table(records) + read = cs.genotype_rules(work, "S") or [] + page = "\n".join(rr.rules_html({"rules": read}, "S")) + got = next(r for r in lines if r["rule"] == rid) + check(f"HTML report: {rid} with result {want}", rr.E(got["trait"]) in page and rr.E(want) in page, page[:300]) + records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(rules["APOL1"]["sites"], ("A/G", "T/G"))] + _, lines = table(records) + reading = next(r for r in lines if r["rule"] == "APOL1")["reading"] + check("APOL1 G0/G1 says G2 is not read and the other copy could carry it", + "G2 is not read" in reading and "G1/G2" in reading, reading) print("RESULT:", "ALL PASS" if not FAILS else f"{FAILS} FAILED") return int(bool(FAILS)) From d0c12a52666fe22a01112b095ab9df2d2f129c77 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:22:13 +0100 Subject: [PATCH 8/8] fix(traits): preserve plain TSV and reject directory outputs before publication --- .github/workflows/guard.yml | 37 +++++++ CHANGELOG.md | 2 +- rust/crates/og-steps/src/traits.rs | 104 +++++++++++++------- tests/parity/evidence/rust-c2b-controls.txt | 40 +++++++- tests/test_traits_report.py | 2 +- 5 files changed, 144 insertions(+), 41 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 66800c3a..d2a8d74a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -386,6 +386,43 @@ jobs: done exit "$fail" + - name: Traits independent expectations reject changed reference labels and marker metadata + working-directory: . + env: + OG: ${{ github.workspace }}/bin/og + OG_REQUIRED: 1 + run: | + python3 - <<'PY' + import contextlib, io, pathlib, runpy, tempfile + test = runpy.run_path("tests/test_traits_report.py")["main"].__globals__ + original = pathlib.Path(test["RULES"]).read_text() + with tempfile.TemporaryDirectory() as work: + for rid, key, label in [("APOL1", "G0/G0", "G0/G0"), ("CAD9p21", "nonrisk/nonrisk", "no risk copy")]: + path = pathlib.Path(work) / "rules.tsv" + old = f"combination\t{rid}\t{key}\t{label}\t" + assert old in original + path.write_text(original.replace(old, f"combination\t{rid}\t{key}\tWRONG_REFERENCE\t")) + test["RULES"], test["FAILS"] = str(path), 0 + log = io.StringIO() + with contextlib.redirect_stdout(log): + rc = test["main"]() + print(log.getvalue()) + assert rc == 1 and test["FAILS"] == 1 + assert f"[FAIL] {rid}, reference blocks only: {label}" in log.getvalue() + print(f"Control: {rid} WRONG_REFERENCE failed its independent expectation") + marker = test["tr"].MARKERS["SMN1_27134"] + rs, chrom, pos, ref, alt = marker["dbsnp"] + marker["dbsnp"] = rs, chrom, pos + 1, ref, alt + test["FAILS"] = 0 + log = io.StringIO() + with contextlib.redirect_stdout(log): + test["marker_checks"]() + print(log.getvalue()) + assert test["FAILS"] == 1 + assert "[FAIL] marker SMN1_27134: the pinned dbSNP record agrees" in log.getvalue() + print("Control: retained SMN1 off-copy metadata failed its independent expectation") + PY + # The fake-docker job builds no og, so its cn-engine-calls case says it # is skipped there; here bin/og is built and the case must run. - name: The fake-docker case cn-engine-calls (steps 35, 41 and 46 through og cn) with og diff --git a/CHANGELOG.md b/CHANGELOG.md index b6245e95..3e56f60b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published, so an unwritable second output leaves the previous traits table intact. +- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. - The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index a0f323c3..fa6fef4b 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -1,7 +1,6 @@ //! Step 42: bounded site genotypes, haplotype rules and read-backed phase. use indexmap::IndexMap; use og_io::{AtomicFile, Lines, Utf8}; -use pycompat::csv::{Quoting, Reader, Writer}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::io::Write; @@ -111,7 +110,7 @@ fn row(keys: &[&str], values: &[String]) -> Row { .collect() } fn records(path: &str) -> Result)>> { - // Parse even the header as a CSV record. Comments do not enter the parser. + // Catalogs are plain TSV: quotes are literal and never hide a tab. let mut lines = Lines::new( og_io::open(path).map_err(|e| og_filters::os_error(&e, path))?, Utf8::Strict, @@ -124,14 +123,11 @@ fn records(path: &str) -> Result)>> { if text.trim_end_matches('\n').is_empty() || text.starts_with('#') { continue; } - let cells = Reader::new( - std::iter::once(Ok::<_, std::io::Error>(text.clone())), - '\t', - Quoting::Minimal, - ) - .next() - .ok_or("empty record")? - .map_err(|e| e.to_string())?; + let cells = text + .trim_end_matches('\n') + .split('\t') + .map(str::to_owned) + .collect(); out.push((number, cells)); } Ok(out) @@ -857,16 +853,10 @@ fn rule_line(r: &Rule, gt: &Genotypes, phased: Option<&Phased>) -> Row { } fn stage_tsv(path: &str, columns: &[&str], rows: &[Row]) -> Result { let mut file = AtomicFile::create(path).map_err(|e| format!("cannot write {path}: {e}"))?; - { - let mut writer = Writer::new(&mut file, '\t'); - writer - .write_row(columns.iter().copied()) - .map_err(|e| format!("cannot write {path}: {e}"))?; - for row in rows { - writer - .write_row(columns.iter().map(|k| row[*k].as_str())) - .map_err(|e| format!("cannot write {path}: {e}"))?; - } + writeln!(file, "{}", columns.join("\t")).map_err(|e| format!("cannot write {path}: {e}"))?; + for row in rows { + let fields: Vec<_> = columns.iter().map(|k| row[*k].as_str()).collect(); + writeln!(file, "{}", fields.join("\t")).map_err(|e| format!("cannot write {path}: {e}"))?; } file.flush() .map_err(|e| format!("cannot write {path}: {e}"))?; @@ -1048,6 +1038,13 @@ fn run(verb: &str, opts: &HashMap) -> Result { // Compute both tables before opening either output. Each rename is atomic. let out_path = required("out")?; let rules_path = required("rules-out")?; + // Refuse known rename failures before either previous output is replaced. + // Publication still uses two renames, not one transaction. + for path in [out_path, rules_path] { + if std::path::Path::new(path).is_dir() { + return Err(format!("cannot write {path}: destination is a directory")); + } + } let traits_file = stage_tsv(out_path, OUT, &out)?; let rules_file = stage_tsv(rules_path, RULE_OUT, &lines)?; traits_file @@ -1435,19 +1432,35 @@ mod tests { ); } - #[test] fn catalog_quotes_are_literal_and_embedded_tabs_are_columns() { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); let text = std::fs::read_to_string(root.join("traits.tsv")).unwrap(); let rules = std::fs::read_to_string(root.join("genotype_rules.tsv")).unwrap(); let scratch = Scratch::new(); - let path = scratch.put("quoted-field", &text.replacen("\tMCM6\t", "\t\"MCM6\"\t", 1)); - assert!(check_rows(&read_table(&path).unwrap()).iter().any(|e| e.contains("quote"))); - let path = scratch.put("quoted-rule", &rules.replacen("\tAPOE\t", "\t\"APOE\"\t", 1)); + let path = scratch.put( + "quoted-field", + &text.replacen("\tMCM6\t", "\t\"MCM6\"\t", 1), + ); + assert!( + check_rows(&read_table(&path).unwrap()) + .iter() + .any(|e| e.contains("quote")) + ); + let path = scratch.put( + "quoted-rule", + &rules.replacen("\tAPOE\tAPOE\t", "\tAPOE\t\"APOE\"\t", 1), + ); assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); - let path = scratch.put("embedded-tab", &text.replacen("\tMCM6\t", "\t\"MCM6\textra\"\t", 1)); - assert!(read_table(&path).unwrap_err().contains("14 columns, expected 13")); + let path = scratch.put( + "embedded-tab", + &text.replacen("\tMCM6\t", "\t\"MCM6\textra\"\t", 1), + ); + assert!( + read_table(&path) + .unwrap_err() + .contains("14 columns, expected 13") + ); } #[test] @@ -1458,14 +1471,33 @@ mod tests { for directory_key in ["out", "rules-out"] { let previous = scratch.put("previous.tsv", "previous complete table\n"); let mut opts: HashMap = [ - ("table", root.join("traits.tsv").to_string_lossy().into_owned()), - ("rules", root.join("genotype_rules.tsv").to_string_lossy().into_owned()), - ("genotypes", gt.clone()), ("sample", "S".into()), - ("out", previous.clone()), ("rules-out", previous.clone()), - ].into_iter().map(|(k,v)| (k.into(),v)).collect(); - opts.insert(directory_key.into(), scratch.0.to_string_lossy().into_owned()); + ( + "table", + root.join("traits.tsv").to_string_lossy().into_owned(), + ), + ( + "rules", + root.join("genotype_rules.tsv") + .to_string_lossy() + .into_owned(), + ), + ("genotypes", gt.clone()), + ("sample", "S".into()), + ("out", previous.clone()), + ("rules-out", previous.clone()), + ] + .into_iter() + .map(|(k, v)| (k.into(), v)) + .collect(); + opts.insert( + directory_key.into(), + scratch.0.to_string_lossy().into_owned(), + ); assert!(run("table", &opts).unwrap_err().contains("cannot write")); - assert_eq!(std::fs::read_to_string(&previous).unwrap(), "previous complete table\n", "{directory_key}"); + assert!( + std::fs::read_to_string(&previous).unwrap() == "previous complete table\n", + "{directory_key}: previous output changed" + ); assert_eq!(std::fs::read_dir(&scratch.0).unwrap().count(), 2); } } @@ -1480,7 +1512,11 @@ mod tests { .join("\t"); text.push_str("\r\n"); let path = scratch.put("quoted", &text); - assert!(read_table(&path).unwrap_err().contains("unexpected traits header")); + assert!( + read_table(&path) + .unwrap_err() + .contains("unexpected traits header") + ); let path = scratch.put("r", "rule\té\tG\tT\tno\tyes\tN\t1\n"); assert!(!check_rules(&read_rules(&path).unwrap(), None).is_empty()); assert!(stage_tsv(scratch.0.join("absent/x").to_str().unwrap(), OUT, &[]).is_err()); diff --git a/tests/parity/evidence/rust-c2b-controls.txt b/tests/parity/evidence/rust-c2b-controls.txt index bed580b7..9637cfe0 100644 --- a/tests/parity/evidence/rust-c2b-controls.txt +++ b/tests/parity/evidence/rust-c2b-controls.txt @@ -19,7 +19,7 @@ failures: ---- traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls stdout ---- -thread 'traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls' (4288523) panicked at crates/og-steps/src/traits.rs:1242:17: +thread 'traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls' (4288523) panicked at traits.rs (line 1242, column 17): assertion `left == right` failed left: "other_haplotype" right: "called" @@ -27,12 +27,12 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace ---- traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory stdout ---- -thread 'traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory' (4288529) panicked at crates/og-steps/src/traits.rs:1336:45: +thread 'traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory' (4288529) panicked at traits.rs (line 1336, column 45): called `Result::unwrap_err()` on an `Ok` value: {("chr2", "135851076"): SiteGt { alleles: ["G", "A"], status: Called, gq: "42", snv: true }} ---- traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported stdout ---- -thread 'traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported' (4288526) panicked at crates/og-steps/src/traits.rs:1390:9: +thread 'traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported' (4288526) panicked at traits.rs (line 1390, column 9): assertion `left == right` failed left: 0 right: 3 @@ -41,12 +41,12 @@ assertion `left == right` failed ERROR: /Volumes/Data/repos/personal/.worktrees/pgp-rust-c2b/rust/crates/og-steps/../../../assets/genotype_rules.tsv: rule APOE (line 19): no combination line for e2/e4|e1/e3 ERROR: /Volumes/Data/repos/personal/.worktrees/pgp-rust-c2b/rust/crates/og-steps/../../../assets/genotype_rules.tsv: rule APOE (line 19): the combination line e1/e3|e2/e4 can never be read (a rule that needs phase reads one diplotype) -thread 'traits::tests::unwritable_rules_output_keeps_existing_traits_table' (4288530) panicked at crates/og-steps/src/traits.rs:1433:37: +thread 'traits::tests::unwritable_rules_output_keeps_existing_traits_table' (4288530) panicked at traits.rs (line 1433, column 37): called `Result::unwrap_err()` on an `Ok` value: 1 ---- traits::tests::committed_catalogs_and_every_rule_genotype stdout ---- -thread 'traits::tests::committed_catalogs_and_every_rule_genotype' (4288524) panicked at crates/og-steps/src/traits.rs:1216:9: +thread 'traits::tests::committed_catalogs_and_every_rule_genotype' (4288524) panicked at traits.rs (line 1216, column 9): assertion `left == right` failed left: ["rule APOE (line 19): no combination line for e2/e4|e1/e3", "rule APOE (line 19): the combination line e1/e3|e2/e4 can never be read (a rule that needs phase reads one diplotype)"] right: [] @@ -67,3 +67,33 @@ error: 1 target failed: test traits::tests::apoe_grid_keeps_haplotype_order_and_missing_calls ... FAILED test traits::tests::stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory ... FAILED test traits::tests::phase_requires_one_block_and_only_multiple_hets_are_exported ... FAILED + +Fix round 1: local red-first controls, Rust 1.93.0, synthetic inputs only. +The tests-only commit b7be2494 leaves production code unchanged. +Command: cargo test --locked -p og-steps traits::tests --no-fail-fast +test traits::tests::catalog_quotes_are_literal_and_embedded_tabs_are_columns ... FAILED +test traits::tests::plain_header_unicode_rules_and_atomic_write_errors ... FAILED +test traits::tests::directory_destinations_keep_both_previous_tables ... FAILED + +Command: OG_REQUIRED=1 python3 tests/test_traits_report.py (unfixed binary) +[FAIL] plain TSV control: quoted header is refused -- +[FAIL] plain TSV control: quoted catalog field is refused -- +[FAIL] plain TSV control: quoted rule text is refused -- +[FAIL] plain TSV control: embedded tab is refused -- +[FAIL] plain TSV control: embedded tab preserves both outputs -- +[FAIL] plain TSV control: embedded tab leaves collector-readable output -- + +Independent reference controls: a scratch copy of genotype_rules.tsv changes +only the reference label to WRONG_REFERENCE. The actual og tables must still +match the six fixed labels in tests/test_traits_report.py. +APOL1 G0/G0 changed: the APOL1 reference assertion fails. +CAD9p21 no risk copy changed: the CAD9p21 reference assertion fails. +SMN1 retained metadata: dbSNP position moved one base off its copies. +The pinned dbSNP agreement assertion fails. +Guard now repeats these three controls, requires the exact failed assertion +and exactly one failure per mutation, and keeps the catalogs unchanged. + +After fixes: all 10 traits Rust tests pass. The CLI controls refuse quoted +headers, quoted fields, quoted rule text and embedded tabs. The previous +outputs survive; collect_summary reads the unchanged table with QUOTE_NONE. +Both directory-destination controls preserve the other previous output. diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index b93c2155..cf55a959 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -290,7 +290,7 @@ def plain_tsv_checks(): for name, text, rules_text in [ ("quoted header", source.replace("chrom\t", '\"chrom\"\t', 1), rules_source), ("quoted catalog field", source.replace("\tMCM6\t", '\t\"MCM6\"\t', 1), rules_source), - ("quoted rule text", source, rules_source.replace("\tAPOE\t", '\t\"APOE\"\t', 1)), + ("quoted rule text", source, rules_source.replace("\tAPOE\tAPOE\t", '\tAPOE\t\"APOE\"\t', 1)), ("embedded tab", source.replace("\tMCM6\t", '\t\"MCM6\textra\"\t', 1), rules_source), ]: # Keep complete, collector-readable outputs from the previous run.