From 7c82b0efae81a7ff6cc8ea5a384ece1e39c99f24 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:00:37 +0100 Subject: [PATCH 01/76] test(vntyper): fetch VNtyper test BAMs with curl and gzip, no Python The Zenodo archive never changes, so the deflated bytes of each member sit at a fixed offset. Pin them, fetch one range per member, and let gzip check the CRC-32 and size the archive stores; the sha256 pin stays. Bead: pgp-bs9u.39 --- tests/e2e/vntyper-1.sh | 4 +- tests/smoke/commands.tsv | 2 +- tests/smoke/vntyper.sh | 4 +- tests/smoke/vntyper_testdata.py | 103 -------------------------------- tests/smoke/vntyper_testdata.sh | 65 ++++++++++++++++++++ 5 files changed, 70 insertions(+), 108 deletions(-) delete mode 100644 tests/smoke/vntyper_testdata.py create mode 100755 tests/smoke/vntyper_testdata.sh diff --git a/tests/e2e/vntyper-1.sh b/tests/e2e/vntyper-1.sh index 7cf376973..9e30e05dd 100755 --- a/tests/e2e/vntyper-1.sh +++ b/tests/e2e/vntyper-1.sh @@ -5,7 +5,7 @@ # over MUC1): VNtyper has nothing to read, and the step says "not # assessed" with that reason, never "negative"; # - two BAMs of VNtyper's own public test data (tests/smoke/ -# vntyper_testdata.py reads them out of its Zenodo archive, sha256 +# vntyper_testdata.sh reads them out of its Zenodo archive, sha256 # pinned): their reads over MUC1 carried onto the fixture's header and # added to case 20's BAM, as a whole-genome BAM would hold them. The # sample with a frameshift reads positive, with the variant, and the one @@ -38,7 +38,7 @@ check "the fixture: the message says the BAM has no read over MUC1" has 'no read # --- VNtyper's positive and negative samples on the fixture's header ------------------- check "VNtyper's two test BAMs download and match their sha256" \ - retry python3 -I "${REPO}/tests/smoke/vntyper_testdata.py" "$T" positive.bam negative.bam + retry sh "${REPO}/tests/smoke/vntyper_testdata.sh" "$T" positive.bam negative.bam H="${SAMPLE}muc" for s in positive negative; do dk "$SAMTOOLS_IMAGE" samtools index "${T}/${s}.bam" diff --git a/tests/smoke/commands.tsv b/tests/smoke/commands.tsv index 11e807eb3..7ff8d98b9 100644 --- a/tests/smoke/commands.tsv +++ b/tests/smoke/commands.tsv @@ -52,7 +52,7 @@ PARASCOPY_IMAGE net,also=PARASCOPY_DATA_VERSION fullref,slice sh /smoke/parascop PARAPHASE_IMAGE net fullref v=${PARAPHASE_IMAGE##*:} && u=https://raw.githubusercontent.com/PacificBiosciences/paraphase/v${v%%--*}/tests/data/bams/HG01175_smn1_extracted.bam && wget -q -O hg.bam "$u" && wget -q -O hg.bam.bai "${u}.bai" && { printf '%s hg.bam\n%s hg.bam.bai\n' c9659364ad9e60e41ee4771cf92dc017313b6b75a1e08df3e8c0e95ef38eadca 8b5dd238fdf1cc508e85105ff2a157a2c3d73aa3f441d08afdba9458beecca51 | sha256sum -c || { echo 'the demo BAM or its index is not the file Paraphase v4.1.0 ships'; exit 1; }; } && paraphase -b hg.bam -r /in/ref.fa -o pp -p HG01175 -g smn1 -t 4 && cp pp/HG01175.paraphase.json smn1.json && rm -rf pp hg.bam hg.bam.bai py "SMN1, SMN2 and SMN exon 7-8 deletion copy numbers of HG01175 as Paraphase's demo publishes them (2, 1, 1)" "import json; s = json.load(open('smn1.json'))['smn1']; print(json.dumps(s)[:3000]); r = s['region_specific_info']; c = (r['smn1_cn'], r['smn2_cn'], r['smn_del78_cn']); print(c); assert c == (2, 1, 1)" # MEGAnE's call needs a whole chromosome: on mini.fa's 500 kb it stops in its own code (IndexError). The e2e case mei-1 runs it on chr20. MEGANE_IMAGE - ref build_kmerset -fa /in/mini.fa -prefix mini -outdir /out/mk && { call_genotype_38 -h > help.txt 2>&1 || true; } at_least "repeated 32-mers in the k-mer set of mini.fa" "$(cat mk/mini.mi)" 1000; same "the .mk holds the 8-byte k-mers the .mi counts" "$(( $(wc -c < mk/mini.mk) / 8 ))" "$(cat mk/mini.mi)"; contains "call_genotype_38 takes the -mainchr step 50 passes" help.txt "-mainchr"; contains "and -no_sex_chr" help.txt "-no_sex_chr"; contains "and -only_ins" help.txt "-only_ins" -# Step 52 on VNtyper's own test data, two BAMs read out of its Zenodo archive by range requests (tests/smoke/vntyper_testdata.py, +# Step 52 on VNtyper's own test data, two BAMs read out of its Zenodo archive by range requests (tests/smoke/vntyper_testdata.sh, # sha256 pinned): a sample with a MUC1 VNTR frameshift must be called, and one without must be negative (the red control); # a BAM with no read over MUC1 must stop VNtyper with the message step 52 turns into "no read over MUC1". VNTYPER_IMAGE net - sh /smoke/vntyper.sh has "the positive sample: Kestrel calls a frameshift with high precision" "$(awk -F'\t' '$1 == "positive" {print $2}' calls.tsv)" "^High_Precision"; same "the negative sample: no call" "$(awk -F'\t' '$1 == "negative" {print $2}' calls.tsv)" Negative; at_least "the positive sample: mean depth over the VNTR" "$(awk -F'\t' '$1 == "positive" {print int($4)}' calls.tsv)" 100; at_least "the negative sample: mean depth over the VNTR" "$(awk -F'\t' '$1 == "negative" {print int($4)}' calls.tsv)" 100; same "a BAM with no read over MUC1: VNtyper stops with exit 1" "$(cat empty_rc.txt)" 1; contains "and its log names the empty read layout step 52 reads" empty.log "FASTQ layout .empty."; has "vntyper --version is the release of the image tag" "$(cat version.txt)" "${VNTYPER_IMAGE##*:}" diff --git a/tests/smoke/vntyper.sh b/tests/smoke/vntyper.sh index 0c3d7c7bb..cd4d6cec1 100644 --- a/tests/smoke/vntyper.sh +++ b/tests/smoke/vntyper.sh @@ -1,12 +1,12 @@ # shellcheck shell=sh # VNTYPER_IMAGE row: what step 52 runs, on VNtyper's own public test data -# (tests/smoke/vntyper_testdata.py fetches two BAMs out of its Zenodo archive): +# (tests/smoke/vntyper_testdata.sh fetches two BAMs out of its Zenodo archive): # a sample with a MUC1 VNTR frameshift and one without. The image's samtools # indexes them, then VNtyper runs with the step's flags (and no IGV view in its HTML) # from /opt/vntyper, # the folder its config.json paths are relative to. set -e -python3 /smoke/vntyper_testdata.py /out/data positive.bam negative.bam +sh /smoke/vntyper_testdata.sh /out/data positive.bam negative.bam for s in positive negative; do samtools index "/out/data/${s}.bam" (cd /opt/vntyper && vntyper pipeline --bam "/out/data/${s}.bam" -o "/out/${s}" --sample-name "$s" \ diff --git a/tests/smoke/vntyper_testdata.py b/tests/smoke/vntyper_testdata.py deleted file mode 100644 index b33731d94..000000000 --- a/tests/smoke/vntyper_testdata.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -"""Fetch members of VNtyper's public test data without the whole archive. - -VNtyper publishes its test BAMs as one 1.2 GB zip on Zenodo (record 19181821, -"VNtyper 2.0 test data", the archive_file of tests/test_data_config.json in -VNtyper v2.0.47). A zip keeps its file list at the end, so this script reads -that list with an HTTP range request, then fetches and inflates only the -members it is asked for, about 10 MB instead of 1.2 GB (the .bai files are -not fetched: samtools index writes them). Each member is checked -against the CRC-32 the archive stores and against the sha256 pinned below. - -Usage: vntyper_testdata.py OUTDIR NAME... (NAME: a key of MEMBERS) -Standard library only: it runs in VNTYPER_IMAGE's Python and on the host. -""" -import hashlib -import os -import struct -import sys -import time -import urllib.request -import zlib - -URL = "https://zenodo.org/records/19181821/files/data.zip?download=1" -SIZE = 1174975271 # bytes; Zenodo lists md5 fdd5958d58c733cdca452f3287898964 for the whole file - -# name -> (path in the archive, sha256 of the inflated file) -MEMBERS = { - # Positive: a MUC1 VNTR frameshift VNtyper calls High_Precision* (its - # integration test example_dfc3_GRCh38_fast runs the same reads on NCBI - # contig names). Reads realigned by VNtyper to GRCh38 chr1, UCSC names. - "positive.bam": ("data/remapped/bwa/hg38/example_dfc3_hg38_bwa.bam", - "5934330a8ac2144ac0cc6ae8c84d7d9262e90b28127417e4068f7d662b7c64bb"), - # Negative: no variant called (its integration test - # example_40cf_hg38_subset_fast_gdp_guard), mean VNTR depth about 148x. - "negative.bam": ("data/example_40cf_hg38_subset.bam", - "44aff8e3b2096b7ea3529989660052c61b29c8a26afd7f48de4988f7047d0652"), -} - - -def get(start, end): - """Bytes start..end (inclusive) of the archive, with retries.""" - for attempt in range(4): - try: - req = urllib.request.Request(URL, headers={"Range": f"bytes={start}-{end}"}) - with urllib.request.urlopen(req, timeout=120) as r: - if r.status != 206: - raise OSError(f"HTTP {r.status}, want 206 (a range)") - data = r.read() - if len(data) != end - start + 1: - raise OSError(f"got {len(data)} bytes, want {end - start + 1}") - return data - except OSError as e: - if attempt == 3: - raise - print(f"range {start}-{end}: {e}; again in {15 * (attempt + 1)} s", file=sys.stderr) - time.sleep(15 * (attempt + 1)) - raise AssertionError("unreachable") - - -def directory(): - """{path: (method, crc, compressed size, size, local header offset)}""" - tail = get(SIZE - 65536, SIZE - 1) - i = tail.rfind(b"PK\x05\x06") - if i < 0: - raise SystemExit("no end-of-central-directory record in the last 64 kB") - _, _, _, _, n, cd_size, cd_off, _ = struct.unpack("> 8 & 255)) $(($1 >> 16 & 255)) $(($1 >> 24 & 255)))" +} + +[ $# -ge 2 ] || { echo "usage: vntyper_testdata.sh OUTDIR NAME... (NAME of positive.bam, negative.bam)" >&2; exit 2; } +out=$1 +shift +for name in "$@"; do + member "$name" > /dev/null || { echo "vntyper_testdata.sh: unknown member ${name} (positive.bam, negative.bam)" >&2; exit 2; } +done +mkdir -p "$out" +for name in "$@"; do + read -r path off csize size crc want <&2; exit 1; } + # gzip member: magic, deflate, no flags, no time, no extra flags, unknown OS; + # then the raw deflate stream; then CRC-32 and size, both little-endian. + { printf '\037\213\010\000\000\000\000\000\000\377'; cat "${part}.raw"; le32 "$((0x$crc))"; le32 "$size"; } > "${part}.gz" + gzip -dc "${part}.gz" > "$part" || { echo "${path}: size or CRC-32 differs from the archive's directory" >&2; exit 1; } + got=$(sha256sum "$part" | cut -d' ' -f1) + [ "$got" = "$want" ] || { echo "${path}: sha256 ${got}, pinned ${want}" >&2; exit 1; } + rm -f "${part}.raw" "${part}.gz" + mv "$part" "${out}/${name}" + printf '%s\t%s\t%s\t%s\n' "$name" "$path" "$size" "$got" +done From 463305a90714986c75c043b356e29e5e72bf6912 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:08:09 +0100 Subject: [PATCH 02/76] feat(dev): og dev fixture and og dev evidence verbs og dev fixture comphet writes the compound-het fixture byte for byte (CPython's Mersenne Twister for the seeded contig); og dev fixture trim-pharmcat trims a PharmCAT report; og dev evidence clashes and hg002-gvcf replace the public-evidence workflow's inline Python, streaming, with the covered positions kept as merged spans. Parity cases against the Python, and a golden for the site past a contig's end. Bead: pgp-bs9u.39 --- rust/crates/og-dev/src/evidence.rs | 354 ++++++++++ rust/crates/og-dev/src/fixture.rs | 649 ++++++++++++++++++ rust/crates/og-dev/src/lib.rs | 10 +- tests/parity/cases.tsv | 10 + .../past-the-end/stdout | 1 + .../inputs/dev-evidence-clashes/chrA.tsv.gz | Bin 0 -> 58 bytes .../inputs/dev-evidence-clashes/chrB.tsv.gz | Bin 0 -> 45 bytes .../inputs/dev-evidence-clashes/chrZ.tsv.gz | Bin 0 -> 45 bytes .../inputs/dev-evidence-clashes/notes.tsv.gz | Bin 0 -> 37 bytes .../inputs/dev-evidence-clashes/raw.vcf.gz | Bin 0 -> 192 bytes .../inputs/dev-evidence-clashes/split.vcf.gz | Bin 0 -> 192 bytes .../sites-past-end.tsv | 4 + .../inputs/dev-evidence-hg002-gvcf/sites.tsv | 19 + .../truth-past-end.bed | 1 + .../inputs/dev-evidence-hg002-gvcf/truth.bed | 4 + .../dev-evidence-hg002-gvcf/truth.vcf.gz | Bin 0 -> 173 bytes tests/parity/oracle.ref | 4 + 17 files changed, 1055 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-dev/src/evidence.rs create mode 100644 rust/crates/og-dev/src/fixture.rs create mode 100644 tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout create mode 100644 tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz create mode 100644 tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz create mode 100644 tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz create mode 100644 tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz create mode 100644 tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz create mode 100644 tests/parity/inputs/dev-evidence-clashes/split.vcf.gz create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs new file mode 100644 index 000000000..c9b76d7fc --- /dev/null +++ b/rust/crates/og-dev/src/evidence.rs @@ -0,0 +1,354 @@ +//! `og dev evidence`: the measurements of `.github/workflows/public-evidence.yml` +//! on public data (GIAB's HG002), which were Python inside the workflow. +//! +//! - `og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv`: the +//! positions with two records or more, one of them an SNV, in the VCF as +//! GIAB writes it and split with `bcftools norm -m-`, on the chromosomes +//! of the archaic tables in DATA_DIR (`chr*.tsv.gz`), and how many rows of +//! those tables sit at such a position. OUT holds `raw_positions`, +//! `raw_on_table`, `split_positions` and `split_on_table`. +//! - `og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf`: +//! a gVCF of GIAB's genotypes plus a one-base reference block at every +//! panel site inside the benchmark regions that no GIAB record covers, +//! with the reference base from the FASTA (through its `.fai`). The +//! output is unsorted: the workflow sorts it with bcftools. +//! +//! Both stream their VCFs. hg002-gvcf keeps, per chromosome, the merged +//! spans the GIAB records cover instead of every covered position, and +//! reads one reference base per site with a positioned read. +//! +//! Exit 0 when written, 1 when an input cannot be read or parsed, 2 on a +//! usage error. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io::{BufRead, BufWriter, Write}; +use std::os::unix::fs::FileExt; +use std::path::Path; + +const USAGE: &str = "usage: og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv\n og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf\n"; + +/// Runs `og dev evidence ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev evidence: error: {msg}"); + 2 + }; + let rest = args.get(1..).unwrap_or(&[]); + let result = match args.first().map(String::as_str) { + Some("clashes") if rest.len() == 4 => clashes(&rest[0], &rest[1], &rest[2], &rest[3]), + Some("hg002-gvcf") if rest.len() == 5 => { + hg002_gvcf(&rest[0], &rest[1], &rest[2], &rest[3], &rest[4]).map(|(records, blocks)| { + let _ = writeln!( + out, + "gVCF: {records} GIAB records, {blocks} reference blocks at panel sites" + ); + }) + } + Some("clashes" | "hg002-gvcf") => return usage(err, "wrong number of arguments"), + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown evidence '{other}'")), + None => return usage(err, "the following arguments are required: evidence"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev evidence: {e}"); + 1 + } + } +} + +fn open(path: &str) -> Result, String> { + og_io::open(path).map_err(|e| format!("{path}: {e}")) +} + +/// Each line of `path` without its line end, to `f`. +fn lines(path: &str, mut f: impl FnMut(&[u8]) -> Result<(), String>) -> Result<(), String> { + let mut r = open(path)?; + let mut buf = Vec::new(); + loop { + buf.clear(); + let n = r + .read_until(b'\n', &mut buf) + .map_err(|e| format!("{path}: {e}"))?; + if n == 0 { + return Ok(()); + } + let line = buf.strip_suffix(b"\n").unwrap_or(&buf); + f(line.strip_suffix(b"\r").unwrap_or(line))?; + } +} + +fn pos(field: &[u8], path: &str) -> Result { + std::str::from_utf8(field) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + format!( + "{path}: position '{}' is not a number", + String::from_utf8_lossy(field) + ) + }) +} + +/// The positions of `path` on `chroms` with two records or more, one of +/// them an SNV. The VCF is sorted, so a position's records are adjacent. +fn clash_positions( + path: &str, + chroms: &BTreeSet>, +) -> Result, u64)>, String> { + let mut hit = BTreeSet::new(); + let (mut key, mut n, mut snvs): (Option<(Vec, u64)>, u32, u32) = (None, 0, 0); + lines(path, |line| { + if line.first() == Some(&b'#') { + return Ok(()); + } + let c: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if c.len() < 5 || !chroms.contains(c[0]) { + return Ok(()); + } + let k = (c[0].to_vec(), pos(c[1], path)?); + if key.as_ref() != Some(&k) { + key = Some(k.clone()); + (n, snvs) = (0, 0); + } + n += 1; + let one = |a: &[u8]| std::str::from_utf8(a).map_or(a.len(), |s| s.chars().count()) == 1; + snvs += u32::from(one(c[3]) && one(c[4])); + if n >= 2 && snvs >= 1 { + hit.insert(k); + } + Ok(()) + })?; + Ok(hit) +} + +fn clashes(raw: &str, split: &str, data: &str, out: &str) -> Result<(), String> { + let mut tables = BTreeMap::new(); + for e in fs::read_dir(data).map_err(|e| format!("{data}: {e}"))? { + let name = e + .map_err(|e| format!("{data}: {e}"))? + .file_name() + .into_string() + .unwrap_or_default(); + if let Some(chrom) = name + .strip_suffix(".tsv.gz") + .filter(|c| c.starts_with("chr")) + { + tables.insert(chrom.as_bytes().to_vec(), Path::new(data).join(&name)); + } + } + let chroms: BTreeSet> = tables.keys().cloned().collect(); + let sets = [ + ("raw", clash_positions(raw, &chroms)?), + ("split", clash_positions(split, &chroms)?), + ]; + let mut on_table = [0u64; 2]; + for (chrom, table) in &tables { + let want: Vec> = sets + .iter() + .map(|(_, s)| { + s.iter() + .filter(|(c, _)| c == chrom) + .map(|(_, p)| *p) + .collect() + }) + .collect(); + if want.iter().all(BTreeSet::is_empty) { + continue; + } + let path = table.to_string_lossy().into_owned(); + lines(&path, |line| { + let c: Vec<&[u8]> = line.splitn(3, |&b| b == b'\t').collect(); + if c[0] != chrom.as_slice() || c.len() < 2 { + return Ok(()); + } + let p = pos(c[1], &path)?; + for (k, w) in want.iter().enumerate() { + on_table[k] += u64::from(w.contains(&p)); + } + Ok(()) + })?; + } + let mut text = String::new(); + for (k, (name, s)) in sets.iter().enumerate() { + text += &format!( + "{name}_positions\t{}\n{name}_on_table\t{}\n", + s.len(), + on_table[k] + ); + } + fs::write(out, text).map_err(|e| format!("{out}: {e}")) +} + +/// One `.fai` row: length, byte offset, bases per line, bytes per line. +struct Fai { + len: u64, + offset: u64, + line_bases: u64, + line_bytes: u64, +} + +fn read_fai(path: &str) -> Result, Fai>, String> { + let mut fai = HashMap::new(); + lines(path, |line| { + let f: Vec<&[u8]> = line.split(|&b| b == b'\t').collect(); + let n = |i: usize| { + f.get(i) + .ok_or_else(|| format!("{path}: a row has fewer than 5 columns")) + .and_then(|x| pos(x, path)) + }; + let row = Fai { + len: n(1)?, + offset: n(2)?, + line_bases: n(3)?, + line_bytes: n(4)?, + }; + if row.line_bases == 0 { + return Err(format!("{path}: a row has 0 bases per line")); + } + fai.insert(f[0].to_vec(), row); + Ok(()) + })?; + Ok(fai) +} + +/// Sorted, merged half-open spans `[start, end)` of 1-based positions. +fn merge(mut spans: Vec<(u64, u64)>) -> Vec<(u64, u64)> { + spans.sort_unstable(); + let mut out: Vec<(u64, u64)> = Vec::with_capacity(spans.len()); + for (s, e) in spans { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + out +} + +/// Is `p` inside one of the sorted, disjoint spans? +fn inside(spans: &[(u64, u64)], p: u64) -> bool { + let i = spans.partition_point(|&(s, _)| s <= p); + i > 0 && p < spans[i - 1].1 +} + +fn hg002_gvcf( + truth: &str, + bed: &str, + reference: &str, + sites: &str, + out: &str, +) -> Result<(u64, u64), String> { + let fai = read_fai(&format!("{reference}.fai"))?; + let fasta = File::open(reference).map_err(|e| format!("{reference}: {e}"))?; + // The benchmark regions, BED 0-based: (start, end) per chromosome, + // sorted. A site p (1-based) is confident when the region with the + // largest start below p ends at p or later, as the Python's bisect did. + let mut conf: HashMap, Vec<(u64, u64)>> = HashMap::new(); + lines(bed, |line| { + let f: Vec<&[u8]> = line + .split(|b| b.is_ascii_whitespace()) + .filter(|x| !x.is_empty()) + .collect(); + if f.is_empty() { + return Ok(()); + } + if f.len() < 3 { + return Err(format!("{bed}: a row has fewer than 3 columns")); + } + conf.entry(f[0].to_vec()) + .or_default() + .push((pos(f[1], bed)?, pos(f[2], bed)?)); + Ok(()) + })?; + for v in conf.values_mut() { + v.sort_unstable(); + } + let confident = |c: &[u8], p: u64| { + conf.get(c).is_some_and(|v| { + let i = v.partition_point(|&(s, _)| s < p); + i > 0 && v[i - 1].1 >= p + }) + }; + + let file = File::create(out).map_err(|e| format!("{out}: {e}"))?; + let mut w = BufWriter::with_capacity(1 << 20, file); + let werr = |e: std::io::Error| format!("{out}: {e}"); + let mut covered: HashMap, Vec<(u64, u64)>> = HashMap::new(); + let mut records = 0u64; + lines(truth, |line| { + if line.starts_with(b"##") { + } else if line.first() == Some(&b'#') { + w.write_all(b"##INFO=\n##ALT=\n") + .map_err(werr)?; + } else { + let f: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if f.len() < 5 { + return Err(format!("{truth}: a record has fewer than 5 columns")); + } + let p = pos(f[1], truth)?; + covered + .entry(f[0].to_vec()) + .or_default() + .push((p, p + f[3].len() as u64)); + records += 1; + } + w.write_all(line) + .and_then(|()| w.write_all(b"\n")) + .map_err(werr) + })?; + let covered: HashMap, Vec<(u64, u64)>> = + covered.into_iter().map(|(c, v)| (c, merge(v))).collect(); + + let mut blocks = 0u64; + let mut first = true; + let mut base = [0u8; 1]; + lines(sites, |line| { + if std::mem::take(&mut first) { + return Ok(()); + } + let f: Vec<&[u8]> = line.splitn(4, |&b| b == b'\t').collect(); + if f.len() < 3 { + return Err(format!("{sites}: a row has fewer than 3 columns")); + } + let (c, p) = (f[1], pos(f[2], sites)?); + let Some(row) = fai.get(c) else { return Ok(()) }; + if p == 0 || p > row.len || covered.get(c).is_some_and(|v| inside(v, p)) || !confident(c, p) + { + return Ok(()); + } + let at = + row.offset + ((p - 1) / row.line_bases) * row.line_bytes + (p - 1) % row.line_bases; + fasta + .read_exact_at(&mut base, at) + .map_err(|e| format!("{reference}: {e}"))?; + let b = base[0].to_ascii_uppercase(); + if !b"ACGT".contains(&b) { + return Ok(()); + } + blocks += 1; + w.write_all(c) + .and_then(|()| writeln!(w, "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t0/0", b as char)) + .map_err(werr) + })?; + w.flush().map_err(werr)?; + Ok((records, blocks)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spans_merge_and_answer_inside() { + let m = merge(vec![(10, 12), (5, 6), (11, 15), (15, 16)]); + assert_eq!(m, [(5, 6), (10, 16)]); + assert!( + inside(&m, 5) && !inside(&m, 6) && inside(&m, 15) && !inside(&m, 16) && !inside(&m, 4) + ); + } +} diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs new file mode 100644 index 000000000..bb163c9f9 --- /dev/null +++ b/rust/crates/og-dev/src/fixture.rs @@ -0,0 +1,649 @@ +//! `og dev fixture`: writers of committed or generated test fixtures. +//! +//! - `og dev fixture comphet OUT_DIR [--swap]`: the compound-het phasing +//! fixture of `tests/e2e/comphet-phasing-1-reads.sh` (made-up reads on a +//! made-up contig). It replaces `tests/fixtures/comphet/make_fixture.py` +//! and writes the same bytes: the contig is Python's `random.Random(31)` +//! choosing among `ACGT`, so [`Mt19937`] is CPython's generator. +//! - `og dev fixture trim-pharmcat REPORT OUT [MAX_ANNOTATIONS]`: a PharmCAT +//! `report.json` cut down to the fields `og pgx` reads. It replaces +//! `tests/fixtures/pharmcat/trim_report.py` and writes the same bytes. +//! +//! Exit 0 when written, 1 when an input cannot be read or has the wrong +//! shape, 2 on a usage error. + +use crate::py_os_error; +use indexmap::IndexMap; +use pycompat::json::{self, Dump, Value}; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::Path; + +const USAGE: &str = "usage: og dev fixture comphet OUT_DIR [--swap]\n og dev fixture trim-pharmcat REPORT.json OUT.json [MAX_ANNOTATIONS]\n"; + +/// Runs `og dev fixture ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev fixture: error: {msg}"); + 2 + }; + let result = match args.first().map(String::as_str) { + Some("comphet") => { + let rest: Vec<&String> = args[1..].iter().filter(|a| *a != "--swap").collect(); + if rest.len() != 1 { + return usage(err, "comphet takes one OUT_DIR and an optional --swap"); + } + comphet::write(Path::new(rest[0]), args[1..].iter().any(|a| a == "--swap")) + } + Some("trim-pharmcat") => { + let rest = &args[1..]; + if !(2..=3).contains(&rest.len()) { + return usage( + err, + "trim-pharmcat takes REPORT.json OUT.json [MAX_ANNOTATIONS]", + ); + } + let max = match rest.get(2).map(|m| m.trim().parse::()) { + None => 4, + Some(Ok(n)) => n, + Some(Err(_)) => { + return usage( + err, + &format!("MAX_ANNOTATIONS '{}' is not a whole number", rest[2]), + ); + } + }; + trim_pharmcat(Path::new(&rest[0]), Path::new(&rest[1]), max) + } + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown fixture '{other}'")), + None => return usage(err, "the following arguments are required: fixture"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev fixture: {e}"); + 1 + } + } +} + +/// CPython's Mersenne Twister (`random.Random`), only as far as +/// `Random(int).choice(seq)` needs it. +pub struct Mt19937 { + state: [u32; 624], + index: usize, +} + +impl Mt19937 { + /// `random.Random(seed)` for a seed below 2^32: `init_by_array([seed])`. + pub fn new(seed: u32) -> Self { + let mut mt = Mt19937 { + state: [0; 624], + index: 624, + }; + mt.init_genrand(19_650_218); + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624.max(key.len()) { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_566_083_941)).wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + } + mt.state[0] = 0x8000_0000; + mt + } + + fn init_genrand(&mut self, s: u32) { + self.state[0] = s; + for i in 1..624 { + let prev = self.state[i - 1] ^ (self.state[i - 1] >> 30); + self.state[i] = 1_812_433_253u32.wrapping_mul(prev).wrapping_add(i as u32); + } + self.index = 624; + } + + /// `genrand_uint32`. + pub fn next_u32(&mut self) -> u32 { + if self.index >= 624 { + for k in 0..624 { + let y = (self.state[k] & 0x8000_0000) | (self.state[(k + 1) % 624] & 0x7fff_ffff); + let mag = if y & 1 == 1 { 0x9908_b0df } else { 0 }; + self.state[k] = self.state[(k + 397) % 624] ^ (y >> 1) ^ mag; + } + self.index = 0; + } + let mut y = self.state[self.index]; + self.index += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + /// `_randbelow(n)` for `0 < n < 2^32`: rejection sampling on + /// `getrandbits(n.bit_length())`. + pub fn below(&mut self, n: u32) -> u32 { + let k = 32 - n.leading_zeros(); + loop { + let r = self.next_u32() >> (32 - k); + if r < n { + return r; + } + } + } +} + +pub mod comphet { + //! The compound-het phasing fixture: sample CHPH on contig chrP (80 kb + //! of seeded random sequence), read pairs of 100 bp from 500 bp + //! fragments, and three pairs of heterozygous variants: + //! + //! - GENEC: SNV at 1000 and a 2-base deletion at 1003, both on copy 1 + //! (cis); single reads cover both. `--swap` moves the deletion to + //! copy 2 (trans). + //! - GENET: SNVs at 5000 and 5400, one on each copy (trans); only a read + //! and its mate cover both. + //! - GENEF: SNVs at 20000 and 70000, one on each copy, 50 kb apart: no + //! read pair covers both (unknown). + //! + //! Files: `reads.sam`, `vep.vcf` (with a VEP CSQ, for step 31) and + //! `comphet.vcf` (the pairs as slivar names them, for `og comphet + //! phase`). + + use super::Mt19937; + use std::fmt::Write as _; + use std::fs; + use std::path::Path; + + pub const SAMPLE: &str = "CHPH"; + pub const CONTIG: &str = "chrP"; + pub const LENGTH: usize = 80_000; + const READ: usize = 100; + const FRAG: usize = 500; + const FIELDS: &str = "Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|Existing_variation|MAX_AF|gnomADe_AF|gnomADg_AF|CLIN_SIG"; + + /// One variant: 1-based position, alleles, consequence and the copy + /// (0 or 1) that carries it. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct Variant { + pub gene: &'static str, + pub pos: usize, + pub alt_ref: String, + pub alt: String, + pub consequence: &'static str, + pub copy: u8, + } + + /// The contig: `random.Random(31)` choosing among `ACGT` 80,000 times. + pub fn reference() -> Vec { + let mut rng = Mt19937::new(31); + (0..LENGTH) + .map(|_| b"ACGT"[rng.below(4) as usize]) + .collect() + } + + /// The base before `b` in `ACGT`, wrapping (`A` gives `T`). + fn other(b: u8) -> u8 { + let i = b"ACGT" + .iter() + .position(|&x| x == b) + .expect("a base of ACGT"); + b"ACGT"[(i + 3) % 4] + } + + pub fn variants(seq: &[u8], swap: bool) -> Vec { + let s = |a: usize, b: usize| String::from_utf8_lossy(&seq[a..b]).into_owned(); + let snv = |gene, pos: usize, consequence, copy| Variant { + gene, + pos, + alt_ref: s(pos - 1, pos), + alt: (other(seq[pos - 1]) as char).to_string(), + consequence, + copy, + }; + vec![ + snv("GENEC", 1000, "stop_gained", 0), + Variant { + gene: "GENEC", + pos: 1003, + alt_ref: s(1002, 1005), + alt: s(1002, 1003), + consequence: "frameshift_variant", + copy: u8::from(swap), + }, + snv("GENET", 5000, "stop_gained", 0), + snv("GENET", 5400, "stop_gained", 1), + snv("GENEF", 20000, "stop_gained", 0), + snv("GENEF", 70000, "stop_gained", 1), + ] + } + + /// The sequence and CIGAR of a read of `READ` bases from 0-based + /// `start` on one copy. + fn read_on(seq: &[u8], vs: &[Variant], copy: u8, start: usize) -> (String, String) { + let mine = || vs.iter().filter(|v| v.copy == copy); + let snv_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() == v.alt.len() && v.pos - 1 == p) + .map(|v| v.alt.as_bytes()[0]) + }; + // A deletion is keyed on the first deleted base. + let del_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() > v.alt.len() && v.pos + v.alt.len() - 1 == p) + .map(|v| v.alt_ref.len() - v.alt.len()) + }; + let (mut out, mut ops, mut p) = + (Vec::with_capacity(READ), Vec::<(u8, usize)>::new(), start); + while out.len() < READ { + if let Some(n) = del_at(p) { + ops.push((b'D', n)); + p += n; + continue; + } + out.push(snv_at(p).unwrap_or(seq[p])); + ops.push((b'M', 1)); + p += 1; + } + let mut cigar = String::new(); + let mut i = 0; + while i < ops.len() { + let (op, mut n) = ops[i]; + let mut j = i + 1; + while j < ops.len() && ops[j].0 == op { + n += ops[j].1; + j += 1; + } + let _ = write!(cigar, "{n}{}", op as char); + i = j; + } + (String::from_utf8_lossy(&out).into_owned(), cigar) + } + + pub fn sam(seq: &[u8], vs: &[Variant]) -> String { + let mut s = format!( + "@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:{CONTIG}\tLN:{LENGTH}\n@RG\tID:{SAMPLE}\tSM:{SAMPLE}\n" + ); + // Fragment starts (0-based) around each pair: read 1 covers the + // first position, its mate the second one when 400 bp apart. + let starts: [(&str, std::iter::StepBy>); 4] = [ + ("c", (920..991).step_by(7)), + ("t", (4905..4996).step_by(10)), + ("f", (19905..19996).step_by(10)), + ("g", (69905..69996).step_by(10)), + ]; + let q = "I".repeat(READ); + for (region, rs) in starts { + for copy in 0..2u8 { + for st in rs.clone() { + let m = st + FRAG - READ; + let (s1, c1) = read_on(seq, vs, copy, st); + let (s2, c2) = read_on(seq, vs, copy, m); + let name = format!("{region}{copy}_{st}"); + let _ = writeln!( + s, + "{name}\t99\t{CONTIG}\t{}\t60\t{c1}\t=\t{}\t{FRAG}\t{s1}\t{q}\tRG:Z:{SAMPLE}", + st + 1, + m + 1 + ); + let _ = writeln!( + s, + "{name}\t147\t{CONTIG}\t{}\t60\t{c2}\t=\t{}\t-{FRAG}\t{s2}\t{q}\tRG:Z:{SAMPLE}", + m + 1, + st + 1 + ); + } + } + } + s + } + + fn vcf_head(info: &str) -> String { + format!( + "##fileformat=VCFv4.2\n##contig=\n##FILTER=\n{info}\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n" + ) + } + + pub fn vep_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head(&format!( + "##INFO=\n" + )); + for v in vs { + let (g, p) = (v.gene, v.pos); + let _ = writeln!( + out, + "{CONTIG}\t{p}\t.\t{}\t{}\t50\tPASS\tCSQ={}|{}|HIGH|{g}|ENSG{g}|Transcript|ENST{p}|protein_coding|YES|NM_{p}.1|rs{p}|0.0001|0.0001|0.0001|\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.alt_ref, v.alt, v.alt, v.consequence + ); + } + out + } + + pub fn comphet_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head( + "##INFO=\n", + ); + for (i, v) in vs.iter().enumerate() { + let mate = if i % 2 == 0 { &vs[i + 1] } else { &vs[i - 1] }; + let pid = match v.gene { + "GENEC" => 1, + "GENET" => 2, + _ => 3, + }; + let _ = writeln!( + out, + "{CONTIG}\t{}\t.\t{}\t{}\t50\tPASS\tslivar_comphet={SAMPLE}/{}/{pid}/{CONTIG}/{}/{}/{}\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.pos, v.alt_ref, v.alt, v.gene, mate.pos, mate.alt_ref, mate.alt + ); + } + out + } + + /// Writes the three files into `dir`, made if missing. + pub fn write(dir: &Path, swap: bool) -> Result<(), String> { + let seq = reference(); + let vs = variants(&seq, swap); + fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for (name, text) in [ + ("reads.sam", sam(&seq, &vs)), + ("vep.vcf", vep_vcf(&vs)), + ("comphet.vcf", comphet_vcf(&vs)), + ] { + let path = dir.join(name); + fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))?; + } + Ok(()) + } +} + +/// `x.get(k)`: the value, or JSON null. +fn get(x: &IndexMap, k: &str) -> Value { + x.get(k).cloned().unwrap_or(Value::Null) +} + +/// Python's truth of a JSON value. +fn truthy(v: &Value) -> bool { + pycompat::value::truthy(v) +} + +/// `v or []` as a list; a truthy value that is not a list is an error. +fn list<'a>(v: Option<&'a Value>, what: &str) -> Result<&'a [Value], String> { + match v { + Some(Value::Array(a)) => Ok(a), + Some(x) if truthy(x) => Err(format!("{what} is not a list")), + _ => Ok(&[]), + } +} + +fn object<'a>(v: &'a Value, what: &str) -> Result<&'a IndexMap, String> { + match v { + Value::Object(o) => Ok(o), + _ => Err(format!("{what} is not an object")), + } +} + +/// The keys of `x` among `keys`, in that order. +fn pick(x: &IndexMap, keys: &[&str]) -> IndexMap { + keys.iter() + .filter_map(|k| x.get(*k).map(|v| ((*k).to_owned(), v.clone()))) + .collect() +} + +/// One listed diplotype with the fields `og pgx` reads. +fn diplotype(x: &Value) -> Result { + let Value::Object(x) = x else { + return Ok(x.clone()); + }; + let mut out = pick( + x, + &[ + "gene", + "label", + "phenotypes", + "activityScore", + "outsidePhenotype", + ], + ); + for a in ["allele1", "allele2"] { + let Some(v) = x.get(a) else { continue }; + let kept = match v { + Value::Object(o) if !o.is_empty() => { + Value::Object(pick(o, &["gene", "name", "function"])) + } + v if !truthy(v) => v.clone(), + _ => return Err(format!("diplotype {a} is not an object")), + }; + out.insert(a.to_owned(), kept); + } + Ok(Value::Object(out)) +} + +/// `items[:n]` with Python's slice rule for a negative `n`. +fn head(items: &[Value], n: i64) -> &[Value] { + let len = items.len() as i64; + let end = if n < 0 { (len + n).max(0) } else { n.min(len) }; + &items[..end as usize] +} + +/// The trimmed report: every gene with its listed diplotypes and related +/// drugs, and for each drug of each guidance source up to `max` of the +/// annotations PharmCAT matched, with only the fields `og pgx` reads. +pub fn trim(d: &Value, max: i64) -> Result { + let d = object(d, "the report")?; + let mut genes = IndexMap::new(); + for (g, v) in object(d.get("genes").unwrap_or(&Value::Null), "genes")? { + let v = object(v, &format!("gene {g}"))?; + let mut gene = pick(v, &["geneSymbol", "chr", "phased", "callSource"]); + let mut related = Vec::new(); + for r in list(v.get("relatedDrugs"), "relatedDrugs")? { + let r = object(r, "a related drug")?; + let name = r.get("name").ok_or("a related drug has no name")?.clone(); + related.push(Value::Object(IndexMap::from([ + ("name".to_owned(), name), + ("id".to_owned(), get(r, "id")), + ]))); + } + gene.insert("relatedDrugs".to_owned(), Value::Array(related)); + // sourceDiplotypes is what og pgx reads; recommendationDiplotypes + // only when sourceDiplotypes is absent, so it is kept only then. + let k = if v.get("sourceDiplotypes").is_some_and(truthy) { + "sourceDiplotypes" + } else { + "recommendationDiplotypes" + }; + if v.contains_key(k) { + let dips = list(v.get(k), k)? + .iter() + .map(diplotype) + .collect::>()?; + gene.insert(k.to_owned(), Value::Array(dips)); + } + genes.insert(g.clone(), Value::Object(gene)); + } + let mut drugs: IndexMap = IndexMap::new(); + for (source, by) in object(d.get("drugs").unwrap_or(&Value::Null), "drugs")? { + for (name, rep) in object(by, &format!("drugs of {source}"))? { + let rep = object(rep, &format!("drug {name}"))?; + let mut gls = Vec::new(); + for gl in list(rep.get("guidelines"), "guidelines")? { + let gl = object(gl, "a guideline")?; + let mut anns = Vec::new(); + for a in head(list(gl.get("annotations"), "annotations")?, max) { + let a = object(a, "an annotation")?; + let mut gts = Vec::new(); + for gt in list(a.get("genotypes"), "genotypes")? { + let gt = object(gt, "a genotype")?; + let dips = list(gt.get("diplotypes"), "diplotypes")? + .iter() + .filter_map(|x| match x { + Value::Object(x) => Some(Value::Object(IndexMap::from([ + ("gene".to_owned(), get(x, "gene")), + ("label".to_owned(), get(x, "label")), + ]))), + _ => None, + }) + .collect(); + gts.push(Value::Object(IndexMap::from([( + "diplotypes".to_owned(), + Value::Array(dips), + )]))); + } + anns.push(Value::Object(IndexMap::from([ + ("phenotypes".to_owned(), get(a, "phenotypes")), + ("classification".to_owned(), get(a, "classification")), + ( + "drugRecommendation".to_owned(), + get(a, "drugRecommendation"), + ), + ("genotypes".to_owned(), Value::Array(gts)), + ]))); + } + if !anns.is_empty() { + gls.push(Value::Object(IndexMap::from([ + ("name".to_owned(), get(gl, "name")), + ("source".to_owned(), get(gl, "source")), + ("annotations".to_owned(), Value::Array(anns)), + ]))); + } + } + if !gls.is_empty() { + let entry = drugs + .entry(source.clone()) + .or_insert_with(|| Value::Object(IndexMap::new())); + if let Value::Object(e) = entry { + e.insert( + name.clone(), + Value::Object(IndexMap::from([ + ("name".to_owned(), get(rep, "name")), + ("id".to_owned(), get(rep, "id")), + ("guidelines".to_owned(), Value::Array(gls)), + ])), + ); + } + } + } + } + let mut out = pick(d, &["title", "timestamp", "pharmcatVersion", "dataVersion"]); + out.insert("genes".to_owned(), Value::Object(genes)); + out.insert("drugs".to_owned(), Value::Object(drugs)); + let mut note = String::new(); + let _ = write!( + note, + "Trimmed by og dev fixture trim-pharmcat: every gene, and up to {max} matched annotations per drug guideline, with only the fields og pgx reads." + ); + out.insert("_fixture_note".to_owned(), Value::Str(note)); + Ok(Value::Object(out)) +} + +fn trim_pharmcat(src: &Path, dst: &Path, max: i64) -> Result<(), String> { + let text = fs::read(src).map_err(|e| py_os_error(&e, &src.to_string_lossy()))?; + let text = String::from_utf8(text).map_err(|_| format!("{}: not UTF-8", src.display()))?; + let d = json::loads(&text).map_err(|e| format!("{}: {e}", src.display()))?; + let out = trim(&d, max)?; + let opts = Dump { + separators: Some((",", ":")), + ..Dump::default() + }; + let text = json::dumps(&out, &opts).map_err(|e| e.to_string())?; + fs::write(dst, text).map_err(|e| py_os_error(&e, &dst.to_string_lossy())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mt19937_matches_cpython() { + // random.Random(31): [r.getrandbits(32) for _ in range(3)] and the + // first ten of "".join(r.choice("ACGT") ...) from a fresh seed. + let mut r = Mt19937::new(31); + let first: Vec = (0..3).map(|_| r.next_u32()).collect(); + assert_eq!(first, [52_734_659, 2_016_822_975, 482_749_413]); + assert_eq!(&comphet::reference()[..10], b"ATATCACACC"); + } + + #[test] + fn swap_moves_only_the_deletion() { + let seq = comphet::reference(); + let (a, b) = ( + comphet::variants(&seq, false), + comphet::variants(&seq, true), + ); + assert_eq!(a[1].copy, 0); + assert_eq!(b[1].copy, 1); + assert_eq!(a[0], b[0]); + assert_eq!(a[2..], b[2..]); + } + + #[test] + fn a_read_over_the_deletion_has_a_d_in_its_cigar() { + let seq = comphet::reference(); + let vs = comphet::variants(&seq, false); + let sam = comphet::sam(&seq, &vs); + let line = sam + .lines() + .find(|l| l.starts_with("c0_920\t99")) + .expect("read c0_920"); + assert!(line.contains("\t83M2D17M\t"), "{line}"); + let line = sam + .lines() + .find(|l| l.starts_with("c1_920\t99")) + .expect("read c1_920"); + assert!(line.contains("\t100M\t"), "{line}"); + } + + #[test] + fn trim_keeps_the_fields_og_pgx_reads() { + let d = json::loads( + r#"{"title":"t","extra":1,"genes":{"G":{"geneSymbol":"G","x":1,"relatedDrugs":[{"name":"d"}], + "sourceDiplotypes":[],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1","y":2},"allele2":null,"z":3},"str"]}}, + "drugs":{"CPIC":{"d":{"name":"d","guidelines":[{"name":"g","annotations":[{"genotypes":[{"diplotypes":[{"gene":"G"},5]}]},{},{}]},{"annotations":[]}]}}}}"#, + ) + .expect("json"); + let out = trim(&d, 2).expect("trim"); + let text = json::dumps( + &out, + &Dump { + separators: Some((",", ":")), + ..Dump::default() + }, + ) + .expect("dumps"); + assert!(text.starts_with(r#"{"title":"t","genes":{"G":{"geneSymbol":"G","relatedDrugs":[{"name":"d","id":null}],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1"},"allele2":null},"str"]}}"#), "{text}"); + assert!(text.contains(r#""annotations":[{"phenotypes":null,"classification":null,"drugRecommendation":null,"genotypes":[{"diplotypes":[{"gene":"G","label":null}]}]},{"phenotypes":null"#), "{text}"); + assert_eq!(text.matches("drugRecommendation").count(), 2, "{text}"); + // A negative count is Python's slice: all but the last. + let out = trim(&d, -2).expect("trim"); + let text = json::dumps(&out, &Dump::default()).expect("dumps"); + assert_eq!(text.matches("drugRecommendation").count(), 1, "{text}"); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 55c93fdf0..918237714 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -12,8 +12,14 @@ //! the report carries the report-only error strategy. //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev fixture` ([`fixture`]): writers of test fixtures (the +//! compound-het reads, a trimmed PharmCAT report). +//! - `og dev evidence` ([`evidence`]): the measurements of the public +//! evidence workflow on GIAB's HG002. pub mod contract; +pub mod evidence; +pub mod fixture; pub mod parity; pub mod remnants; pub mod report_only; @@ -22,7 +28,7 @@ mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -57,6 +63,8 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("fixture") => run(fixture::cli, &args[1..]), + Some("evidence") => run(evidence::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 795bb7945..692b0f540 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -874,3 +874,13 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +dev-fixture-comphet plain - out out/reads.sam,out/vep.vcf,out/comphet.vcf bytes PYTHON_IMAGE +dev-fixture-comphet swap - out --swap out/reads.sam,out/vep.vcf,out/comphet.vcf bytes PYTHON_IMAGE +dev-fixture-trim-pharmcat report-3.2.0 gen:pgx-fixtures.sh report-3.2.0.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat report-3.4.0 gen:pgx-fixtures.sh report-3.4.0.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat pharmcat-docs-example gen:pgx-fixtures.sh pharmcat-docs-example.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat one-annotation gen:pgx-fixtures.sh report-3.4.0.json out.json 1 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat all-but-last gen:pgx-fixtures.sh pharmcat-docs-example.json out.json -1 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-evidence-clashes synthetic dev-evidence-clashes/raw.vcf.gz,dev-evidence-clashes/split.vcf.gz,dev-evidence-clashes/chrA.tsv.gz,dev-evidence-clashes/chrB.tsv.gz,dev-evidence-clashes/chrZ.tsv.gz,dev-evidence-clashes/notes.tsv.gz raw.vcf.gz split.vcf.gz . affected.tsv affected.tsv bytes PYTHON_IMAGE +dev-evidence-hg002-gvcf synthetic dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites.tsv truth.vcf.gz truth.bed ref.fasta sites.tsv out.vcf out.vcf bytes PYTHON_IMAGE +dev-evidence-hg002-gvcf past-the-end dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth-past-end.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites-past-end.tsv truth.vcf.gz truth-past-end.bed ref.fasta sites-past-end.tsv out.vcf out.vcf bytes PYTHON_IMAGE diff --git a/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout new file mode 100644 index 000000000..233285d28 --- /dev/null +++ b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout @@ -0,0 +1 @@ +gVCF: 4 GIAB records, 1 reference blocks at panel sites diff --git a/tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..5dc766e8d450338fc6372800f67e9c3fc0ecd581 GIT binary patch literal 58 zcmb2|=3oE;rvKjOG)|p=s&hjBS;-kq&ofWF&S?}qF?nXvV92A$Q#_-CwI?x!VG1)t Lh}*wOc0e5fk|7n8 literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..2ff2824f7570b5fb663c33b4e8d667bcf2f245de GIT binary patch literal 45 ycmb2|=3oE;rvKjOG)|p=s&hjBS;-kq&ofWF&S?}qGI?t9fQg}K^O3odKt%wPJ`z9x literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..987e91c2ce6814ccb80b83d2cf042cfeb10c57d8 GIT binary patch literal 45 ycmb2|=3oE;rvKjOG)|p=s&hjBS;-kq&ofWF&S`W#GI?V1fQjMq66L>=Kt%wRY7%Gw literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..035133bbf944bc169e37959bc9f666d9342e05d7 GIT binary patch literal 37 qcmb2|=3oE;rvKjOG)|p=s&hjBnb$duqGu*gm>9BVi~JA-N&*1&xD8MM literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz b/tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..386cc04141ddaee0666a6957d26e90295f313b42 GIT binary patch literal 192 zcmV;x06+g9iwFP!00002|9#Ct3xYrp1>p1RZ+Hz`E$NcMw7Dv1Te@`*Q4`8Qq~D(# z1wm`aneV+B%c`1dS>3zax%W?8JQlV)v8>|W_D$`t;;mm?I+|X~w(c*!$JRf%`QTh5 zk20BJFLT6OWPt!B3Wp;l5OJb{4Dn7#2$`j%5QU7n6a4^l*a)HEG3+VN^`yXN&X>{< uF!OSl34A(MciFJuqxCnFkvH)4b@iWhz{DKRi^IQa=sp1(Jr?Yk0RR9Ton2f2 literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-clashes/split.vcf.gz b/tests/parity/inputs/dev-evidence-clashes/split.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..4a4e80c1e428b4fbd7e40229992c3d58421c08b6 GIT binary patch literal 192 zcmV;x06+g9iwFP!00002|9#EP3W7is1>p7ZHQa`cnzYGa932%jOQ-fB{(&io^!zzd z5Tq{e{mwmCRm-L=%6@p>=x@R19Qxvzu8~ancEjXPPgc^0RRAO6I%5E literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv new file mode 100644 index 000000000..bb13404a5 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv @@ -0,0 +1,4 @@ +id chrom pos +s1 chrB 30 +s2 chrB 31 +s3 chrB 35 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv new file mode 100644 index 000000000..b6a517815 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv @@ -0,0 +1,19 @@ +id chrom pos ref +s0 chrA 3 X +s1 chrA 11 X +s2 chrA 12 X +s3 chrA 13 X +s4 chrA 15 X +s5 chrA 16 X +s6 chrA 19 X +s7 chrA 21 X +s8 chrA 27 X +s9 chrA 45 X +s10 chrA 46 X +s11 chrB 3 X +s12 chrB 8 X +s13 chrB 10 X +s14 chrB 30 X +s15 chrC 1 X +s16 chrD 4 X +s17 chrA 1 X diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed new file mode 100644 index 000000000..d6806d6ff --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed @@ -0,0 +1 @@ +chrB 0 40 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed new file mode 100644 index 000000000..6a2ed2e11 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed @@ -0,0 +1,4 @@ +chrA 18 45 +chrA 0 15 +chrB 0 30 +chrC 0 10 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..f13c00115d4f489b2140634ee513945ddb31d804 GIT binary patch literal 173 zcmV;e08;-SiwFP!00002|8>pF3W7iw1>p7ZHD((&YC?-97)M71O&h27A>P0gM0);| zXk{(W`Th$bb=Oz*I6WUTy9@a`n6^PkISzBzuv6@_vYq&@uZCuBnM=K{r~^~M%hj3 literal 0 HcmV?d00001 diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 226fc54b2..ca861a0ed 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -58,3 +58,7 @@ mito-copies 954ab367004d63ea639521982299b4976927ccfd file:bin/collect_summary.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +dev-fixture-comphet fe6b67e09cef81c396c2277472df10407c56d7ac file:tests/fixtures/comphet/make_fixture.py python3 dev fixture comphet +dev-fixture-trim-pharmcat fe6b67e09cef81c396c2277472df10407c56d7ac file:tests/fixtures/pharmcat/trim_report.py python3 dev fixture trim-pharmcat +dev-evidence-clashes fe6b67e09cef81c396c2277472df10407c56d7ac file:.github/workflows/public-evidence.yml sh -c 'd=$(mktemp -d) && awk -v n=1 "/<<.PY.\$/ {k++; if (k == n) {on = 1; next}} on && /^ *PY\$/ {exit} on {sub(/^ /, \"\"); print}" "$0" > "$d/p.py" && exec python3 -I "$d/p.py" "$@"' dev evidence clashes +dev-evidence-hg002-gvcf fe6b67e09cef81c396c2277472df10407c56d7ac file:.github/workflows/public-evidence.yml sh -c 'd=$(mktemp -d) && awk -v n=2 "/<<.PY.\$/ {k++; if (k == n) {on = 1; next}} on && /^ *PY\$/ {exit} on {sub(/^ /, \"\"); print}" "$0" > "$d/p.py" && exec python3 -I "$d/p.py" "$@"' dev evidence hg002-gvcf From 1ec05547d9a382e213ebf9dfca50022418dac0a1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:09:59 +0100 Subject: [PATCH 03/76] feat(og): og dev scan, versions and pgs verbs for the CI scanners The personal-data scan, the image scan, the coupled-version check, the run-versions check, the URL expander of check-links.sh and the PGS Catalog answer readers, ported from the Python inside five scripts/ci scripts. Bead: pgp-bs9u.37 --- rust/crates/og-dev/Cargo.toml | 2 +- rust/crates/og-dev/src/images.rs | 406 ++++++++++++++++++ rust/crates/og-dev/src/lib.rs | 42 +- rust/crates/og-dev/src/lookaround.rs | 103 +++++ rust/crates/og-dev/src/personal_data.rs | 538 ++++++++++++++++++++++++ rust/crates/og-dev/src/pgs.rs | 320 ++++++++++++++ rust/crates/og-dev/src/versions.rs | 263 ++++++++++++ rust/crates/og-dev/src/versions_env.rs | 115 +++++ rust/og/src/main.rs | 3 + 9 files changed, 1785 insertions(+), 7 deletions(-) create mode 100644 rust/crates/og-dev/src/images.rs create mode 100644 rust/crates/og-dev/src/lookaround.rs create mode 100644 rust/crates/og-dev/src/personal_data.rs create mode 100644 rust/crates/og-dev/src/pgs.rs create mode 100644 rust/crates/og-dev/src/versions.rs create mode 100644 rust/crates/og-dev/src/versions_env.rs diff --git a/rust/crates/og-dev/Cargo.toml b/rust/crates/og-dev/Cargo.toml index 20bcd7e4c..108491221 100644 --- a/rust/crates/og-dev/Cargo.toml +++ b/rust/crates/og-dev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-dev" -description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only" +description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only, scans and version checks" version.workspace = true edition.workspace = true license.workspace = true diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs new file mode 100644 index 000000000..91ded0e7f --- /dev/null +++ b/rust/crates/og-dev/src/images.rs @@ -0,0 +1,406 @@ +//! `og dev scan images` and `og dev versions coupled`: the two checks of +//! `scripts/ci/check-images.sh` that were Python. The script keeps the +//! lists (which paths are scanned, which files and single literals are +//! exempt, with the reason) and passes them in. +//! +//! ```text +//! og dev scan images --root DIR --exempt-files TEXT --exempt-pairs TEXT < FILES +//! og dev versions coupled --env versions.env +//! ``` +//! +//! `scan images` reads the files to scan (one path relative to DIR per +//! line) on stdin and prints one `FAIL: : names the image +//! ; ...` per literal image, then `scanned N files`. An image is +//! `registry/name:tag`, `name@sha256:...`, or a Docker Hub official name +//! (those `versions.env` uses and the usual base images) with a tag. A line +//! of `--exempt-pairs` (`file image reason`) allows one occurrence of that +//! image in that file; listing a pair twice allows two. +//! +//! `versions coupled` checks that `MINIMAP2_SAMTOOLS_IMAGE`, the one image +//! of the Nextflow alignment task, names in its `versions.env` comment the +//! minimap2 and samtools releases of `MINIMAP2_IMAGE` and `SAMTOOLS_IMAGE`. +//! +//! Exit 0 clean, 1 on a finding, 2 on a usage error or a `versions.env` +//! the strict reader refuses. +//! +//! Where it differs from the Python on purpose: an exempt pair whose file +//! no longer names its image as often as the list allows fails +//! (`FAIL: no longer names ; drop its line ...`), so an +//! exemption cannot outlive its reason; and `versions.env` is read strictly +//! ([`crate::versions_env`]). + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read, Write}; +use std::path::Path; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::lookaround::Rule; +use crate::versions_env::{self, Pin}; + +const USAGE: &str = "usage: og dev scan images --root DIR --exempt-files TEXT --exempt-pairs TEXT < FILES\n og dev versions coupled --env versions.env\n"; + +/// Docker Hub official images that `versions.env` may not use today. +const BASES: [&str; 15] = [ + "ubuntu", + "debian", + "alpine", + "busybox", + "centos", + "rockylinux", + "fedora", + "python", + "node", + "perl", + "r-base", + "openjdk", + "eclipse-temurin", + "golang", + "rust", +]; + +/// Any tag: a version, or a word such as latest or stable. +const TAG: &str = r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-f]{64})"; +/// What may not stand right before an image, nor right after it. +const AHEAD: &str = r"(?:[^A-Za-z0-9_/-]|$)"; + +fn lead(c: char) -> bool { + c.is_ascii_alphanumeric() || "_./:@$-".contains(c) +} + +static URL: LazyLock = + LazyLock::new(|| Regex::new(r"[a-z][a-z0-9+.-]*://[^\s\x1c-\x1f]+").expect("a valid pattern")); +/// A path to a file, such as modules/local/vep/main.nf:23, is not an image. +static FILE: LazyLock = LazyLock::new(|| { + Regex::new(r"\.(nf|sh|md|py|config|ya?ml|json|txt|tsv|csv|vcf|gz|html|css|js|R|r):[0-9]+(-[0-9]+)?$") + .expect("a valid pattern") +}); + +fn bare_names(pins: &[Pin]) -> BTreeSet { + let mut bare: BTreeSet = BASES.iter().map(|s| (*s).to_owned()).collect(); + for p in pins.iter().filter(|p| p.name.ends_with("_IMAGE")) { + if let Some(i) = p.value.find([':', '@']) { + let name = &p.value[..i]; + if !name.is_empty() && !name.contains('/') { + bare.insert(name.to_owned()); + } + } + } + bare +} + +/// The two patterns: `ns/name:tag`, and an official name with a tag. +fn patterns(bare: &BTreeSet) -> [Rule; 2] { + let names: Vec = bare.iter().map(|b| regex::escape(b)).collect(); + [ + Rule::new( + &format!(r"([a-z][a-z0-9._-]*/[a-z][a-z0-9._/-]*{TAG}){AHEAD}"), + Some(lead), + ) + .expect("a valid pattern"), + Rule::new(&format!(r"((?:{}){TAG}){AHEAD}", names.join("|")), Some(lead)) + .expect("a valid pattern"), + ] +} + +/// The text of a file as Python's text mode reads it: strict UTF-8, any +/// newline turned into `\n`. None for a file that is not UTF-8 or not a +/// readable file. +fn read_text(path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + let text = String::from_utf8(bytes).ok()?; + Some(text.replace("\r\n", "\n").replace('\r', "\n")) +} + +/// The literal images of one file, with their line numbers; exempt +/// occurrences are taken off `left`. +fn scan_file( + rel: &str, + text: &str, + rules: &[Rule; 2], + left: &mut BTreeMap<(String, String), usize>, + out: &mut dyn Write, +) -> bool { + let mut bad = false; + for (i, raw) in text.split('\n').enumerate() { + // docker://image:tag (a step image in a workflow) is an image, not a URL. + let line = raw.replace("docker://", " "); + let line = URL.replace_all(&line, " "); + for r in rules { + for (s, e) in r.find_iter(&line) { + // A full stop ends the sentence, not the tag. + let image = line[s..e].trim_end_matches('.'); + if FILE.is_match(image) || image.starts_with("example/") { + continue; // example/ is the placeholder namespace of the self-tests + } + if let Some(n) = left.get_mut(&(rel.to_owned(), image.to_owned())) + && *n > 0 + { + *n -= 1; + continue; + } + let _ = writeln!( + out, + "FAIL: {rel}:{} names the image {image}; use its *_IMAGE variable from versions.env", + i + 1 + ); + bad = true; + } + } + } + bad +} + +struct ScanArgs { + root: String, + exempt_files: Vec, + pairs: BTreeMap<(String, String), usize>, +} + +fn parse_scan(args: &[String]) -> Result { + let (mut root, mut files, mut pairs) = (None, None, None); + let mut it = args.iter(); + while let Some(a) = it.next() { + let v = it.next().ok_or_else(|| format!("{a} needs a value")); + match a.as_str() { + "--root" => root = Some(v?.clone()), + "--exempt-files" => files = Some(v?.clone()), + "--exempt-pairs" => pairs = Some(v?.clone()), + s => return Err(format!("unknown argument {s}")), + } + } + let exempt_files = files + .ok_or("--exempt-files is required")? + .lines() + .filter_map(|l| l.split_whitespace().next().map(str::to_owned)) + .collect(); + let mut counted = BTreeMap::new(); + for l in pairs.ok_or("--exempt-pairs is required")?.lines() { + let w: Vec<&str> = l.split_whitespace().take(2).collect(); + if let [f, image] = w[..] { + *counted.entry((f.to_owned(), image.to_owned())).or_insert(0) += 1; + } else if !w.is_empty() { + return Err(format!("an exempt pair needs a file and an image: {l}")); + } + } + Ok(ScanArgs { + root: root.ok_or("--root is required")?, + exempt_files, + pairs: counted, + }) +} + +fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { + let files: Vec<&str> = list.lines().filter(|f| !f.is_empty()).collect(); + if files.is_empty() { + let _ = writeln!(out, "FAIL: no file to scan"); + return Ok(false); + } + let root = Path::new(&a.root); + let pins = versions_env::read(&root.join("versions.env"))?; + let rules = patterns(&bare_names(&pins)); + let mut left = a.pairs.clone(); + let mut bad = false; + for f in &files { + let skipped = a + .exempt_files + .iter() + .any(|s| *f == s || (s.ends_with('/') && f.starts_with(s.as_str()))); + if skipped { + continue; + } + if let Some(text) = read_text(&root.join(f)) { + bad |= scan_file(f, &text, &rules, &mut left, out); + } + } + for ((f, image), n) in &left { + if *n > 0 { + let _ = writeln!( + out, + "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", + if *n > 1 { format!(" {n} more times") } else { String::new() } + ); + bad = true; + } + } + let _ = writeln!(out, "scanned {} files", files.len()); + Ok(!bad) +} + +/// `og dev scan images ARGS`. +pub fn cli_scan(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let a = match parse_scan(args) { + Ok(a) => a, + Err(e) => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og dev scan images: error: {e}"); + return 2; + } + }; + let mut list = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut list) { + let _ = writeln!(err, "ERROR: cannot read the file list: {e}"); + return 2; + } + match scan(&a, &list, out) { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +/// The release in an image's tag: `quay.io/x/samtools:1.24--h0_0` is 1.24. +fn release(value: &str) -> Option<&str> { + let tag = value.rsplit('/').next().unwrap_or(value); + let (_, t) = tag.split_once(':')?; + Some(t.split("--").next().unwrap_or(t)) +} + +static HELD: LazyLock = LazyLock::new(|| { + Regex::new(r"\b(minimap2|samtools)=([0-9][0-9A-Za-z.]*)").expect("a valid pattern") +}); + +/// The check on parsed pins; prints one line per tool. +pub fn coupled(pins: &[Pin], out: &mut dyn Write) -> bool { + let Some(image) = versions_env::get(pins, "MINIMAP2_SAMTOOLS_IMAGE") else { + let _ = writeln!(out, "FAIL: MINIMAP2_SAMTOOLS_IMAGE is not set in versions.env"); + return false; + }; + let mut held: BTreeMap<&str, &str> = BTreeMap::new(); + for c in HELD.captures_iter(&image.rest) { + let (tool, v) = (c.get(1).map_or("", |m| m.as_str()), c.get(2).map_or("", |m| m.as_str())); + held.insert(tool, v.trim_end_matches('.')); + } + let mut ok = true; + for (tool, var) in [("minimap2", "MINIMAP2_IMAGE"), ("samtools", "SAMTOOLS_IMAGE")] { + let want = versions_env::get(pins, var).and_then(|p| release(&p.value)); + let got = held.get(tool).copied(); + match (want, got) { + (Some(w), Some(g)) if !w.is_empty() && w == g => { + let _ = writeln!( + out, + "OK: MINIMAP2_SAMTOOLS_IMAGE holds {tool} {g}, the release of {var}" + ); + } + _ => { + let want = want.filter(|w| !w.is_empty()); + let _ = writeln!( + out, + "FAIL: MINIMAP2_SAMTOOLS_IMAGE says {tool}={} in its versions.env comment, but {var} is {}: pin a BioContainers mulled image with {tool} {} and name it in that comment (docs/updating.md, Held pins)", + got.unwrap_or("(nothing)"), + want.unwrap_or("(unset)"), + want.unwrap_or("?") + ); + ok = false; + } + } + } + ok +} + +/// `og dev versions coupled --env FILE`. +pub fn cli_coupled(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let [flag, path] = args else { + let _ = write!(err, "{USAGE}"); + return 2; + }; + if flag != "--env" { + let _ = write!(err, "{USAGE}"); + return 2; + } + match versions_env::read(Path::new(path)) { + Ok(pins) => u8::from(!coupled(&pins, out)), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rules() -> [Rule; 2] { + let pins = versions_env::parse("PYTHON_IMAGE=\"python:3.15.2\"\n").unwrap(); + patterns(&bare_names(&pins)) + } + + fn found(line: &str) -> Vec { + let mut left = BTreeMap::new(); + let mut out = Vec::new(); + scan_file("f", line, &rules(), &mut left, &mut out); + String::from_utf8(out) + .unwrap() + .lines() + .map(|l| l.split(" names the image ").nth(1).unwrap().split(';').next().unwrap().to_owned()) + .collect() + } + + // Planted images are split by "~", which is dropped, so this file + // names none of them whole. + fn p(s: &str) -> String { + s.replace('~', "") + } + + #[test] + fn literals_are_found_as_python_finds_them() { + assert_eq!(found(&p("docker run staphb/bcftools:~1.21 x")), [p("staphb/bcftools:~1.21")]); + assert_eq!(found(&p("runs staphb/samtools:~1.19.")), [p("staphb/samtools:~1.19")]); + assert_eq!(found(&p("// container python:~3.12")), [p("python:~3.12")]); + assert_eq!(found(&p("uses: docker://alpine:~3.8")), [p("alpine:~3.8")]); + // A tag followed by a slash backs off to the part before a dot. + assert_eq!(found(&p("x staphb/samtools:~1.2/x")), [p("staphb/samtools:~1")]); + for clean in [ + "see https://quay.io/repository/biocontainers/samtools:1.0", + "modules/local/vep/main.nf:23", + "example/bcftools:1.0", + "$VAR/name:tag", + "python3 x.py", + ] { + assert!(found(clean).is_empty(), "{clean}: {:?}", found(clean)); + } + } + + #[test] + fn a_stale_exemption_fails() { + let dir = std::env::temp_dir().join(format!("og-dev-images-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("versions.env"), "A_IMAGE=\"x/y:1\"\n").unwrap(); + std::fs::write(dir.join("a.md"), p("uses quay.io/x/y:~2\n")).unwrap(); + let a = ScanArgs { + root: dir.to_string_lossy().into_owned(), + exempt_files: vec![], + pairs: BTreeMap::from([ + (("a.md".into(), p("quay.io/x/y:~2")), 1), + (("a.md".into(), p("quay.io/x/y:~3")), 1), + ]), + }; + let mut out = Vec::new(); + let ok = scan(&a, "a.md\n", &mut out).unwrap(); + let out = String::from_utf8(out).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert!(!ok); + assert!( + out.contains(&p("FAIL: a.md no longer names quay.io/x/y:~3; drop its line")), + "{out}" + ); + assert!(!out.contains(&p("y:~2;")), "{out}"); + } + + #[test] + fn coupled_releases() { + let env = "MINIMAP2_IMAGE=\"q/minimap2:2.31--h1_0\"\nSAMTOOLS_IMAGE=\"q/samtools:1.24--h2_0\"\nMINIMAP2_SAMTOOLS_IMAGE=\"q/mulled:abc\" # minimap2=2.31,samtools=1.24.\n"; + let mut out = Vec::new(); + assert!(coupled(&versions_env::parse(env).unwrap(), &mut out)); + let bumped = env.replace("samtools:1.24", "samtools:9.99"); + let mut out = Vec::new(); + assert!(!coupled(&versions_env::parse(&bumped).unwrap(), &mut out)); + let out = String::from_utf8(out).unwrap(); + assert!(out.contains("FAIL: MINIMAP2_SAMTOOLS_IMAGE says samtools=1.24 in its versions.env comment, but SAMTOOLS_IMAGE is 9.99:"), "{out}"); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 55c93fdf0..3ca2ae5c4 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -12,17 +12,33 @@ //! the report carries the report-only error strategy. //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev scan personal-data` ([`personal_data`]): no tracked file holds +//! a home path, a private address or a genotype. +//! - `og dev scan images`, `og dev versions coupled` ([`images`]): no file +//! names an image outside `versions.env`; the alignment task's image holds +//! the releases of step 02's two images. +//! - `og dev versions check-run`, `og dev versions expand` ([`versions`]): a +//! run used the images `conf/containers.config` names; download URLs with +//! their `versions.env` variables filled. +//! - `og dev pgs` ([`pgs`]): the PGS Catalog's answers against +//! `assets/pgs_scores.tsv`. pub mod contract; +pub mod images; +mod lookaround; pub mod parity; +pub mod personal_data; +pub mod pgs; pub mod remnants; pub mod report_only; pub mod schema; +pub mod versions; +pub mod versions_env; mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -49,6 +65,12 @@ fn run(f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, args: &[String]) code } +fn unknown(verb: &str) -> u8 { + eprint!("{USAGE}"); + eprintln!("og dev: error: unknown verb '{verb}'"); + 2 +} + /// Runs `og dev ARGS`; returns the exit code. pub fn main(args: &[String]) -> u8 { match args.first().map(String::as_str) { @@ -57,15 +79,23 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("scan") => match args.get(1).map(String::as_str) { + Some("personal-data") => run(personal_data::cli, &args[2..]), + Some("images") => run(images::cli_scan, &args[2..]), + _ => unknown(args.get(1).map_or("scan", String::as_str)), + }, + Some("versions") => match args.get(1).map(String::as_str) { + Some("check-run") => run(versions::cli_check_run, &args[2..]), + Some("expand") => run(versions::cli_expand, &args[2..]), + Some("coupled") => run(images::cli_coupled, &args[2..]), + _ => unknown(args.get(1).map_or("versions", String::as_str)), + }, + Some("pgs") => run(pgs::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 } - Some(other) => { - eprint!("{USAGE}"); - eprintln!("og dev: error: unknown verb '{other}'"); - 2 - } + Some(other) => unknown(other), None => { eprint!("{USAGE}"); eprintln!("og dev: error: the following arguments are required: verb"); diff --git a/rust/crates/og-dev/src/lookaround.rs b/rust/crates/og-dev/src/lookaround.rs new file mode 100644 index 000000000..d40d83b0d --- /dev/null +++ b/rust/crates/og-dev/src/lookaround.rs @@ -0,0 +1,103 @@ +//! Python's one-character look-behind and look-ahead on the `regex` crate, +//! which has neither. +//! +//! Every look-around of the scanners is a single character class: "not +//! after one of these" before the match and "not followed by one of these" +//! after it. A [`Rule`] keeps the matched text in capture group 1 and +//! writes the look-ahead as a consumed `(?:[^...]|$)` after it, so the +//! engine still finds, at each start, the match Python's backtracking would +//! pick. The look-behind is the character before group 1, checked in code; +//! when it fails, the search starts again one character later, as Python +//! tries the next start. The next match is searched from the end of group +//! 1, so the character the look-ahead read is not used up. + +use regex::Regex; + +/// A pattern with its look-behind. +pub struct Rule { + re: Regex, + /// True for a character the match must not follow. + not_after: Option bool>, +} + +impl Rule { + /// `pattern` holds the match in group 1, or is the match when it has no + /// group. + pub fn new(pattern: &str, not_after: Option bool>) -> Result { + Ok(Rule { + re: Regex::new(pattern)?, + not_after, + }) + } + + /// The spans of `re.finditer` over `line`, in order. + pub fn find_iter<'a>(&'a self, line: &'a str) -> impl Iterator + 'a { + let mut pos = 0; + std::iter::from_fn(move || { + while pos <= line.len() { + let caps = self.re.captures_at(line, pos)?; + let m = caps.get(1).or_else(|| caps.get(0))?; + let (start, end) = (m.start(), m.end()); + let refused = self.not_after.is_some_and(|bad| { + line[..start].chars().next_back().is_some_and(bad) + }); + if refused { + pos = next_char(line, start); + continue; + } + pos = if end > start { + end + } else { + next_char(line, end) + }; + return Some((start, end)); + } + None + }) + } + + /// `re.search(line) is not None`. + pub fn is_match(&self, line: &str) -> bool { + self.find_iter(line).next().is_some() + } +} + +/// The byte offset of the character after `i`, or one past the end. +fn next_char(s: &str, i: usize) -> usize { + s[i..].chars().next().map_or(s.len() + 1, |c| i + c.len_utf8()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digit_or_dot(c: char) -> bool { + c.is_ascii_digit() || c == '.' + } + + #[test] + fn behind_and_ahead_as_python() { + // Python: re.finditer(r'(?(r: &Rule, s: &'a str) -> Vec<&'a str> { + r.find_iter(s).map(|(a, b)| &s[a..b]).collect() + } + assert_eq!(spans(&r, "17/19"), ["17/19"]); + assert_eq!(spans(&r, "1.17/19 x"), Vec::<&str>::new()); + assert_eq!(spans(&r, "17/19/3"), ["19/3"]); + assert_eq!(spans(&r, "a 17/19 and 2/3"), ["17/19", "2/3"]); + // The look-ahead character is not used up: two matches side by side. + let w = Rule::new(r"(ab)(?:[^b]|$)", None).unwrap(); + assert_eq!(w.find_iter("abab").count(), 2); + } + + #[test] + fn a_shorter_match_where_the_longest_fails_ahead() { + // Python finds "a/b:1" in "a/b:1.2/x": the tag backs off to "1", + // which a "." follows. + let r = Rule::new(r"([a-z]+/[a-z]+:[0-9][0-9.]*)(?:[^0-9a-z/]|$)", None).unwrap(); + let s = "a/b:1.2/x"; + let got: Vec<_> = r.find_iter(s).map(|(a, b)| &s[a..b]).collect(); + assert_eq!(got, ["a/b:1"]); + } +} diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs new file mode 100644 index 000000000..a10289f56 --- /dev/null +++ b/rust/crates/og-dev/src/personal_data.rs @@ -0,0 +1,538 @@ +//! `og dev scan personal-data`: fail when a tracked text file holds +//! something that looks like personal data. It replaces the Python inside +//! `scripts/ci/check-personal-data.sh`, which keeps the lists of documented +//! examples (`--allow`) and skipped files (`--skip`) and passes them in. +//! +//! ```text +//! og dev scan personal-data --root DIR --allow TEXT --skip TEXT +//! [--patterns FILE] [--cached] [--] [FILE...] +//! ``` +//! +//! With no FILE, every file `git ls-files` lists under DIR. `--cached` +//! reads each file's staged blob (one `git cat-file --batch` process) +//! instead of the working copy. `--patterns` adds one regular expression +//! per line (kind `own`). +//! +//! Each line of a text file (no NUL byte in its first 8 KiB) is matched +//! against the rules below; a match inside an allowed example of that file +//! is skipped, the rest of the line still counts. Output: one +//! `FAIL :: ` per rule and line, then +//! `scanned N text files`. Exit 0 clean, 1 on a finding or when no text +//! file was scanned, 2 on a usage error or a file or pattern that cannot be +//! read. +//! +//! Where it differs from the Python on purpose: a scan that read no text +//! file fails (`FAIL: no text file was scanned`) instead of passing, and a +//! file is placed under DIR after resolving symbolic links in both, so +//! DIR given as `/tmp/x` still finds `/private/tmp/x/CLAUDE.md`'s examples. + +use std::collections::HashMap; +use std::ffi::OsStr; +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::LazyLock; + +use pycompat::mtime::fnmatch; +use pycompat::num::py_strip; + +use crate::lookaround::Rule; + +const USAGE: &str = "usage: og dev scan personal-data --root DIR --allow TEXT --skip TEXT [--patterns FILE] [--cached] [--] [FILE...]\n"; + +/// This file's own path: it holds every allowed text, so all of them apply. +const SELF: &str = "scripts/ci/check-personal-data.sh"; + +/// How many leading bytes are searched for a NUL to call a file binary. +const BINARY_PROBE: usize = 8192; + +fn not_word(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' +} + +fn digit_or_dot(c: char) -> bool { + c.is_ascii_digit() || c == '.' +} + +fn digit_dot_slash(c: char) -> bool { + c.is_ascii_digit() || c == '.' || c == '/' +} + +/// `[A-Za-z]` under Python's `(?i)`, which also folds to the long s and +/// the Kelvin sign. +fn letter_ci(c: char) -> bool { + c.is_ascii_alphabetic() || c == '\u{17F}' || c == '\u{212A}' +} + +const STR_GENES: &str = "HTT|FMR1|C9orf72|C9ORF72|ATXN[0-9]+|DMPK|FXN|CNBP|RFC1|PABPN1|CACNA1A|TCF4|AR|PPP2R2B|TBP|JPH3|NOP56|ATN1|CSTB|AFF2|FMR2|NOTCH2NLC|DAB1|BEAN1|STARD7|YEATS2"; +const OCTET: &str = "(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"; +/// Python's `\s` also holds the four separators U+001C to U+001F. +const SPACE: &str = r"\s\x1c-\x1f"; + +/// A rule: its kind, its pattern, and a pattern the line must also match. +struct Check { + kind: &'static str, + rule: Rule, + cond: Option<&'static [Rule]>, +} + +fn rule(pattern: &str, not_after: Option bool>) -> Rule { + Rule::new(pattern, not_after).expect("a valid built-in pattern") +} + +/// A sample label, which may carry a number after a separator: sample7, +/// sample_01, donor-3; or a public sample id such as NA12878 or HG002. +static LABEL: LazyLock<[Rule; 2]> = LazyLock::new(|| { + [ + rule( + r"(?i)((?:sample|patient|subject|proband|participant|individual|donor)s?(?:[_-]?[0-9]+)?)(?:[^A-Za-z]|$)", + Some(letter_ci), + ), + rule(r"\b(?:NA|HG)[0-9]{3,5}\b", None), + ] +}); + +static REPCN: LazyLock<[Rule; 1]> = LazyLock::new(|| [rule(r"\bREPCN\b", None)]); + +/// The built-in rules, in the order they are reported. Each pattern marks +/// only the leaked value, so an allowed example covers just its span. +fn builtin() -> Vec { + let gt = r"\(?[ACGTDI][/|;]?[ACGTDI]\)?"; + vec![ + Check { + kind: "home", + rule: rule( + r"/home/[A-Za-z]|/Users/[A-Za-z]|/mnt/use[r]\b|[A-Za-z]:\\\\?Users\\\\?[A-Za-z]", + Some(not_word), + ), + cond: None, + }, + Check { + kind: "address", + rule: rule( + &format!( + r"((?:10\.{OCTET}|172\.(?:1[6-9]|2[0-9]|3[01])|192\.168|100\.(?:6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7]))\.{OCTET}\.{OCTET})(?:$|\.$|\.[^0-9]|[^0-9.])" + ), + Some(digit_or_dot), + ), + cond: None, + }, + Check { + kind: "address", + rule: rule(r"[A-Za-z0-9-]\.ts\.net\b", None), + cond: None, + }, + Check { + kind: "diplotype", + rule: rule(r"\*[0-9]+[A-Za-z0-9.]*/\*[0-9]+", None), + cond: Some(&*LABEL), + }, + Check { + kind: "rsid", + rule: rule( + &format!(r#"(\brs[0-9]+\b[{SPACE}:=,;|`'"\[\]-]{{1,6}}{gt})(?:[^A-Za-z0-9]|$)"#), + None, + ), + cond: None, + }, + Check { + kind: "rsid", + rule: rule(r"\brs[0-9]+\t[0-9XYMT]{1,2}\t[0-9]+\t[ACGTDI-]{1,2}\b", None), + cond: None, + }, + Check { + kind: "repeat", + rule: rule(r"([0-9]{1,4}/[0-9]{1,4})(?:[^0-9/]|$)", Some(digit_dot_slash)), + cond: Some(&*REPCN), + }, + // Python: \b(?:GENES)\b[^/\n]{0,40}?(?, + skip: Vec, + patterns: Option, + cached: bool, + files: Vec, +} + +fn parse(args: &[String]) -> Result { + let (mut root, mut allow, mut skip, mut patterns, mut cached) = (None, None, None, None, false); + let mut files = Vec::new(); + let mut it = args.iter(); + while let Some(a) = it.next() { + let mut value = |name: &str| { + it.next() + .cloned() + .ok_or_else(|| format!("{name} needs a value")) + }; + match a.as_str() { + "--root" => root = Some(value("--root")?), + "--allow" => allow = Some(value("--allow")?), + "--skip" => skip = Some(value("--skip")?), + "--patterns" => patterns = Some(value("--patterns")?), + "--cached" => cached = true, + "--" => { + files.extend(it.by_ref().map(PathBuf::from)); + break; + } + s if s.starts_with('-') => return Err(format!("unknown option {s}")), + s => files.push(PathBuf::from(s)), + } + } + let root = root.ok_or("--root is required")?; + let allow = allow + .ok_or("--allow is required")? + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| { + let l = py_strip(l); + let (path, text) = l.split_once(' ').unwrap_or((l, "")); + (path.to_owned(), py_strip(text).to_owned()) + }) + .collect(); + let skip = skip + .ok_or("--skip is required")? + .lines() + .filter_map(|l| l.split_whitespace().next().map(str::to_owned)) + .collect(); + Ok(Args { + root: PathBuf::from(root), + allow, + skip, + patterns: patterns.filter(|p| !p.is_empty()), + cached, + files, + }) +} + +/// The user's own patterns, one per line; `#` lines and blank lines skipped. +fn own_rules(path: &str) -> Result, String> { + let text = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?; + let mut out = Vec::new(); + for l in text.lines() { + let p = py_strip(l); + if p.is_empty() || l.starts_with('#') { + continue; + } + let rule = Rule::new(p, None).map_err(|e| format!("{path}: bad pattern {p}: {e}"))?; + out.push(Check { + kind: "own", + rule, + cond: None, + }); + } + Ok(out) +} + +/// `path` relative to `root` when it lies below it (both with symbolic +/// links resolved), else `path` as given. +fn relative(path: &Path, root: &Path) -> String { + let abs = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().map_or_else(|_| path.to_path_buf(), |d| d.join(path)) + }; + // The folder is resolved, not the file: a tracked symbolic link keeps + // its own name. + let placed = match (abs.parent(), abs.file_name()) { + (Some(dir), Some(name)) => dir + .canonicalize() + .map_or_else(|_| abs.clone(), |d| d.join(name)), + _ => abs.clone(), + }; + match placed.strip_prefix(root) { + Ok(rel) if !rel.as_os_str().is_empty() => rel.to_string_lossy().into_owned(), + _ => path.to_string_lossy().into_owned(), + } +} + +fn git_files(root: &Path) -> Result, String> { + let out = Command::new("git") + .arg("-C") + .arg(root) + .args(["ls-files", "-z"]) + .stderr(Stdio::inherit()) + .output() + .map_err(|e| format!("cannot run git: {e}"))?; + if !out.status.success() { + return Err(format!("git ls-files failed in {}", root.display())); + } + Ok(out + .stdout + .split(|&b| b == 0) + .filter(|f| !f.is_empty()) + .map(|f| root.join(OsStr::from_bytes(f))) + .collect()) +} + +/// The staged blob of each path (relative to `root`), from one +/// `git cat-file --batch`. A path with no blob in the index is absent. +fn staged(root: &Path, rels: &[String]) -> Result>, String> { + let wanted: Vec<&String> = rels.iter().filter(|r| !r.starts_with('/') && !r.contains('\n')).collect(); + let mut blobs = HashMap::new(); + if wanted.is_empty() { + return Ok(blobs); + } + let mut child = Command::new("git") + .arg("-C") + .arg(root) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| format!("cannot run git: {e}"))?; + let mut stdin = child.stdin.take().ok_or("no stdin for git")?; + let input: Vec = wanted + .iter() + .flat_map(|r| format!(":{r}\n").into_bytes()) + .collect(); + let writer = std::thread::spawn(move || stdin.write_all(&input)); + let mut out = BufReader::new(child.stdout.take().ok_or("no stdout for git")?); + for rel in &wanted { + let mut head = String::new(); + if out.read_line(&mut head).map_err(|e| e.to_string())? == 0 { + return Err("git cat-file stopped early".into()); + } + let fields: Vec<&str> = head.trim_end().split(' ').collect(); + let [_, kind, size] = fields[..] else { + continue; // " missing" or "ambiguous" + }; + let size: usize = size.parse().map_err(|_| format!("git cat-file: {head}"))?; + let mut data = vec![0; size + 1]; + out.read_exact(&mut data).map_err(|e| e.to_string())?; + data.pop(); + if kind == "blob" { + blobs.insert((*rel).clone(), data); + } + } + let _ = writer.join(); + let _ = child.wait(); + Ok(blobs) +} + +/// One line of the report: kind padded as Python's `%-9s`, the stripped +/// line cut at 160 characters. +fn report(out: &mut dyn Write, kind: &str, rel: &str, n: usize, line: &str) { + let text: String = py_strip(line).chars().take(160).collect(); + let _ = writeln!(out, "FAIL {kind:<9} {rel}:{n}: {text}"); +} + +/// Scans one file's text; true when something was reported. +fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, data: &[u8]) -> bool { + let mut bad = false; + let text = String::from_utf8_lossy(data); + for (i, line) in text.split('\n').enumerate() { + let spans: Vec<(usize, usize)> = texts + .iter() + .flat_map(|t| line.match_indices(*t).map(|(a, s)| (a, a + s.len()))) + .collect(); + for c in rules { + if c.cond.is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) { + continue; + } + let leaked = c + .rule + .find_iter(line) + .find(|&(s, e)| !spans.iter().any(|&(a, b)| a <= s && e <= b)); + if leaked.is_some() { + report(out, c.kind, rel, i + 1, line); + bad = true; + } + } + } + bad +} + +fn run(a: &Args, out: &mut dyn Write) -> Result { + let root = a + .root + .canonicalize() + .map_err(|e| format!("--root {}: {e}", a.root.display()))?; + let mut rules = builtin(); + if let Some(p) = &a.patterns { + rules.extend(own_rules(p)?); + } + let files = if a.files.is_empty() { + git_files(&root)? + } else { + a.files.clone() + }; + if files.is_empty() { + let _ = writeln!(out, "FAIL: no file to scan"); + return Ok(false); + } + let todo: Vec<(&PathBuf, String)> = files + .iter() + .map(|f| (f, relative(f, &root))) + .filter(|(_, rel)| !a.skip.iter().any(|g| fnmatch(rel, g))) + .collect(); + let blobs = if a.cached { + staged(&root, &todo.iter().map(|(_, r)| r.clone()).collect::>())? + } else { + HashMap::new() + }; + let (mut bad, mut scanned) = (false, 0usize); + for (path, rel) in &todo { + let data = if a.cached { + match blobs.get(rel) { + Some(d) => std::borrow::Cow::Borrowed(d.as_slice()), + None => continue, + } + } else { + match std::fs::read(path) { + Ok(d) => std::borrow::Cow::Owned(d), + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::IsADirectory + ) => + { + continue; + } + Err(e) => return Err(format!("cannot read {}: {e}", path.display())), + } + }; + if data[..data.len().min(BINARY_PROBE)].contains(&0) { + continue; + } + scanned += 1; + let texts: Vec<&str> = a + .allow + .iter() + .filter(|(p, _)| rel == SELF || fnmatch(rel, p)) + .map(|(_, t)| t.as_str()) + .collect(); + bad |= scan_text(out, &rules, &texts, rel, &data); + } + let _ = writeln!(out, "scanned {scanned} text files"); + if scanned == 0 { + let _ = writeln!(out, "FAIL: no text file was scanned"); + return Ok(false); + } + Ok(!bad) +} + +/// `og dev scan personal-data ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let a = match parse(args) { + Ok(a) => a, + Err(e) => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og dev scan personal-data: error: {e}"); + return 2; + } + }; + match run(&a, out) { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(line: &str) -> Vec<&'static str> { + let rules = builtin(); + let mut out = Vec::new(); + for c in &rules { + if c.cond.is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) { + continue; + } + if c.rule.is_match(line) { + out.push(c.kind); + } + } + out + } + + // Every planted value is split by "~", which is dropped, so this file + // holds none of them whole. + fn p(s: &str) -> String { + s.replace('~', "") + } + + #[test] + fn each_kind_is_caught() { + for (line, kind) in [ + ("cp results.html /home/~alice/genome/", "home"), + ("C:\\~Users\\bob", "home"), + ("scp out.vcf.gz root@192.~168.7.20:/data/", "address"), + ("ssh nas.tail1234.ts.~net", "address"), + ("| sample7 | CYP2D6 | *1/~*4 |", "diplotype"), + ("sample_01 CYP2D6 *1/~*4", "diplotype"), + ("HG0~02 *1/~*2", "diplotype"), + ("rs4244285~ AG", "rsid"), + ("rs123: (C;~T)", "rsid"), + ("rs4477212\t1\t82154\t~AA", "rsid"), + ("HTT CAG repeats: 17/~19", "repeat"), + ("GT:REPCN 0/1:17/~19", "repeat"), + ] { + let line = p(line); + assert!(kinds(&line).contains(&kind), "{line}: {:?}", kinds(&line)); + } + } + + #[test] + fn edges_as_python_reads_them() { + for line in [ + "- `*1/*4` -- Intermediate metabolizer", + "samtools 1.21, GATK 4.6.2.0", + "https://example.org/home/page", + "CYP2C19*2 is rs4244285 (G>A)", + // An address with a fifth part, or a version that looks like one. + "v10.1.2.3.4", + "1.10.1.2.3", + "10.1.2.256", + // The word must stand alone: "samples" is a label, "sampler" is not. + "sampler *1/*4", + // A repeat count with another digit or slash on either side. + "REPCN 1.17/19", + "REPCN 17/19/3", + "/home/1x", + "my/home/user", + ] { + assert!(kinds(line).is_empty(), "{line}: {:?}", kinds(line)); + } + // Python takes the label "sample" out of "sample7a" by backing off + // the number: the line still counts as labelled. + assert_eq!(kinds(&p("sample7a *1/~*4")), ["diplotype"]); + assert_eq!(kinds(&p("samples: *1/~*4")), ["diplotype"]); + assert_eq!(kinds(&p("10.1.2.3.")), ["address"]); + assert_eq!(kinds(&p("FMR1 has 29/~31")), ["repeat"]); + } + + #[test] + fn an_allowed_example_covers_only_its_own_text() { + let rules = builtin(); + let ex = "(`/mnt/user/`, `/home/username/`, etc.)"; + let mut out = Vec::new(); + assert!(!scan_text(&mut out, &rules, &[ex], "x.md", ex.as_bytes())); + let line = p(&format!("{ex} see /home/~bob/x")); + assert!(scan_text(&mut out, &rules, &[ex], "x.md", line.as_bytes())); + let got = String::from_utf8(out).unwrap(); + assert!(got.starts_with("FAIL home x.md:1: "), "{got}"); + } +} diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs new file mode 100644 index 000000000..3bc7acd3c --- /dev/null +++ b/rust/crates/og-dev/src/pgs.rs @@ -0,0 +1,320 @@ +//! `og dev pgs`: the readers of `scripts/ci/check-pgs-labels.sh` that were +//! Python. The script still asks the PGS Catalog with curl; these verbs +//! read its answer on stdin and never touch the network. +//! +//! ```text +//! og dev pgs trait --id PGS000018 < score.json +//! og dev pgs label-match LABEL TRAIT +//! og dev pgs metric-match --id PGS --ppm PPM --ancestry GROUP --effect "OR 1.61" < performance.json +//! ``` +//! +//! - `trait` prints the score's `trait_reported`, or exits 1 when the +//! answer is not JSON, is for another id (the API answers `{}` with HTTP +//! 200 for an unknown one) or has no trait. +//! - `label-match` exits 0 when every word of LABEL is a word of TRAIT, +//! after lower-casing and dropping apostrophes and punctuation. +//! - `metric-match` prints what the catalog says of the evaluation +//! (`score, ancestries, effects`) and exits 0 when it belongs to the +//! score, was measured only in samples of that ancestry group and reports +//! that odds or hazard ratio; 1 when it does not; 2 when the answer is not +//! an evaluation with that id. +//! +//! Where it differs from the Python on purpose: a JSON `true` is never the +//! number 1, so `"estimate": true` no longer matches `OR 1`; and an answer +//! whose samples or effects are not the objects and strings the catalog +//! sends is exit 2 (not an evaluation) instead of a traceback read as a +//! mismatch. + +use std::io::{Read, Write}; +use std::sync::LazyLock; + +use pycompat::json::{Value, loads}; +use pycompat::value::{py_str, truthy}; +use regex::Regex; + +const USAGE: &str = "usage: og dev pgs trait --id PGS_ID < ANSWER\n og dev pgs label-match LABEL TRAIT\n og dev pgs metric-match --id PGS_ID --ppm PPM_ID --ancestry GROUP --effect 'OR|HR N' < ANSWER\n"; + +/// The PGS Catalog `ancestry_broad` terms of each 1000 Genomes group +/// pgsc_calc reports (its ancestry_distribution uses the same codes). +const GROUPS: [(&str, &[&str]); 5] = [ + ("EUR", &["European"]), + ( + "AFR", + &[ + "African unspecified", + "African American or Afro-Caribbean", + "Sub-Saharan African", + ], + ), + ("AMR", &["Hispanic or Latin American"]), + ("EAS", &["East Asian"]), + ("SAS", &["South Asian"]), +]; + +fn group(term: &str) -> Option<&'static str> { + GROUPS + .iter() + .find(|(_, terms)| terms.contains(&term)) + .map(|(g, _)| *g) +} + +fn field<'a>(d: &'a Value, key: &str) -> Option<&'a Value> { + match d { + Value::Object(o) => o.get(key), + _ => None, + } +} + +fn is_str(v: Option<&Value>, want: &str) -> bool { + matches!(v, Some(Value::Str(s)) if s == want) +} + +/// The trait of score `id` in `body`, or None. +pub fn trait_of(body: &str, id: &str) -> Option { + let d = loads(body).ok()?; + if !matches!(d, Value::Object(_)) || !is_str(field(&d, "id"), id) { + return None; + } + match field(&d, "trait_reported") { + Some(Value::Str(t)) => { + let t = pycompat::num::py_strip(t); + (!t.is_empty()).then(|| t.to_owned()) + } + _ => None, + } +} + +static NOT_WORD: LazyLock = + LazyLock::new(|| Regex::new("[^a-z0-9]+").expect("a valid pattern")); + +fn words(s: &str) -> std::collections::BTreeSet { + let s = pycompat::text::lower(s).replace(['\u{2019}', '\''], ""); + NOT_WORD + .replace_all(&s, " ") + .split_whitespace() + .map(str::to_owned) + .collect() +} + +/// True when every word of `label` appears in `trait_`. +pub fn label_matches(label: &str, trait_: &str) -> bool { + let (l, t) = (words(label), words(trait_)); + !l.is_empty() && l.is_subset(&t) +} + +static EFFECT: LazyLock = + LazyLock::new(|| Regex::new(r"^(OR|HR) ([0-9]+(?:\.[0-9]+)?)$").expect("a valid pattern")); + +/// What the catalog says of an evaluation, and whether it matches. Err is +/// the reason the answer is not an evaluation with that id. +pub fn metric(body: &str, id: &str, ppm: &str, anc: &str, eff: &str) -> Result<(String, bool), String> { + let d = loads(body).map_err(|_| "(not JSON)".to_owned())?; + if !matches!(d, Value::Object(_)) || !is_str(field(&d, "id"), ppm) { + return Err("(no such evaluation)".into()); + } + let odd = || "(not an evaluation the check can read)".to_owned(); + let empty = Value::Null; + let or_empty = |v: Option<&'_ Value>| -> Value { + v.filter(|v| truthy(v)).cloned().unwrap_or(Value::Null) + }; + let samples = match or_empty(field(&or_empty(field(&d, "sampleset")), "samples")) { + Value::Null => Vec::new(), + Value::Array(a) => a, + _ => return Err(odd()), + }; + let mut terms = Vec::new(); + for s in &samples { + if !matches!(s, Value::Object(_)) { + return Err(odd()); + } + match or_empty(field(s, "ancestry_broad")) { + Value::Null => terms.push(String::new()), + Value::Str(t) => terms.push(t), + _ => return Err(odd()), + } + } + let found: std::collections::BTreeSet> = terms.iter().map(|t| group(t)).collect(); + let sizes = match or_empty(field(&or_empty(field(&d, "performance_metrics")), "effect_sizes")) { + Value::Null => Vec::new(), + Value::Array(a) => a, + _ => return Err(odd()), + }; + let mut effects = Vec::new(); + for e in &sizes { + if !matches!(e, Value::Object(_)) { + return Err(odd()); + } + let name = field(e, "name_short").unwrap_or(&empty).clone(); + let est = field(e, "estimate").unwrap_or(&empty).clone(); + effects.push((name, est)); + } + let pgs = field(&d, "associated_pgs_id").unwrap_or(&empty); + let show = |v: &Value| py_str(v).map_err(|_| odd()); + let effects_said = effects + .iter() + .map(|(n, v)| Ok(format!("{} {}", show(n)?, show(v)?))) + .collect::, String>>()? + .join("; "); + let terms_said = terms.join("; "); + let said = format!( + "{}, {}, {}", + show(pgs)?, + if terms_said.is_empty() { "no samples" } else { &terms_said }, + if effects_said.is_empty() { "no effect size" } else { &effects_said }, + ); + let ok = is_str(Some(pgs), id) + && group_named(anc) + && found.len() == 1 + && found.contains(&Some(anc)) + && EFFECT.captures(eff).is_some_and(|m| { + let want: f64 = m[2].parse().unwrap_or(f64::NAN); + effects.iter().any(|(n, v)| { + is_str(Some(n), &m[1]) + && match v { + Value::Int(i) => (*i as f64 - want).abs() < 1e-9, + Value::Float(x) => (x - want).abs() < 1e-9, + _ => false, // a bool is never a number + } + }) + }); + Ok((said, ok)) +} + +fn group_named(anc: &str) -> bool { + GROUPS.iter().any(|(g, _)| *g == anc) +} + +fn stdin() -> Result { + let mut s = String::new(); + std::io::stdin() + .read_to_string(&mut s) + .map_err(|e| e.to_string())?; + Ok(s) +} + +/// `--name value` pairs. +fn opts<'a>(args: &'a [String], names: &[&str]) -> Option> { + let mut got = vec![None; names.len()]; + let mut it = args.iter(); + while let Some(a) = it.next() { + let i = names.iter().position(|n| n == a)?; + got[i] = Some(it.next()?.as_str()); + } + got.into_iter().collect() +} + +/// `og dev pgs ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write| { + let _ = write!(err, "{USAGE}"); + 2 + }; + match args.first().map(String::as_str) { + Some("trait") => { + let Some(v) = opts(&args[1..], &["--id"]) else { + return usage(err); + }; + match stdin().ok().and_then(|b| trait_of(&b, v[0])) { + Some(t) => { + let _ = writeln!(out, "{t}"); + 0 + } + None => 1, + } + } + Some("label-match") => match &args[1..] { + [label, trait_] => u8::from(!label_matches(label, trait_)), + _ => usage(err), + }, + Some("metric-match") => { + let Some(v) = opts(&args[1..], &["--id", "--ppm", "--ancestry", "--effect"]) else { + return usage(err); + }; + let body = match stdin() { + Ok(b) => b, + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + return 2; + } + }; + match metric(&body, v[0], v[1], v[2], v[3]) { + Ok((said, ok)) => { + let _ = writeln!(out, "{said}"); + u8::from(!ok) + } + Err(why) => { + let _ = writeln!(out, "{why}"); + 2 + } + } + } + _ => usage(err), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trait_needs_the_same_id_and_a_trait() { + assert_eq!( + trait_of(r#"{"id": "PGS000018", "trait_reported": " Coronary artery disease "}"#, "PGS000018").as_deref(), + Some("Coronary artery disease") + ); + assert_eq!(trait_of("{}", "PGS000018"), None); + assert_eq!(trait_of(r#"{"id": "PGS000020", "trait_reported": "x"}"#, "PGS000018"), None); + assert_eq!(trait_of(r#"{"id": "PGS000018", "trait_reported": ""}"#, "PGS000018"), None); + assert_eq!(trait_of("not json", "PGS000018"), None); + } + + #[test] + fn labels_match_by_words() { + assert!(label_matches("Type 2 diabetes (T2D)", "Type 2 diabetes (T2D)")); + assert!(label_matches("type_2_diabetes", "Type 2 diabetes (T2D)")); + assert!(label_matches("Late-onset Alzheimer\u{2019}s disease", "Late-onset Alzheimer's disease")); + assert!(!label_matches("inflammatory_bowel_disease", "Type 2 diabetes (T2D)")); + assert!(!label_matches("", "anything")); + } + + const ANSWER: &str = r#"{"id": "PPM000591", "associated_pgs_id": "PGS000018", + "sampleset": {"samples": [{"ancestry_broad": "European"}]}, + "performance_metrics": {"effect_sizes": [{"name_short": "HR", "estimate": 1.53}]}}"#; + + #[test] + fn an_evaluation_matches_its_score_ancestry_and_effect() { + let (said, ok) = metric(ANSWER, "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap(); + assert!(ok); + assert_eq!(said, "PGS000018, European, HR 1.53"); + for (id, anc, eff) in [ + ("PGS000004", "EUR", "HR 1.53"), + ("PGS000018", "AFR", "HR 1.53"), + ("PGS000018", "EUR", "HR 1.54"), + ("PGS000018", "EUR", "OR 1.53"), + ("PGS000018", "XYZ", "HR 1.53"), + ("PGS000018", "EUR", "HR"), + ] { + assert!(!metric(ANSWER, id, "PPM000591", anc, eff).unwrap().1, "{id} {anc} {eff}"); + } + assert_eq!(metric("{}", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), "(no such evaluation)"); + assert_eq!(metric("", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), "(not JSON)"); + } + + #[test] + fn a_bool_is_never_a_number() { + let body = ANSWER.replace("1.53", "true"); + let (said, ok) = metric(&body, "PGS000018", "PPM000591", "EUR", "HR 1").unwrap(); + assert!(!ok, "{said}"); + assert_eq!(said, "PGS000018, European, HR True"); + let body = ANSWER.replace("1.53", "2"); + assert!(metric(&body, "PGS000018", "PPM000591", "EUR", "HR 2").unwrap().1); + } + + #[test] + fn no_samples_no_effects() { + let body = r#"{"id": "PPM1", "associated_pgs_id": null, "sampleset": null}"#; + let (said, ok) = metric(body, "PGS1", "PPM1", "EUR", "OR 1").unwrap(); + assert!(!ok); + assert_eq!(said, "None, no samples, no effect size"); + } +} diff --git a/rust/crates/og-dev/src/versions.rs b/rust/crates/og-dev/src/versions.rs new file mode 100644 index 000000000..d08528e2e --- /dev/null +++ b/rust/crates/og-dev/src/versions.rs @@ -0,0 +1,263 @@ +//! `og dev versions check-run` and `og dev versions expand`: the version +//! checks of `scripts/ci/gen-containers-config.sh --check-versions` and +//! `scripts/ci/check-links.sh` that were Python. +//! +//! ```text +//! og dev versions check-run --config conf/containers.config FILE [PROCESS...] +//! og dev versions expand --env versions.env < URLLOC lines +//! ``` +//! +//! `check-run` proves a run used the images `conf/containers.config` gives +//! each process. Each module writes, as the first line of its block in +//! `pipeline_info/software_versions.yml` (FILE), the tag of the image it ran +//! in; a process that runs on the host (an `ext.pipeline_version` selector) +//! writes the release of the pipeline it starts. One `OK PROC key: value` +//! or `FAIL PROC ...` line per process of PROCESS... (default: every +//! selector) and of FILE. Exit 0 when all agree, 1 otherwise, 2 on a usage +//! error or a file that cannot be read. +//! +//! `expand` fills `${NAME}` and `$NAME` in each URL from `versions.env` (a +//! missing file fills nothing), drops trailing `.,;:]` and a `}` that closes +//! nothing, and prints the lines back. Exit 2 on a line with no tab or a +//! `versions.env` the strict reader refuses: on purpose, a line it cannot +//! read is no longer skipped ([`crate::versions_env`]). + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{BufRead, Write}; +use std::path::Path; +use std::sync::LazyLock; + +use regex::{Captures, Regex}; + +use crate::versions_env; + +const USAGE: &str = "usage: og dev versions check-run --config FILE VERSIONS.yml [PROCESS...]\n og dev versions expand --env versions.env < LINES\n og dev versions coupled --env versions.env\n"; + +fn re(s: &str) -> Regex { + Regex::new(s).expect("a valid built-in pattern") +} + +static CONTAINER: LazyLock = + LazyLock::new(|| re(r"withName: '(\w+)' \{ container = '([^']+)' \}")); +static PIPELINE: LazyLock = + LazyLock::new(|| re(r"withName: '(\w+)' \{ ext\.pipeline_version = '([^']+)';")); +static BLOCK: LazyLock = LazyLock::new(|| re(r#"^"([^"]+)":\s*$"#)); +static ENTRY: LazyLock = LazyLock::new(|| re(r"^\s+([\w.-]+):\s*(.*?)\s*$")); +static TAG_PREFIX: LazyLock = LazyLock::new(|| re(r"^[^:@]+[:@]")); + +/// The image (or pipeline release) of every process in the config text. +pub fn selectors(config: &str) -> BTreeMap { + let mut images = BTreeMap::new(); + for re in [&*CONTAINER, &*PIPELINE] { + for c in re.captures_iter(config) { + images.insert(c[1].to_owned(), c[2].to_owned()); + } + } + images +} + +/// The first `key: value` under each `"PATH:PROC":` block of a +/// `software_versions.yml`. +pub fn first_entries(yml: &str) -> BTreeMap { + let mut first = BTreeMap::new(); + let mut proc: Option = None; + for line in yml.split_inclusive('\n') { + let line = line.strip_suffix('\n').unwrap_or(line); + if let Some(c) = BLOCK.captures(line) { + proc = c[1].rsplit(':').next().map(str::to_owned); + continue; + } + if let (Some(c), Some(p)) = (ENTRY.captures(line), &proc) + && !p.is_empty() + && !first.contains_key(p) + { + first.insert(p.clone(), (c[1].to_owned(), c[2].to_owned())); + } + } + first +} + +/// The check on parsed inputs. `expect` empty means every selector. +pub fn check_run( + images: &BTreeMap, + first: &BTreeMap, + yml_name: &str, + expect: &[String], + out: &mut dyn Write, +) -> bool { + let mut ok = true; + if first.is_empty() { + let _ = writeln!(out, "FAIL: {yml_name} has no process block"); + ok = false; + } + let wanted: BTreeSet<&str> = if expect.is_empty() { + images.keys().map(String::as_str).collect() + } else { + expect.iter().map(String::as_str).collect() + }; + let all: BTreeSet<&str> = wanted + .iter() + .copied() + .chain(first.keys().map(String::as_str)) + .collect(); + for p in all { + match (images.get(p), first.get(p)) { + (None, _) => { + let _ = writeln!(out, "FAIL {p:<20} has no selector in conf/containers.config"); + ok = false; + } + (Some(_), None) => { + if wanted.contains(p) { + let _ = writeln!(out, "FAIL {p:<20} missing from {yml_name}"); + ok = false; + } + } + (Some(image), Some((key, value))) => { + let tag = TAG_PREFIX.replace(image, ""); + if *value == tag { + let _ = writeln!(out, "OK {p:<20} {key}: {value}"); + } else { + let _ = writeln!(out, "FAIL {p:<20} {key}: {value}, want {tag} ({image})"); + ok = false; + } + } + } + } + ok +} + +fn read(path: &str) -> Result { + std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}")) +} + +/// `og dev versions check-run ARGS`. +pub fn cli_check_run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let [flag, config, yml, expect @ ..] = args else { + let _ = write!(err, "{USAGE}"); + return 2; + }; + if flag != "--config" { + let _ = write!(err, "{USAGE}"); + return 2; + } + let texts = read(config).and_then(|c| Ok((c, read(yml)?))); + match texts { + Ok((c, y)) => u8::from(!check_run(&selectors(&c), &first_entries(&y), yml, expect, out)), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +static VAR: LazyLock = + LazyLock::new(|| re(r"\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)")); + +/// One URL with its variables filled and its trailing punctuation dropped. +pub fn expand(url: &str, vals: &BTreeMap) -> String { + let filled = VAR.replace_all(url, |c: &Captures| { + let name = c.get(1).or_else(|| c.get(2)).map_or("", |m| m.as_str()); + vals.get(name).cloned().unwrap_or_else(|| c[0].to_owned()) + }); + let mut u = filled.into_owned(); + // Trailing punctuation, and the brace that closes ${VAR:-https://...}. + while let Some(last) = u.chars().next_back() { + let unbalanced = last == '}' && u.matches('}').count() > u.matches('{').count(); + if ".,;:]".contains(last) || unbalanced { + u.pop(); + } else { + break; + } + } + u +} + +/// `og dev versions expand --env FILE`. +pub fn cli_expand(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let [flag, env] = args else { + let _ = write!(err, "{USAGE}"); + return 2; + }; + if flag != "--env" { + let _ = write!(err, "{USAGE}"); + return 2; + } + let pins = match std::fs::read_to_string(Path::new(env)) { + Ok(text) => versions_env::parse(&text), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(e) => Err(format!("cannot read {env}: {e}")), + }; + let vals: BTreeMap = match pins { + Ok(p) => p.into_iter().map(|p| (p.name, p.value)).collect(), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + return 2; + } + }; + for (i, line) in std::io::stdin().lock().lines().enumerate() { + let line = match line { + Ok(l) => l, + Err(e) => { + let _ = writeln!(err, "ERROR: cannot read line {}: {e}", i + 1); + return 2; + } + }; + let Some((url, loc)) = line.split_once('\t') else { + let _ = writeln!(err, "ERROR: line {} has no tab: {line}", i + 1); + return 2; + }; + let _ = writeln!(out, "{}\t{loc}", expand(url, &vals)); + } + 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONFIG: &str = "process {\n withName: 'ROH' { container = 'quay.io/x/bcftools:1.22--h0_0' }\n withName: 'VEP' { container = 'ensemblorg/ensembl-vep@sha256:abc' }\n withName: 'PRS' { ext.pipeline_version = 'v2.1.0'; ext.x = 'y' }\n}\n"; + + fn run(yml: &str, expect: &[&str]) -> (bool, String) { + let expect: Vec = expect.iter().map(|s| (*s).to_owned()).collect(); + let mut out = Vec::new(); + let ok = check_run(&selectors(CONFIG), &first_entries(yml), "v.yml", &expect, &mut out); + (ok, String::from_utf8(out).unwrap()) + } + + #[test] + fn right_tags_pass() { + let (ok, out) = run( + "\"PGX:ROH\":\n bcftools: 1.22--h0_0\n other: 9\n\"VEP\":\n ensemblvep: sha256:abc\n\"CLINICAL:PRS\":\n pgsc_calc: v2.1.0\n", + &[], + ); + assert!(ok, "{out}"); + assert!(out.contains("OK ROH bcftools: 1.22--h0_0\n"), "{out}"); + } + + #[test] + fn a_wrong_missing_or_unknown_process_fails() { + let (ok, out) = run( + "\"PGX:ROH\":\n bcftools: 0.0.0-planted\n\"NEW_TOOL\":\n tool: 1.0\n", + &["ROH", "VEP"], + ); + assert!(!ok); + assert!(out.contains("FAIL ROH bcftools: 0.0.0-planted, want 1.22--h0_0 (quay.io/x/bcftools:1.22--h0_0)"), "{out}"); + assert!(out.contains("FAIL NEW_TOOL has no selector"), "{out}"); + assert!(out.contains("FAIL VEP missing from v.yml"), "{out}"); + assert!(!out.contains("PRS"), "{out}"); + let (ok, out) = run("nothing: here\n", &["ROH"]); + assert!(!ok); + assert!(out.contains("FAIL: v.yml has no process block"), "{out}"); + } + + #[test] + fn urls_expand_and_lose_trailing_punctuation() { + let vals = BTreeMap::from([("REL".to_owned(), "1.2".to_owned())]); + assert_eq!(expand("https://x.org/${REL}/a.gz", &vals), "https://x.org/1.2/a.gz"); + assert_eq!(expand("https://x.org/$REL/a.gz).", &vals), "https://x.org/1.2/a.gz)"); + assert_eq!(expand("https://x.org/${NOPE}/a.", &vals), "https://x.org/${NOPE}/a"); + assert_eq!(expand("${U:-https://x.org/a}", &vals), "${U:-https://x.org/a}"); + assert_eq!(expand("https://x.org/a}", &vals), "https://x.org/a"); + assert_eq!(expand("https://x.org/a];:", &vals), "https://x.org/a"); + } +} diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs new file mode 100644 index 000000000..f6245ecbc --- /dev/null +++ b/rust/crates/og-dev/src/versions_env.rs @@ -0,0 +1,115 @@ +//! A strict reader of `versions.env`, the one list of images and coupled +//! data versions. +//! +//! The bash steps source the file, so a line could hold anything bash +//! accepts. The dev verbs never run it: they read it, and only in one shape, +//! `NAME="value"` with an optional trailing `# comment`, where the value +//! holds no `"`, `$` or backquote (nothing for bash to expand). A blank line +//! or a comment line is skipped. Any other line, and a name set twice, is an +//! error naming the line, so a check never reads a file it does not +//! understand as one with fewer pins. + +use std::path::Path; +use std::sync::LazyLock; + +use regex::Regex; + +/// One `NAME="value"` line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pin { + pub name: String, + pub value: String, + /// What follows the closing quote, as written (the comment, if any). + pub rest: String, + /// The 1-based line number. + pub line: usize, +} + +static LINE: LazyLock = LazyLock::new(|| { + Regex::new(r#"^([A-Z][A-Z0-9_]*)="([^"$`]*)"(\s*(?:#.*)?)$"#).expect("a valid pattern") +}); + +/// Reads `versions.env` text. Lines may end in `\n` or `\r\n`. +pub fn parse(text: &str) -> Result, String> { + let mut pins: Vec = Vec::new(); + for (i, raw) in text.split('\n').enumerate() { + let line = raw.strip_suffix('\r').unwrap_or(raw); + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let Some(c) = LINE.captures(line) else { + return Err(format!( + "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", + i + 1 + )); + }; + let name = c[1].to_owned(); + if let Some(first) = pins.iter().find(|p| p.name == name) { + return Err(format!( + "versions.env:{}: {name} is set again (first on line {})", + i + 1, + first.line + )); + } + pins.push(Pin { + name, + value: c[2].to_owned(), + rest: c[3].to_owned(), + line: i + 1, + }); + } + Ok(pins) +} + +/// Reads the `versions.env` at `path`. +pub fn read(path: &Path) -> Result, String> { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("cannot read {}: {e}", path.display()))?; + parse(&text) +} + +/// The value of `name`, if it is set. +pub fn get<'a>(pins: &'a [Pin], name: &str) -> Option<&'a Pin> { + pins.iter().find(|p| p.name == name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_values_and_comments() { + let p = parse("# head\n\nA_IMAGE=\"x/y:1\" # minimap2=2.31\r\nB=\"\"\n").unwrap(); + assert_eq!(p.len(), 2); + assert_eq!(p[0].name, "A_IMAGE"); + assert_eq!(p[0].value, "x/y:1"); + assert_eq!(p[0].rest, " # minimap2=2.31"); + assert_eq!(p[0].line, 3); + assert_eq!(p[1].value, ""); + } + + #[test] + fn refuses_what_bash_would_expand_or_a_second_value() { + for bad in [ + "A=\"${B}\"", + "A=\"$(date)\"", + "A=\"`x`\"", + "A=x", + "export A=\"x\"", + "A=\"x\" B", + "a=\"x\"", + ] { + assert!(parse(bad).is_err(), "{bad} was read"); + } + let e = parse("A=\"1\"\nA=\"2\"\n").unwrap_err(); + assert!(e.contains("A is set again (first on line 1)"), "{e}"); + } + + #[test] + fn the_repository_file_reads() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../versions.env"); + let pins = read(&path).unwrap(); + assert!(get(&pins, "SAMTOOLS_IMAGE").is_some()); + } +} diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 7b8f792ab..0006ba019 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -69,6 +69,9 @@ const USAGE: &str = "usage: og [args] og exomiser report [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] + og dev scan personal-data|images [args] + og dev versions check-run|expand|coupled [args] + og dev pgs trait|label-match|metric-match [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash From 2d60f679993627e65726faaf627a7c8d7d120bc1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:11:50 +0100 Subject: [PATCH 04/76] refactor(ci): check the Renovate dry run with og dev renovate-check The 210-line checker the workflow wrote into a file is now an og dev verb; its pins table is data in scripts/ci/renovate-pins.tsv, where a missing file or a glob that matches nothing is an error. On the last saved log of main it prints the same lines as the Python, clean and with planted pins. Bead: pgp-bs9u.39 --- .github/workflows/renovate-dry-run.yml | 237 +------ rust/crates/og-dev/src/lib.rs | 6 +- rust/crates/og-dev/src/renovate.rs | 847 +++++++++++++++++++++++++ scripts/ci/renovate-pins.tsv | 25 + 4 files changed, 894 insertions(+), 221 deletions(-) create mode 100644 rust/crates/og-dev/src/renovate.rs create mode 100644 scripts/ci/renovate-pins.tsv diff --git a/.github/workflows/renovate-dry-run.yml b/.github/workflows/renovate-dry-run.yml index 5722cb025..155568c19 100644 --- a/.github/workflows/renovate-dry-run.yml +++ b/.github/workflows/renovate-dry-run.yml @@ -6,7 +6,8 @@ name: Renovate dry run # matcher that misses an image, or a versioning that cannot read a tag, shows # up here first. The full debug log is kept as an artifact for 14 days. # -# The checks (each one has a planted fault in the self-test step): +# The checks (og dev renovate-check; each one has a planted fault in its +# self-test, which runs after the checks): # count every *_IMAGE line of versions.env is a detected docker dependency # names no package name carries a ':' or '@' (a digest line once became # a package named '...@sha256') @@ -16,9 +17,10 @@ name: Renovate dry run # one-pr a tool gets the same proposal in every file it is in # pins every non-image pin (Nextflow, actionlint, gitleaks, ShellCheck, # the Renovate CLI, the HLA database, pgsc_calc, the demo's Python -# packages, Aldy, the docs workflow's Python) is detected in every -# place it is pinned, as often as the text has them. A setup-nextflow -# step counts as a pin only when its version: line is a literal +# packages, Aldy, the docs workflow's Python; one row each in +# scripts/ci/renovate-pins.tsv) is detected in every place it is +# pinned, as often as the text has them. A setup-nextflow step +# counts as a pin only when its version: line is a literal # version, quoted or not; a computed one (an expression) is not a pin # lookups no lookup failed and no tag was unreadable; the only skipped # dependencies are the ones a rule in renovate.json disables @@ -31,7 +33,8 @@ on: # Every file a manager in renovate.json reads (the workflows hold the # Nextflow, actionlint, gitleaks, ShellCheck, Renovate and docs Python pins, # the docs the NXF_VER lines and the Aldy pin, scripts/lib/common.sh the HLA - # database, the demo script its Python packages), plus the doc generator. + # database, the demo script its Python packages), plus the doc generator, + # the pins table and the checker. # The lookups take about a minute. pull_request: branches: [main] @@ -42,6 +45,8 @@ on: - 'docs/*.md' - '.github/workflows/**' - 'scripts/ci/gen-versions-doc.sh' + - 'scripts/ci/renovate-pins.tsv' + - 'rust/crates/og-dev/src/renovate.rs' - 'scripts/lib/common.sh' - 'tests/demo/retake-screenshots.sh' push: @@ -91,7 +96,7 @@ jobs: lookup: # Public repo: GitHub-hosted runners only. runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 env: RENOVATE_VERSION: 44.199.2 steps: @@ -129,222 +134,14 @@ jobs: path: ${{ runner.temp }}/renovate.log retention-days: 14 - - name: Write the checker + # The checks are og dev renovate-check (rust/crates/og-dev/src/renovate.rs), + # with the non-image pins in scripts/ci/renovate-pins.tsv. + - name: Build og as scripts/setup.sh --og does if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} run: | - cat > "${RUNNER_TEMP}/check.py" <<'PY' - import collections, copy, glob, json, os, re, sys - - ENV = "versions.env" - TWINS = ("docs/versions.md", "conf/containers.config") - RULE_SKIPS = {"disabled", "package-rules"} - # The non-image pins: (dependency, files, how a pin of it looks in that text). - # A setup-nextflow step is a pin when a version: line with a literal - # version follows its uses: line inside the same step (the lines in - # between are blank or start with anything but '-', which opens the - # next step). - SETUP_NEXTFLOW = r'uses: ["\x27]?nf-core/setup-nextflow@[^\n]*\n(?:[ \t]*(?:[^\s-][^\n]*)?\n)*?[ \t]+version: ["\x27]?\d' - DEMO = "tests/demo/retake-screenshots.sh" - PINS = ( - ("nextflow-io/nextflow", [ENV], r'^NEXTFLOW_VERSION="'), - ("nextflow-io/nextflow", sorted(glob.glob(".github/workflows/*.yml")), SETUP_NEXTFLOW), - ("nextflow-io/nextflow", sorted(glob.glob("docs/*.md")), r"NXF_VER=\d"), - ("rhysd/actionlint", sorted(glob.glob(".github/workflows/*.yml")), r"ACTIONLINT_VERSION=\d"), - ("gitleaks/gitleaks", sorted(glob.glob(".github/workflows/*.yml")), r"GITLEAKS_VERSION=\d"), - ("koalaman/shellcheck", [".github/workflows/lint.yml"], r"^[ \t]+version: v\d"), - ("renovate", [".github/workflows/renovate-dry-run.yml"], r"RENOVATE_VERSION: \d"), - ("renovate", ["docs/updating.md"], r"renovate@\d"), - ("python", [".github/workflows/docs.yml"], r"python-version: ['\"]?\d"), - ("ANHIG/IMGTHLA", ["scripts/lib/common.sh"], r"^HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-\d"), - ("PGScatalog/pgsc_calc", [ENV], r'^PGSC_CALC_VERSION="'), - ("PGScatalog/pgsc_calc", ["conf/containers.config"], r"ext\.pipeline_version = '"), - ("playwright", [DEMO], r"playwright==\d"), - ("pillow", [DEMO], r"pillow==\d"), - ("pyobjc-framework-Vision", [DEMO], r"pyobjc-framework-Vision==\d"), - ("aldy", ["docs/21-cyrius.md"], r"aldy==\d|\[Aldy\]\([^)]*\) v\d"), - ) - # Only the files a pin is listed for are counted: python is also the - # docker image of versions.env, which the image checks cover. - PIN_FILES = collections.defaultdict(set) - for _name, _files, _ in PINS: - PIN_FILES[_name].update(_files) - - def pin_counts(): - want = collections.Counter() - for name, files, pattern in PINS: - for f in files: - n = len(re.findall(pattern, open(f).read(), re.M)) - if n: - want[(name, f)] += n - return want - - def load(log): - files = None - for line in open(log): - try: - rec = json.loads(line) - except ValueError: - continue - if rec.get("msg") == "packageFiles with updates": - files = rec["config"] - if files is None: - sys.exit("FAIL: no 'packageFiles with updates' record in %s" % log) - return [(pf["packageFile"], dep) for entries in files.values() for pf in entries for dep in pf.get("deps", [])] - - def env_images(text): - out = [] - for m in re.finditer(r'^[A-Z0-9_]+_IMAGE="([^"]*)"', text, re.M): - # name:tag@sha256:..., name@sha256:... or name:tag -> name - out.append(re.sub(r":[^:/]*$", "", m.group(1).split("@", 1)[0])) - return out - - def key(dep): - return (dep.get("depName"), dep.get("currentValue"), dep.get("currentDigest")) - - def proposals(dep): - return sorted((u.get("newValue") or "", u.get("newDigest") or "") for u in dep.get("updates", [])) - - def check(deps, env_text): - fails = [] - docker = [(f, d) for f, d in deps if d.get("datasource") == "docker"] - by_file = collections.defaultdict(list) - for f, d in docker: - by_file[f].append(d) - want = env_images(env_text) - got = [d.get("depName") for d in by_file[ENV]] - if len(got) != len(want): - fails.append("count: versions.env has %d *_IMAGE lines, Renovate found %d docker dependencies there" % (len(want), len(got))) - for name in sorted(set(want) - set(got)): - fails.append("count: %s (versions.env) is not a detected dependency" % name) - for f, d in docker: - if re.search(r"[:@]", d.get("depName") or ":"): - fails.append("names: %s: package name %r" % (f, d.get("depName"))) - env_keys = collections.Counter(key(d) for d in by_file[ENV]) - for f in TWINS: - keys = collections.Counter(key(d) for d in by_file[f]) - if f == "docs/versions.md" and keys != env_keys: - for k in sorted((keys - env_keys) | (env_keys - keys), key=str): - fails.append("twins: docs/versions.md and versions.env differ on %s" % (k,)) - else: - for k in sorted(set(keys) - set(env_keys), key=str): - fails.append("twins: %s has %s, which versions.env does not" % (f, k)) - seen = {} - for f, d in deps: - name = (d.get("datasource"), d.get("depName"), d.get("currentValue"), d.get("currentDigest")) - p = proposals(d) - if name in seen and seen[name][1] != p: - fails.append("one-pr: %s gets %s in %s but %s in %s" % (d.get("depName"), seen[name][1], seen[name][0], p, f)) - seen.setdefault(name, (f, p)) - for f, d in deps: - if d.get("warnings"): - fails.append("lookups: %s (%s): %s" % (d.get("depName"), f, "; ".join(w.get("message", str(w)) for w in d["warnings"]))) - if d.get("skipReason") and d["skipReason"] not in RULE_SKIPS: - fails.append("lookups: %s (%s) skipped: %s" % (d.get("depName"), f, d["skipReason"])) - got = collections.Counter((d.get("depName"), f) for f, d in deps - if f in PIN_FILES.get(d.get("depName"), ())) - for k in sorted(set(want_pins) | set(got)): - if want_pins[k] != got[k]: - fails.append("pins: %s in %s: the text pins it %d times, Renovate found %d" % (k[0], k[1], want_pins[k], got[k])) - return fails - - def self_test(deps, env_text): - ok = True - base = set(check(deps, env_text)) - def plant(label, fn, pattern): - nonlocal ok - d = copy.deepcopy(deps) - try: - fn(d) - except StopIteration: - print("self-test %-8s NOT PLANTED: no dependency to plant it on" % label) - ok = False - return - # Only a failure the plant added counts as caught. - hits = [x for x in check(d, env_text) if x.startswith(pattern) and x not in base] - print("self-test %-8s %s" % (label, "caught: " + hits[0] if hits else "NOT CAUGHT")) - ok = ok and bool(hits) - def first(d, f): - return next(x for g, x in d if g == f and x.get("datasource") == "docker") - def drop(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("datasource") == "docker")) - def digest_name(d): - first(d, ENV)["depName"] += "@sha256" - def stale_twin(d): - first(d, "docs/versions.md")["currentValue"] = "0.0.0-planted" - def split_pr(d): - first(d, "conf/containers.config")["updates"] = [{"newValue": "9.9.9-planted"}] - def drop_pin(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("depName") == "nextflow-io/nextflow")) - def broken_lookup(d): - first(d, ENV)["skipReason"] = "invalid-value" - plant("count", drop, "count:") - plant("names", digest_name, "names:") - plant("twins", stale_twin, "twins:") - plant("one-pr", split_pr, "one-pr:") - plant("pins", drop_pin, "pins:") - plant("lookups", broken_lookup, "lookups:") - # The setup-nextflow pattern itself, on planted workflow text. - step = (' - uses: %snf-core/setup-nextflow@%s%s # v3.0.1\n' - '%s' - ' if: steps.scope.outputs.run == \'true\'\n' - ' with:\n' - ' version: %s\n' - '\n' - ' - uses: actions/setup-java@%s # v6\n' - ' with:\n' - ' java-version: "17"\n') - for label, quote, gap, version, want in ( - ("computed", "", "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed, uses quoted", '"', "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed after a blank line", "", "\n", "$" + "{{ steps.nf.outputs.version }}", 0), - ("literal", "", "", '"25.10.8"', 1), - ("literal after a blank line", "", "\n", '"25.10.8"', 1), - ("single-quoted literal", "'", "", "'25.10.8'", 1), - ("bare literal, uses quoted", '"', "", "25.10.8", 1)): - text = step % (quote, "a" * 40, quote, gap, version, "b" * 40) - n = len(re.findall(SETUP_NEXTFLOW, text, re.M)) - print("self-test pin-text %-27s counted %d, want %d" % (label, n, want)) - ok = ok and n == want - clean = check(deps, env_text) - print("self-test clean %s" % ("passes" if not clean else "fails: %s" % clean[0])) - return ok - - def summary(deps): - rows, seen = [], set() - files = collections.defaultdict(set) - for f, d in deps: - files[(d.get("datasource"), d.get("depName"))].add(f) - for f, d in deps: - k = (d.get("datasource"), d.get("depName")) - if k in seen: - continue - seen.add(k) - cur = d.get("currentValue") or d.get("currentDigest") or "" - ups = ", ".join("%s (%s)" % (u.get("newValue") or u.get("newDigest"), u.get("updateType")) - for u in d.get("updates", [])) or "none" - note = d.get("skipReason") or "" - rows.append("| `%s` | `%s` | %s | %s | %d |" % (d.get("depName"), cur, ups, note, len(files[k]))) - return "\n".join(["| Dependency | Current | Proposed update | Skipped | Files |", "|---|---|---|---|---|"] + rows) - - deps = load(sys.argv[1]) - env_text = open(ENV).read() - want_pins = pin_counts() - table = summary(deps) - print(table) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: - out.write("## Renovate dry run\n\n" + table + "\n") - fails = check(deps, env_text) - for x in fails: - print("FAIL " + x) - if not self_test(deps, env_text): - fails.append("the self-test did not catch a planted fault") - print("FAIL the self-test did not catch a planted fault") - if fails: - sys.exit(1) - print("OK: %d dependencies; every check passed" % len(deps)) - PY + scripts/setup.sh --og + rust/og.sh check --required - name: Check what Renovate found if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} - run: python3 "${RUNNER_TEMP}/check.py" "${RUNNER_TEMP}/renovate.log" + run: bin/og dev renovate-check "${RUNNER_TEMP}/renovate.log" diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 918237714..ae345abbc 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -16,19 +16,22 @@ //! compound-het reads, a trimmed PharmCAT report). //! - `og dev evidence` ([`evidence`]): the measurements of the public //! evidence workflow on GIAB's HG002. +//! - `og dev renovate-check LOG` ([`renovate`]): what a Renovate dry run +//! found, against every pin in the repository. pub mod contract; pub mod evidence; pub mod fixture; pub mod parity; pub mod remnants; +pub mod renovate; pub mod report_only; pub mod schema; mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -65,6 +68,7 @@ pub fn main(args: &[String]) -> u8 { Some("report-only") => run(report_only::cli, &args[1..]), Some("fixture") => run(fixture::cli, &args[1..]), Some("evidence") => run(evidence::cli, &args[1..]), + Some("renovate-check") => run(renovate::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/crates/og-dev/src/renovate.rs b/rust/crates/og-dev/src/renovate.rs new file mode 100644 index 000000000..f3355c766 --- /dev/null +++ b/rust/crates/og-dev/src/renovate.rs @@ -0,0 +1,847 @@ +//! `og dev renovate-check LOG [--pins FILE] [--repo DIR]`: checks what a +//! Renovate dry run (`renovate --platform=local --dry-run=lookup`, JSON +//! debug log) found against every pin in the repository. It replaces the +//! Python the `renovate-dry-run.yml` workflow wrote into a file and ran. +//! +//! The checks, each with a planted fault in the self-test that runs after +//! them: +//! +//! - `count`: every `*_IMAGE` line of `versions.env` is a detected docker +//! dependency; +//! - `names`: no docker package name carries a `:` or `@`; +//! - `twins`: `docs/versions.md` holds exactly the images of `versions.env`, +//! and `conf/containers.config` holds only those; +//! - `one-pr`: a dependency gets the same proposals in every file it is in; +//! - `pins`: every non-image pin of the pins table +//! (`scripts/ci/renovate-pins.tsv`) is detected as often as its file pins +//! it; +//! - `lookups`: no lookup failed, no tag was unreadable, and the only +//! skipped dependencies are the ones a rule in `renovate.json` disables. +//! +//! The pins table has three tab-separated columns: the dependency, a file +//! or a glob of files (relative to the repository; `*` and `?` in the last +//! part), and a regular expression for one pin of it in that text, read +//! with `^` and `$` at line ends. A file that is missing, or a glob that +//! matches nothing, is an error (exit 2): a pin counted in no file could +//! never fail. +//! +//! Output: the Markdown table of every dependency and its proposed update +//! (also appended to `$GITHUB_STEP_SUMMARY`), one `FAIL : ...` line +//! per problem, the self-test lines, then `OK: N dependencies; every check +//! passed`. Exit 0 when clean, 1 on a finding, a self-test miss, or a log +//! without its `packageFiles with updates` record, 2 on a usage error or a +//! pins table it cannot use. + +use indexmap::IndexMap; +use pycompat::json::{self, Value}; +use pycompat::value::{py_repr, py_str, truthy}; +use regex::Regex; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; + +const USAGE: &str = "usage: og dev renovate-check LOG [--pins FILE] [--repo DIR]\n"; +const ENV: &str = "versions.env"; +const DOCS: &str = "docs/versions.md"; +const CONFIG: &str = "conf/containers.config"; +const TWINS: [&str; 2] = [DOCS, CONFIG]; +const RULE_SKIPS: [&str; 2] = ["disabled", "package-rules"]; +const PINS: &str = "scripts/ci/renovate-pins.tsv"; + +/// One dependency Renovate found: the file it is in and its record. +type Dep = (String, IndexMap); + +/// The pins table, expanded: per (dependency, file) how many pins the text +/// holds, and per dependency the files listed for it. +struct Pins { + want: BTreeMap<(String, String), usize>, + files: HashMap>, + setup_nextflow: Option, +} + +/// Runs `og dev renovate-check ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let (mut log, mut pins, mut repo) = (None, PINS.to_owned(), ".".to_owned()); + let mut it = args.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "-h" | "--help" => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + "--pins" | "--repo" => { + let Some(v) = it.next() else { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: {a} needs a value" + ); + return 2; + }; + if a == "--pins" { + pins = v.clone(); + } else { + repo = v.clone(); + } + } + _ if log.is_none() => log = Some(a.clone()), + _ => { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: unexpected argument '{a}'" + ); + return 2; + } + } + } + let Some(log) = log else { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: the following arguments are required: LOG" + ); + return 2; + }; + let repo = Path::new(&repo); + let pins = match read_pins(repo, &repo.join(&pins)) { + Ok(p) => p, + Err(e) => { + let _ = writeln!(err, "og dev renovate-check: {e}"); + return 2; + } + }; + let env_text = match fs::read_to_string(repo.join(ENV)) { + Ok(t) => t, + Err(e) => { + let _ = writeln!(err, "og dev renovate-check: {ENV}: {e}"); + return 2; + } + }; + let deps = match load(&log) { + Ok(d) => d, + Err(e) => { + let _ = writeln!(err, "{e}"); + return 1; + } + }; + let table = summary(&deps); + let _ = writeln!(out, "{table}"); + if let Some(path) = std::env::var_os("GITHUB_STEP_SUMMARY").filter(|p| !p.is_empty()) { + let appended = fs::OpenOptions::new() + .append(true) + .create(true) + .open(&path) + .and_then(|mut f| write!(f, "## Renovate dry run\n\n{table}\n")); + if let Err(e) = appended { + let _ = writeln!(err, "og dev renovate-check: GITHUB_STEP_SUMMARY: {e}"); + } + } + let mut fails = check(&deps, &env_text, &pins); + for x in &fails { + let _ = writeln!(out, "FAIL {x}"); + } + if !self_test(&deps, &env_text, &pins, out) { + fails.push("the self-test did not catch a planted fault".to_owned()); + let _ = writeln!(out, "FAIL the self-test did not catch a planted fault"); + } + if !fails.is_empty() { + return 1; + } + let _ = writeln!(out, "OK: {} dependencies; every check passed", deps.len()); + 0 +} + +/// `fnmatch` for `*` and `?`, case-sensitive. +fn fnmatch(pat: &[u8], name: &[u8]) -> bool { + match pat.split_first() { + None => name.is_empty(), + Some((b'*', rest)) => (0..=name.len()).any(|i| fnmatch(rest, &name[i..])), + Some((b'?', rest)) => !name.is_empty() && fnmatch(rest, &name[1..]), + Some((c, rest)) => name.first() == Some(c) && fnmatch(rest, &name[1..]), + } +} + +/// The files of `pattern` under `repo`, relative and sorted, as Python's +/// `sorted(glob.glob(pattern))` lists them: a wildcard never matches a name +/// that starts with a dot unless the pattern does. +fn glob(repo: &Path, pattern: &str) -> Result, String> { + let (dir, last) = pattern.rsplit_once('/').unwrap_or(("", pattern)); + if dir.contains(['*', '?']) { + return Err(format!( + "{pattern}: a wildcard is allowed only in the last part" + )); + } + if !last.contains(['*', '?']) { + return if repo.join(pattern).is_file() { + Ok(vec![pattern.to_owned()]) + } else { + Err(format!("{pattern} is not a file")) + }; + } + let mut out = Vec::new(); + let entries = fs::read_dir(repo.join(if dir.is_empty() { "." } else { dir })) + .map_err(|e| format!("{pattern}: {e}"))?; + for e in entries { + let name = e.map_err(|e| format!("{pattern}: {e}"))?.file_name(); + let name = name.to_string_lossy(); + if (name.starts_with('.') && !last.starts_with('.')) + || !fnmatch(last.as_bytes(), name.as_bytes()) + { + continue; + } + out.push(if dir.is_empty() { + name.into_owned() + } else { + format!("{dir}/{name}") + }); + } + out.sort(); + if out.is_empty() { + return Err(format!("{pattern} matches no file")); + } + Ok(out) +} + +fn read_pins(repo: &Path, path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut pins = Pins { + want: BTreeMap::new(), + files: HashMap::new(), + setup_nextflow: None, + }; + let mut texts: HashMap = HashMap::new(); + for (i, line) in text.lines().enumerate() { + if line.is_empty() || line.starts_with('#') { + continue; + } + let at = |msg: &str| format!("{}:{}: {msg}", path.display(), i + 1); + let cols: Vec<&str> = line.split('\t').collect(); + let [dep, files, pattern] = cols[..] else { + return Err(at( + "want three tab-separated columns: dependency, file or glob, pattern", + )); + }; + let re = Regex::new(&format!("(?m){pattern}")).map_err(|e| at(&e.to_string()))?; + if pattern.contains("setup-nextflow") { + pins.setup_nextflow = Some(re.clone()); + } + for f in glob(repo, files).map_err(|e| at(&e))? { + if !texts.contains_key(&f) { + let t = fs::read_to_string(repo.join(&f)).map_err(|e| at(&format!("{f}: {e}")))?; + texts.insert(f.clone(), t); + } + let n = re.find_iter(&texts[&f]).count(); + if n > 0 { + *pins.want.entry((dep.to_owned(), f.clone())).or_default() += n; + } + pins.files.entry(dep.to_owned()).or_default().insert(f); + } + } + Ok(pins) +} + +/// Every dependency of the log's last `packageFiles with updates` record. +fn load(log: &str) -> Result, String> { + let f = fs::File::open(log).map_err(|e| format!("FAIL: cannot read {log}: {e}"))?; + let mut files = None; + let mut r = BufReader::new(f); + let mut buf = Vec::new(); + loop { + buf.clear(); + if r.read_until(b'\n', &mut buf) + .map_err(|e| format!("FAIL: cannot read {log}: {e}"))? + == 0 + { + break; + } + let Ok(Value::Object(mut rec)) = json::loads(&String::from_utf8_lossy(&buf)) else { + continue; + }; + if rec.get("msg") == Some(&Value::Str("packageFiles with updates".to_owned())) { + files = Some(rec.swap_remove("config").unwrap_or(Value::Null)); + } + } + let Some(files) = files else { + return Err(format!( + "FAIL: no 'packageFiles with updates' record in {log}" + )); + }; + let shape = || { + format!( + "FAIL: the 'packageFiles with updates' record of {log} is not a map of package file lists" + ) + }; + let Value::Object(files) = files else { + return Err(shape()); + }; + let mut deps = Vec::new(); + for entries in files.into_values() { + let Value::Array(entries) = entries else { + return Err(shape()); + }; + for pf in entries { + let Value::Object(mut pf) = pf else { + return Err(shape()); + }; + let name = pf + .get("packageFile") + .map(py_str) + .ok_or_else(shape)? + .map_err(|_| shape())?; + if let Some(Value::Array(ds)) = pf.swap_remove("deps") { + for d in ds { + let Value::Object(d) = d else { + return Err(shape()); + }; + deps.push((name.clone(), d)); + } + } + } + } + Ok(deps) +} + +fn get<'a>(d: &'a IndexMap, k: &str) -> &'a Value { + d.get(k).unwrap_or(&Value::Null) +} + +/// Python's `"%s" % v`. +fn s(v: &Value) -> String { + py_str(v).unwrap_or_else(|_| format!("{v:?}")) +} + +/// Python's `repr(v)`. +fn r(v: &Value) -> String { + py_repr(v).unwrap_or_else(|_| format!("{v:?}")) +} + +/// `repr` of a tuple of values. +fn tuple(vs: &[&Value]) -> String { + let items: Vec = vs.iter().map(|v| r(v)).collect(); + format!("({})", items.join(", ")) +} + +fn is_docker(d: &IndexMap) -> bool { + get(d, "datasource") == &Value::Str("docker".to_owned()) +} + +/// The image names of `versions.env`: `name:tag@sha256:...`, +/// `name@sha256:...` and `name:tag` all give `name`. +fn env_images(text: &str) -> Vec { + let line = Regex::new(r#"(?m)^[A-Z0-9_]+_IMAGE="([^"]*)""#).expect("a valid pattern"); + let tag = Regex::new(r":[^:/]*$").expect("a valid pattern"); + line.captures_iter(text) + .map(|c| { + let v = c[1].split('@').next().unwrap_or(""); + tag.replace(v, "").into_owned() + }) + .collect() +} + +/// `(depName, currentValue, currentDigest)` as Python prints the tuple. +fn key(d: &IndexMap) -> String { + tuple(&[ + get(d, "depName"), + get(d, "currentValue"), + get(d, "currentDigest"), + ]) +} + +/// `v or ""`. +fn or_empty(v: &Value) -> Value { + if truthy(v) { + v.clone() + } else { + Value::Str(String::new()) + } +} + +/// The updates as sorted `(newValue or "", newDigest or "")` pairs, printed +/// as Python prints the list. +fn proposals(d: &IndexMap) -> String { + let mut ps: Vec<(Value, Value)> = updates(d) + .iter() + .map(|u| match u { + Value::Object(u) => (or_empty(get(u, "newValue")), or_empty(get(u, "newDigest"))), + _ => (Value::Null, Value::Null), + }) + .collect(); + ps.sort_by(|a, b| (s(&a.0), s(&a.1)).cmp(&(s(&b.0), s(&b.1)))); + let items: Vec = ps.iter().map(|(a, b)| tuple(&[a, b])).collect(); + format!("[{}]", items.join(", ")) +} + +fn updates(d: &IndexMap) -> &[Value] { + match d.get("updates") { + Some(Value::Array(u)) => u, + _ => &[], + } +} + +fn counter<'a>(it: impl Iterator>) -> BTreeMap { + let mut c = BTreeMap::new(); + for d in it { + *c.entry(key(d)).or_default() += 1; + } + c +} + +/// The docker dependencies of one file. +fn in_file<'a>( + docker: &'a [&'a Dep], + f: &'a str, +) -> impl Iterator> { + docker.iter().filter(move |(g, _)| g == f).map(|(_, d)| d) +} + +/// Every problem, as `check: ...` lines. +fn check(deps: &[Dep], env_text: &str, pins: &Pins) -> Vec { + let mut fails = Vec::new(); + let docker: Vec<&Dep> = deps.iter().filter(|(_, d)| is_docker(d)).collect(); + let want = env_images(env_text); + let got: Vec<&Value> = in_file(&docker, ENV).map(|d| get(d, "depName")).collect(); + if got.len() != want.len() { + fails.push(format!( + "count: versions.env has {} *_IMAGE lines, Renovate found {} docker dependencies there", + want.len(), + got.len() + )); + } + let want_set: BTreeSet<&String> = want.iter().collect(); + for name in want_set { + if !got.contains(&&Value::Str(name.clone())) { + fails.push(format!( + "count: {name} (versions.env) is not a detected dependency" + )); + } + } + for (f, d) in &docker { + let name = get(d, "depName"); + let text = if truthy(name) { + s(name) + } else { + ":".to_owned() + }; + if text.contains([':', '@']) { + fails.push(format!("names: {f}: package name {}", r(name))); + } + } + let env_keys = counter(in_file(&docker, ENV)); + for f in TWINS { + let keys = counter(in_file(&docker, f)); + if f == DOCS && keys != env_keys { + let all: BTreeSet<&String> = keys.keys().chain(env_keys.keys()).collect(); + for k in all { + if keys.get(k) != env_keys.get(k) { + fails.push(format!( + "twins: docs/versions.md and versions.env differ on {k}" + )); + } + } + } else { + for k in keys.keys().filter(|k| !env_keys.contains_key(*k)) { + fails.push(format!("twins: {f} has {k}, which versions.env does not")); + } + } + } + let mut seen: HashMap = HashMap::new(); + for (f, d) in deps { + let name = tuple(&[ + get(d, "datasource"), + get(d, "depName"), + get(d, "currentValue"), + get(d, "currentDigest"), + ]); + let p = proposals(d); + match seen.get(&name) { + Some((first, q)) if *q != p => fails.push(format!( + "one-pr: {} gets {q} in {first} but {p} in {f}", + s(get(d, "depName")) + )), + Some(_) => {} + None => { + seen.insert(name, (f, p)); + } + } + } + for (f, d) in deps { + let warnings = get(d, "warnings"); + if truthy(warnings) { + let msgs: Vec = match warnings { + Value::Array(ws) => ws + .iter() + .map(|w| match w { + Value::Object(o) if o.contains_key("message") => s(get(o, "message")), + w => s(w), + }) + .collect(), + w => vec![s(w)], + }; + fails.push(format!( + "lookups: {} ({f}): {}", + s(get(d, "depName")), + msgs.join("; ") + )); + } + let skip = get(d, "skipReason"); + if truthy(skip) && !RULE_SKIPS.contains(&s(skip).as_str()) { + fails.push(format!( + "lookups: {} ({f}) skipped: {}", + s(get(d, "depName")), + s(skip) + )); + } + } + let mut got: BTreeMap<(String, String), usize> = BTreeMap::new(); + for (f, d) in deps { + if let Value::Str(name) = get(d, "depName") + && pins.files.get(name).is_some_and(|fs| fs.contains(f)) + { + *got.entry((name.clone(), f.clone())).or_default() += 1; + } + } + let all: BTreeSet<&(String, String)> = pins.want.keys().chain(got.keys()).collect(); + for k in all { + let (w, g) = ( + pins.want.get(k).copied().unwrap_or(0), + got.get(k).copied().unwrap_or(0), + ); + if w != g { + fails.push(format!( + "pins: {} in {}: the text pins it {w} times, Renovate found {g}", + k.0, k.1 + )); + } + } + fails +} + +/// Plants one fault of each kind in a copy of the dependencies and shows +/// that the check it belongs to catches it; true when every one is caught. +fn self_test(deps: &[Dep], env_text: &str, pins: &Pins, out: &mut dyn Write) -> bool { + let mut ok = true; + let base: BTreeSet = check(deps, env_text, pins).into_iter().collect(); + let first = |d: &[Dep], f: &str| d.iter().position(|(g, x)| g == f && is_docker(x)); + type Plant = fn(&mut Vec, usize); + let plants: [(&str, &str, Option<&str>, Plant); 6] = [ + ("count", "count:", Some(ENV), |d, i| { + d.remove(i); + }), + ("names", "names:", Some(ENV), |d, i| { + let name = format!("{}@sha256", s(get(&d[i].1, "depName"))); + d[i].1.insert("depName".to_owned(), Value::Str(name)); + }), + ("twins", "twins:", Some(DOCS), |d, i| { + d[i].1.insert( + "currentValue".to_owned(), + Value::Str("0.0.0-planted".to_owned()), + ); + }), + ("one-pr", "one-pr:", Some(CONFIG), |d, i| { + let u = IndexMap::from([( + "newValue".to_owned(), + Value::Str("9.9.9-planted".to_owned()), + )]); + d[i].1 + .insert("updates".to_owned(), Value::Array(vec![Value::Object(u)])); + }), + ("pins", "pins:", None, |d, i| { + d.remove(i); + }), + ("lookups", "lookups:", Some(ENV), |d, i| { + d[i].1.insert( + "skipReason".to_owned(), + Value::Str("invalid-value".to_owned()), + ); + }), + ]; + for (label, pattern, file, plant) in plants { + let at = match file { + Some(f) => first(deps, f), + None => deps.iter().position(|(g, x)| { + g == ENV && get(x, "depName") == &Value::Str("nextflow-io/nextflow".to_owned()) + }), + }; + let Some(i) = at else { + let _ = writeln!( + out, + "self-test {label:<8} NOT PLANTED: no dependency to plant it on" + ); + ok = false; + continue; + }; + let mut d = deps.to_vec(); + plant(&mut d, i); + // Only a failure the plant added counts as caught. + let hit = check(&d, env_text, pins) + .into_iter() + .find(|x| x.starts_with(pattern) && !base.contains(x)); + match hit { + Some(h) => { + let _ = writeln!(out, "self-test {label:<8} caught: {h}"); + } + None => { + let _ = writeln!(out, "self-test {label:<8} NOT CAUGHT"); + ok = false; + } + } + } + // The setup-nextflow pattern itself, on planted workflow text. + let Some(setup) = &pins.setup_nextflow else { + let _ = writeln!( + out, + "self-test pin-text NOT PLANTED: the pins table has no setup-nextflow pattern" + ); + return false; + }; + let computed = "${{ steps.nf.outputs.version }}"; + let cases: [(&str, &str, &str, &str, usize); 7] = [ + ("computed", "", "", computed, 0), + ("computed, uses quoted", "\"", "", computed, 0), + ("computed after a blank line", "", "\n", computed, 0), + ("literal", "", "", "\"25.10.8\"", 1), + ("literal after a blank line", "", "\n", "\"25.10.8\"", 1), + ("single-quoted literal", "'", "", "'25.10.8'", 1), + ("bare literal, uses quoted", "\"", "", "25.10.8", 1), + ]; + for (label, quote, gap, version, want) in cases { + let text = format!( + " - uses: {quote}nf-core/setup-nextflow@{a}{quote} # v3.0.1\n{gap} if: steps.scope.outputs.run == 'true'\n with:\n version: {version}\n\n - uses: actions/setup-java@{b} # v6\n with:\n java-version: \"17\"\n", + a = "a".repeat(40), + b = "b".repeat(40), + ); + let n = setup.find_iter(&text).count(); + let _ = writeln!( + out, + "self-test pin-text {label:<27} counted {n}, want {want}" + ); + ok = ok && n == want; + } + let clean = check(deps, env_text, pins); + let verdict = clean + .first() + .map_or_else(|| "passes".to_owned(), |c| format!("fails: {c}")); + let _ = writeln!(out, "self-test clean {verdict}"); + ok +} + +/// The Markdown table of every dependency, once per datasource and name. +fn summary(deps: &[Dep]) -> String { + let id = |d: &IndexMap| tuple(&[get(d, "datasource"), get(d, "depName")]); + let mut files: HashMap> = HashMap::new(); + for (f, d) in deps { + files.entry(id(d)).or_default().insert(f); + } + let mut rows = vec![ + "| Dependency | Current | Proposed update | Skipped | Files |".to_owned(), + "|---|---|---|---|---|".to_owned(), + ]; + let mut seen = BTreeSet::new(); + for (_, d) in deps { + let k = id(d); + if !seen.insert(k.clone()) { + continue; + } + let cur = [get(d, "currentValue"), get(d, "currentDigest")] + .into_iter() + .find(|v| truthy(v)) + .map_or_else(String::new, s); + let ups: Vec = updates(d) + .iter() + .map(|u| { + let u = match u { + Value::Object(u) => u, + _ => return s(u), + }; + let new = if truthy(get(u, "newValue")) { + get(u, "newValue") + } else { + get(u, "newDigest") + }; + format!("{} ({})", s(new), s(get(u, "updateType"))) + }) + .collect(); + let ups = if ups.is_empty() { + "none".to_owned() + } else { + ups.join(", ") + }; + let skip = get(d, "skipReason"); + let note = if truthy(skip) { s(skip) } else { String::new() }; + rows.push(format!( + "| `{}` | `{cur}` | {ups} | {note} | {} |", + s(get(d, "depName")), + files[&k].len() + )); + } + rows.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fnmatch_and_env_images() { + assert!( + fnmatch(b"*.yml", b"e2e.yml") + && !fnmatch(b"*.yml", b"e2e.yaml") + && fnmatch(b"a?c", b"abc") + ); + let env = "A_IMAGE=\"x/y:1.2@sha256:ab\"\nB_IMAGE=\"z@sha256:cd\"\nC_IMAGE=\"host:5000/w:3\"\nD=\"1\"\n"; + assert_eq!(env_images(env), ["x/y", "z", "host:5000/w"]); + } + + #[test] + fn keys_and_proposals_print_as_python() { + let d: IndexMap = match json::loads( + r#"{"depName":"it's","currentValue":"1","updates":[{"newValue":"3"},{"newValue":"2","newDigest":"d"}]}"#, + ) { + Ok(Value::Object(o)) => o, + other => panic!("{other:?}"), + }; + assert_eq!(key(&d), r#"("it's", '1', None)"#); + assert_eq!(proposals(&d), "[('2', 'd'), ('3', '')]"); + } + + /// A repository of the files the check reads, and a log that matches it. + fn planted(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("og-renovate-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + for sub in ["docs", "conf", "scripts/ci", ".github/workflows"] { + fs::create_dir_all(d.join(sub)).expect("mkdir"); + } + let w = |p: &str, t: &str| fs::write(d.join(p), t).expect("write"); + w( + ENV, + "NEXTFLOW_VERSION=\"25.10.8\"\nA_IMAGE=\"x/a:1@sha256:aa\"\nB_IMAGE=\"x/b:2\"\n", + ); + w(DOCS, "| a | x/a:1 |\n"); + w(CONFIG, "container = 'x/a:1'\n"); + w( + ".github/workflows/ci.yml", + " - uses: nf-core/setup-nextflow@abc # v3\n with:\n version: \"25.10.8\"\n", + ); + w( + ".github/workflows/.hidden.yml", + " - uses: nf-core/setup-nextflow@abc # v3\n with:\n version: \"25.10.8\"\n", + ); + w( + PINS, + "# comment\nnextflow-io/nextflow\tversions.env\t^NEXTFLOW_VERSION=\"\nnextflow-io/nextflow\t.github/workflows/*.yml\tuses: [\"\\x27]?nf-core/setup-nextflow@[^\\n]*\\n(?:[ \\t]*(?:[^\\s-][^\\n]*)?\\n)*?[ \\t]+version: [\"\\x27]?\\d\n", + ); + let dep = |n: &str, v: &str, ds: &str| { + format!(r#"{{"depName":"{n}","currentValue":"{v}","datasource":"{ds}"}}"#) + }; + let log = format!( + "not json\n{{\"msg\":\"other\"}}\n{{\"msg\":\"packageFiles with updates\",\"config\":{{\"regex\":[{{\"packageFile\":\"versions.env\",\"deps\":[{},{},{}]}},{{\"packageFile\":\"docs/versions.md\",\"deps\":[{},{}]}},{{\"packageFile\":\"conf/containers.config\",\"deps\":[{}]}},{{\"packageFile\":\".github/workflows/ci.yml\",\"deps\":[{}]}}]}}}}\n", + dep("x/a", "1", "docker"), + dep("x/b", "2", "docker"), + dep("nextflow-io/nextflow", "25.10.8", "github-releases"), + dep("x/a", "1", "docker"), + dep("x/b", "2", "docker"), + dep("x/a", "1", "docker"), + dep("nextflow-io/nextflow", "25.10.8", "github-releases"), + ); + w("renovate.log", &log); + d + } + + fn run(d: &Path) -> (u8, String) { + let args: Vec = [ + d.join("renovate.log").to_string_lossy().into_owned(), + "--repo".to_owned(), + d.to_string_lossy().into_owned(), + ] + .into(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8_lossy(&out).into_owned() + &String::from_utf8_lossy(&err), + ) + } + + #[test] + fn a_clean_tree_passes_and_every_plant_is_caught() { + let d = planted("clean"); + let (code, out) = run(&d); + assert_eq!(code, 0, "{out}"); + assert!( + out.contains("OK: 7 dependencies; every check passed"), + "{out}" + ); + assert!(out.contains("| `x/a` | `1` | none | | 3 |"), "{out}"); + assert_eq!(out.matches("caught: ").count(), 6, "{out}"); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_pin_renovate_misses_fails() { + let d = planted("pin"); + fs::write( + d.join(".github/workflows/more.yml"), + " - uses: 'nf-core/setup-nextflow@abc'\n\n with:\n version: '26.04.0'\n", + ) + .expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL pins: nextflow-io/nextflow in .github/workflows/more.yml: the text pins it 1 times, Renovate found 0"), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn an_image_renovate_misses_and_a_stale_twin_fail() { + let d = planted("image"); + let log = fs::read_to_string(d.join("renovate.log")).expect("read"); + let log = log.replacen( + r#"{"depName":"x/b","currentValue":"2","datasource":"docker"},"#, + "", + 1, + ); + let log = log.replacen( + r#""docs/versions.md","deps":[{"depName":"x/a","currentValue":"1""#, + r#""docs/versions.md","deps":[{"depName":"x/a","currentValue":"0""#, + 1, + ); + fs::write(d.join("renovate.log"), log).expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL count: x/b (versions.env) is not a detected dependency"), + "{out}" + ); + assert!( + out.contains( + "FAIL twins: docs/versions.md and versions.env differ on ('x/a', '0', None)" + ), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_glob_that_matches_nothing_is_refused() { + let d = planted("glob"); + fs::write(d.join(PINS), "aldy\tdocs/*.txt\taldy==\\d\n").expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 2, "{out}"); + assert!(out.contains("docs/*.txt matches no file"), "{out}"); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_log_without_the_record_fails() { + let d = planted("nolog"); + fs::write(d.join("renovate.log"), "{\"msg\":\"other\"}\n").expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL: no 'packageFiles with updates' record in"), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } +} diff --git a/scripts/ci/renovate-pins.tsv b/scripts/ci/renovate-pins.tsv new file mode 100644 index 000000000..ba15cba87 --- /dev/null +++ b/scripts/ci/renovate-pins.tsv @@ -0,0 +1,25 @@ +# The non-image pins `og dev renovate-check` counts (.github/workflows/renovate-dry-run.yml): +# each must be a dependency Renovate detects in every place it is pinned, as often as the text has it. +# dependencyfile or glob (wildcards in the last part only)regular expression for one pin, +# with ^ and $ at line ends. A missing file or a glob that matches nothing is an error. +# Only the files listed for a dependency are counted: python is also the docker image of +# versions.env, which the image checks cover. +# A setup-nextflow step is a pin when a version: line with a literal version follows its uses: +# line inside the same step (the lines in between are blank or start with anything but '-', +# which opens the next step); a computed version (an expression) is not a pin. +nextflow-io/nextflow versions.env ^NEXTFLOW_VERSION=" +nextflow-io/nextflow .github/workflows/*.yml uses: ["\x27]?nf-core/setup-nextflow@[^\n]*\n(?:[ \t]*(?:[^\s-][^\n]*)?\n)*?[ \t]+version: ["\x27]?\d +nextflow-io/nextflow docs/*.md NXF_VER=\d +rhysd/actionlint .github/workflows/*.yml ACTIONLINT_VERSION=\d +gitleaks/gitleaks .github/workflows/*.yml GITLEAKS_VERSION=\d +koalaman/shellcheck .github/workflows/lint.yml ^[ \t]+version: v\d +renovate .github/workflows/renovate-dry-run.yml RENOVATE_VERSION: \d +renovate docs/updating.md renovate@\d +python .github/workflows/docs.yml python-version: ['\"]?\d +ANHIG/IMGTHLA scripts/lib/common.sh ^HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-\d +PGScatalog/pgsc_calc versions.env ^PGSC_CALC_VERSION=" +PGScatalog/pgsc_calc conf/containers.config ext\.pipeline_version = ' +playwright tests/demo/retake-screenshots.sh playwright==\d +pillow tests/demo/retake-screenshots.sh pillow==\d +pyobjc-framework-Vision tests/demo/retake-screenshots.sh pyobjc-framework-Vision==\d +aldy docs/21-cyrius.md aldy==\d|\[Aldy\]\([^)]*\) v\d From c6750ad148e209167fd066eba81c8aa115431308 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:19:14 +0100 Subject: [PATCH 05/76] refactor(ci): switch the evidence workflow and fixtures to og, delete their Python public-evidence.yml runs og dev evidence clashes and hg002-gvcf; the comphet e2e case builds its fixture with og dev fixture comphet. The comphet generator and the PharmCAT trimmer are deleted. Bead: pgp-bs9u.39 --- .github/workflows/e2e.yml | 1 - .github/workflows/public-evidence.yml | 106 +++-------------- scripts/ci/e2e-case-paths.tsv | 2 +- scripts/ci/e2e-scope.sh | 2 +- tests/e2e/comphet-phasing-1-reads.sh | 4 +- tests/fixtures/comphet/README.md | 3 +- tests/fixtures/comphet/make_fixture.py | 150 ------------------------- tests/fixtures/pharmcat/README.md | 4 +- tests/fixtures/pharmcat/trim_report.py | 57 ---------- 9 files changed, 22 insertions(+), 307 deletions(-) delete mode 100755 tests/fixtures/comphet/make_fixture.py delete mode 100755 tests/fixtures/pharmcat/trim_report.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e7c02c998..39d4b272d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -67,7 +67,6 @@ on: - '.github/workflows/e2e.yml' - '.github/actions/**' - 'tests/fixtures/annotsv/class45.sh' - - 'tests/fixtures/comphet/make_fixture.py' - 'tests/fixtures/consumer_export/sites.tsv.gz' - 'tests/fixtures/families/stub_families.tsv' - 'tests/fixtures/pharmcat/refblocks.g.vcf' diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..aa6e7dcc7 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -166,6 +166,11 @@ jobs: name: archaic-1kg-sites path: sites + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --archaic and step 43 on HG002 (chromosome ${{ matrix.part }}) id: run continue-on-error: ${{ matrix.part == 'empty' }} @@ -230,47 +235,7 @@ jobs: # same-position rule of archaic_report.py decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz - python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' - import glob, gzip, os, sys - raw, split, data, out = sys.argv[1:] - tables = sorted(glob.glob(os.path.join(data, "chr*.tsv.gz"))) - chroms = {os.path.basename(t)[:-7] for t in tables} # this part's chromosomes only - def clashes(path): - """Positions of this part with two records or more, one an SNV (the VCF is sorted).""" - hit, key, n, snvs = set(), None, 0, 0 - for line in gzip.open(path, "rt"): - if line.startswith("#"): - continue - c = line.split("\t", 6) - if c[0] not in chroms: - continue - k = (c[0], int(c[1])) - if k != key: - key, n, snvs = k, 0, 0 - n += 1 - snvs += len(c[3]) == 1 and len(c[4]) == 1 - if n >= 2 and snvs >= 1: - hit.add(key) - return hit - sets = {"raw": clashes(raw), "split": clashes(split)} - on_table = {k: 0 for k in sets} - for t in tables: - chrom = os.path.basename(t)[:-7] - want = {k: {p for c, p in v if c == chrom} for k, v in sets.items()} - if not any(want.values()): - continue - for line in gzip.open(t, "rt"): - c = line.split("\t", 2) - if c[0] != chrom: - continue - p = int(c[1]) - for k in want: - if p in want[k]: - on_table[k] += 1 - with open(out, "w") as f: - for k in sets: - f.write(f"{k}_positions\t{len(sets[k])}\n{k}_on_table\t{on_table[k]}\n") - PY + bin/og dev evidence clashes "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" cat "$OUT/affected.tsv" - name: The red control failed, as it must @@ -597,6 +562,11 @@ jobs: gzip -dc "${R}/${f}.gz" > "${R}/GRCh38_no_alt_analysis_set.fasta" rm -f "${R}/${f}.gz" + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --ancient-panel and step 44 on HG002, measured env: THREADS: '4' @@ -672,57 +642,9 @@ jobs: mkdir -p "${G}/HG002/vcf" curl -fsSL --retry 5 -o "${EV}/truth.vcf.gz" "${GIAB_HG002}.vcf.gz" curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed" - python3 -I - "${EV}/truth.vcf.gz" "${EV}/truth.bed" "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \ - "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" <<'PY' - import bisect, gzip, sys - truth, bed, ref, sites, out = sys.argv[1:] - fai = {l.split("\t")[0]: [int(x) for x in l.split("\t")[1:5]] for l in open(ref + ".fai")} - fa = open(ref, "rb") - def base(c, p): - n, off, lb, lw = fai[c] - fa.seek(off + ((p - 1) // lb) * lw + (p - 1) % lb) - return fa.read(1).decode().upper() - conf = {} - for l in open(bed): - c, s, e = l.split()[:3] - conf.setdefault(c, []).append((int(s), int(e))) - starts = {c: [s for s, _ in sorted(v)] for c, v in conf.items()} - ends = {c: [e for _, e in sorted(v)] for c, v in conf.items()} - def confident(c, p): - i = bisect.bisect_right(starts.get(c, []), p - 1) - 1 - return i >= 0 and ends[c][i] >= p - covered, head, body = {}, [], [] - for l in gzip.open(truth, "rt"): - if l.startswith("##"): - head.append(l) - continue - if l.startswith("#"): - head.append('##INFO=\n') - head.append('##ALT=\n') - head.append(l) - continue - f = l.split("\t", 5) - p = int(f[1]) - covered.setdefault(f[0], set()).update(range(p, p + len(f[3]))) - body.append(l) - blocks = 0 - with open(sites) as s: - next(s) - for l in s: - _, c, p = l.split("\t")[:3] - p = int(p) - if c not in fai or p in covered.get(c, ()) or not confident(c, p): - continue - b = base(c, p) - if b not in "ACGT": - continue - body.append(f"{c}\t{p}\t.\t{b}\t<*>\t.\t.\tEND={p}\tGT\t0/0\n") - blocks += 1 - with open(out, "w") as o: - o.writelines(head) - o.writelines(body) - print(f"gVCF: {len(body) - blocks} GIAB records, {blocks} reference blocks at panel sites") - PY + /usr/bin/time -v -o "${M}/gvcf.time" bin/og dev evidence hg002-gvcf "${EV}/truth.vcf.gz" "${EV}/truth.bed" \ + "${G}/reference/GRCh38_no_alt_analysis_set.fasta" "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" + grep -E 'Maximum resident|Elapsed' "${M}/gvcf.time" docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/ev" -v "${G}:/g" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' bcftools sort -m 2G -T /ev/sort -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /ev/hg002.unsorted.vcf bcftools index -t /g/HG002/vcf/HG002.g.vcf.gz' diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 02c026afa..93069dae9 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -78,7 +78,7 @@ cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py +comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh diff --git a/scripts/ci/e2e-scope.sh b/scripts/ci/e2e-scope.sh index 4e254dc56..81ddd26d0 100755 --- a/scripts/ci/e2e-scope.sh +++ b/scripts/ci/e2e-scope.sh @@ -46,7 +46,7 @@ export LC_ALL=C SCOPE_DIRS=(scripts modules workflows bin conf assets tests/e2e .github/actions) SCOPE_FILES=(tests/fixtures/VERSION main.nf nextflow.config nextflow_schema.json versions.env .github/workflows/e2e.yml # The committed fixtures the cases read (their rows name them too). - tests/fixtures/annotsv/class45.sh tests/fixtures/comphet/make_fixture.py tests/fixtures/consumer_export/sites.tsv.gz + tests/fixtures/annotsv/class45.sh tests/fixtures/consumer_export/sites.tsv.gz tests/fixtures/families/stub_families.tsv tests/fixtures/pharmcat/refblocks.g.vcf tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf tests/fixtures/vcf/grch37_no_contigs.vcf) diff --git a/tests/e2e/comphet-phasing-1-reads.sh b/tests/e2e/comphet-phasing-1-reads.sh index 2dc4e2ccf..5107b6ca6 100755 --- a/tests/e2e/comphet-phasing-1-reads.sh +++ b/tests/e2e/comphet-phasing-1-reads.sh @@ -2,7 +2,7 @@ # Step 31 phases each compound-het pair from the reads, and the Nextflow # COMPHET_PHASE process writes the same pairs table. # -# The fixture is tests/fixtures/comphet/make_fixture.py: read pairs on a +# The fixture is og dev fixture comphet (tests/fixtures/comphet): read pairs on a # made-up contig (chrP) and a VEP VCF with three pairs, all rare and HIGH: # GENEC SNV at 1000 and a deletion at 1003 on one copy -> cis, 22 reads # GENET SNVs at 5000 and 5400, one on each copy; only a read and its mate @@ -28,7 +28,7 @@ cell() { make_sample() { local s=$1 f="${CASE_TMP}/fixture-$1" rm -rf "$f" "${GENOME_DIR:?}/${s}" - python3 "${REPO}/tests/fixtures/comphet/make_fixture.py" "$f" "${@:2}" + "${REPO}/bin/og" dev fixture comphet "$f" "${@:2}" mkdir -p "${GENOME_DIR}/${s}/vep" "${GENOME_DIR}/${s}/aligned" sed "s/CHPH/${s}/g" "${f}/vep.vcf" > "${GENOME_DIR}/${s}/vep/${s}_vep.vcf" sed "s/CHPH/${s}/g" "${f}/reads.sam" > "${GENOME_DIR}/${s}/aligned/${s}.sam" diff --git a/tests/fixtures/comphet/README.md b/tests/fixtures/comphet/README.md index b0a5816c6..fc17a306e 100644 --- a/tests/fixtures/comphet/README.md +++ b/tests/fixtures/comphet/README.md @@ -1,6 +1,7 @@ # Made-up reads for the compound-het phase (step 31) -`make_fixture.py OUT_DIR [--swap]` writes the fixture: read pairs of 100 bp +`bin/og dev fixture comphet OUT_DIR [--swap]` writes the fixture +(`rust/crates/og-dev/src/fixture.rs`): read pairs of 100 bp from 500 bp fragments on a made-up 80 kb contig (`chrP`, seeded random sequence), a VEP VCF of six heterozygous variants for step 31, and the three pairs as slivar names them. The reads and variants are made up; they describe diff --git a/tests/fixtures/comphet/make_fixture.py b/tests/fixtures/comphet/make_fixture.py deleted file mode 100755 index 38d842fc6..000000000 --- a/tests/fixtures/comphet/make_fixture.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -"""Write the compound-het phasing fixture: made-up reads on a made-up contig. - - make_fixture.py OUT_DIR [--swap] - -writes, for the sample CHPH on contig chrP (80 kb of seeded random sequence): - - reads.sam read pairs of 100 bp, 500 bp fragments (mates 400 bp apart) - vep.vcf the six heterozygous variants with a VEP CSQ, as step 13 writes - them, for step 31 (all rare, HIGH impact, three genes) - comphet.vcf the three pairs as slivar compound-hets names them - (INFO/slivar_comphet), for og comphet phase alone - -The three pairs and what the reads show: - - GENEC SNV at 1000, 2-base deletion at 1003: both on copy 1 (cis); single - reads cover both - GENET SNVs at 5000 and 5400, one on each copy (trans); no single read - covers both, only a read and its mate do - GENEF SNVs at 20000 and 70000, one on each copy; 50 kb apart, so no read - pair covers both and the phase cannot be read (unknown) - ---swap moves GENEC's deletion to copy 2: the same positions, now in trans. -The output is the same for the same arguments; nothing is random at run time. -""" -import os -import random -import sys - -SAMPLE = "CHPH" -CONTIG = "chrP" -LENGTH = 80000 -READ = 100 -FRAG = 500 -FIELDS = ("Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|" - "Existing_variation|MAX_AF|gnomADe_AF|gnomADg_AF|CLIN_SIG") - - -def reference(): - rng = random.Random(31) - return "".join(rng.choice("ACGT") for _ in range(LENGTH)) - - -def other(base): - return "ACGT"["ACGT".index(base) - 1] - - -def variants(seq, swap=False): - """[(gene, pos, ref, alt, consequence, copy)]: pos 1-based, copy 0 or 1.""" - snv = lambda pos: (seq[pos - 1], other(seq[pos - 1])) # noqa: E731 - return [ - ("GENEC", 1000, *snv(1000), "stop_gained", 0), - ("GENEC", 1003, seq[1002:1005], seq[1002], "frameshift_variant", 1 if swap else 0), - ("GENET", 5000, *snv(5000), "stop_gained", 0), - ("GENET", 5400, *snv(5400), "stop_gained", 1), - ("GENEF", 20000, *snv(20000), "stop_gained", 0), - ("GENEF", 70000, *snv(70000), "stop_gained", 1), - ] - - -def read_on(seq, vs, copy, start): - """(sequence, cigar) of a READ-base read from 0-based start on one copy.""" - snvs = {v[1] - 1: v[3] for v in vs if v[5] == copy and len(v[2]) == len(v[3])} - dels = {v[1] + len(v[3]) - 1: len(v[2]) - len(v[3]) for v in vs if v[5] == copy and len(v[2]) > len(v[3])} - out, ops, p = [], [], start - while len(out) < READ: - if p in dels: - ops.append(("D", dels[p])) - p += dels[p] - continue - out.append(snvs.get(p, seq[p])) - ops.append(("M", 1)) - p += 1 - cigar, last, n = "", None, 0 - for op, k in ops + [(None, 0)]: - if op == last: - n += k - continue - if last: - cigar += f"{n}{last}" - last, n = op, k - return "".join(out), cigar - - -def sam(seq, vs): - lines = ["@HD\tVN:1.6\tSO:unsorted", f"@SQ\tSN:{CONTIG}\tLN:{LENGTH}", f"@RG\tID:{SAMPLE}\tSM:{SAMPLE}"] - # Fragment starts (0-based) around each pair: read 1 covers the first - # position, its mate the second one when they are 400 bp apart. - starts = {"c": range(920, 991, 7), "t": range(4905, 4996, 10), "f": range(19905, 19996, 10), - "g": range(69905, 69996, 10)} - for region, rs in starts.items(): - for copy in (0, 1): - for s in rs: - name = f"{region}{copy}_{s}" - m = s + FRAG - READ - s1, c1 = read_on(seq, vs, copy, s) - s2, c2 = read_on(seq, vs, copy, m) - q = "I" * READ - lines.append(f"{name}\t99\t{CONTIG}\t{s + 1}\t60\t{c1}\t=\t{m + 1}\t{FRAG}\t{s1}\t{q}\tRG:Z:{SAMPLE}") - lines.append(f"{name}\t147\t{CONTIG}\t{m + 1}\t60\t{c2}\t=\t{s + 1}\t-{FRAG}\t{s2}\t{q}\tRG:Z:{SAMPLE}") - return "\n".join(lines) + "\n" - - -def vcf_head(info): - return ("##fileformat=VCFv4.2\n" - f"##contig=\n" - '##FILTER=\n' - + info - + '##FORMAT=\n' - '##FORMAT=\n' - '##FORMAT=\n' - '##FORMAT=\n' - f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n") - - -def vep_vcf(vs): - out = vcf_head('##INFO=\n') - for gene, pos, ref, alt, cons, _ in vs: - csq = (f"{alt}|{cons}|HIGH|{gene}|ENSG{gene}|Transcript|ENST{pos}|protein_coding|YES|NM_{pos}.1|" - f"rs{pos}|0.0001|0.0001|0.0001|") - out += f"{CONTIG}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tCSQ={csq}\tGT:GQ:DP:AD\t0/1:50:30:15,15\n" - return out - - -def comphet_vcf(vs): - out = vcf_head('##INFO=\n') - pid = {"GENEC": 1, "GENET": 2, "GENEF": 3} - for i, (gene, pos, ref, alt, _, _) in enumerate(vs): - mate = vs[i + 1] if i % 2 == 0 else vs[i - 1] - tag = f"{SAMPLE}/{gene}/{pid[gene]}/{CONTIG}/{mate[1]}/{mate[2]}/{mate[3]}" - out += f"{CONTIG}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tslivar_comphet={tag}\tGT:GQ:DP:AD\t0/1:50:30:15,15\n" - return out - - -def write(out_dir, swap=False): - seq = reference() - vs = variants(seq, swap) - os.makedirs(out_dir, exist_ok=True) - for name, text in (("reads.sam", sam(seq, vs)), ("vep.vcf", vep_vcf(vs)), ("comphet.vcf", comphet_vcf(vs))): - with open(os.path.join(out_dir, name), "w") as fh: - fh.write(text) - return vs - - -if __name__ == "__main__": - args = [a for a in sys.argv[1:] if a != "--swap"] - if len(args) != 1: - sys.exit(__doc__.split("\n\n")[1]) - write(args[0], swap="--swap" in sys.argv[1:]) diff --git a/tests/fixtures/pharmcat/README.md b/tests/fixtures/pharmcat/README.md index a4451df8d..9e15c7b00 100644 --- a/tests/fixtures/pharmcat/README.md +++ b/tests/fixtures/pharmcat/README.md @@ -1,7 +1,7 @@ # PharmCAT report fixtures Real PharmCAT output for `tests/test_cpic_parser.py` (one `report-.json` per release in its `REAL_REPORTS`), trimmed with -`trim_report.py` to the fields `og pgx` reads (every gene with all +`og dev fixture trim-pharmcat` (formerly `trim_report.py`, which wrote the files below) to the fields `og pgx` reads (every gene with all its listed diplotypes and `relatedDrugs`; up to 4 matched annotations per drug guideline). @@ -16,7 +16,7 @@ with `VERSION` replaced by the PharmCAT release (for example `3.4.0`), and add the release to `REAL_REPORTS` in `tests/test_cpic_parser.py`: ```bash -python3 tests/fixtures/pharmcat/trim_report.py report.json tests/fixtures/pharmcat/report-VERSION.json +bin/og dev fixture trim-pharmcat report.json tests/fixtures/pharmcat/report-VERSION.json ``` ## Reference blocks over PharmCAT's positions longer than one base diff --git a/tests/fixtures/pharmcat/trim_report.py b/tests/fixtures/pharmcat/trim_report.py deleted file mode 100755 index 33adcc5a2..000000000 --- a/tests/fixtures/pharmcat/trim_report.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""trim_report.py: cut a PharmCAT report.json down to a test fixture. - - python3 tests/fixtures/pharmcat/trim_report.py REPORT.json OUT.json [MAX_ANNOTATIONS] - -Keeps every gene with all its listed diplotypes (sourceDiplotypes, or -recommendationDiplotypes when that is all there is) and relatedDrugs, and for -each drug of each guidance source up to MAX_ANNOTATIONS (default 4) of the -annotations PharmCAT matched, with only the fields og pgx reads. -Writes compact JSON. See README.md for how each fixture was made. -""" -import json -import sys - -src, dst = sys.argv[1], sys.argv[2] -MAX_ANN = int(sys.argv[3]) if len(sys.argv) > 3 else 4 -d = json.load(open(src)) - - -def dip(x): - if not isinstance(x, dict): return x - out = {k: x[k] for k in ("gene", "label", "phenotypes", "activityScore", "outsidePhenotype") if k in x} - for a in ("allele1", "allele2"): - if a in x: - out[a] = {k: x[a][k] for k in ("gene", "name", "function") if x[a] and k in x[a]} if x[a] else x[a] - return out - - -genes = {} -for g, v in d["genes"].items(): - genes[g] = {k: v[k] for k in ("geneSymbol", "chr", "phased", "callSource") if k in v} - genes[g]["relatedDrugs"] = [{"name": r["name"], "id": r.get("id")} for r in v.get("relatedDrugs") or []] - # sourceDiplotypes is what og pgx reads; recommendationDiplotypes only - # when sourceDiplotypes is absent, so it is kept only then. - k = "sourceDiplotypes" if v.get("sourceDiplotypes") else "recommendationDiplotypes" - if k in v: genes[g][k] = [dip(x) for x in v[k] or []] -drugs = {} -for srcname, by in d["drugs"].items(): - for name, rep in by.items(): - gls = [] - for gl in rep.get("guidelines") or []: - anns = [] - for a in (gl.get("annotations") or [])[:MAX_ANN]: - anns.append({"phenotypes": a.get("phenotypes"), "classification": a.get("classification"), - "drugRecommendation": a.get("drugRecommendation"), - "genotypes": [{"diplotypes": [{"gene": x.get("gene"), "label": x.get("label")} - for x in (gt.get("diplotypes") or []) if isinstance(x, dict)]} - for gt in (a.get("genotypes") or [])]}) - if anns: gls.append({"name": gl.get("name"), "source": gl.get("source"), "annotations": anns}) - if gls: - drugs.setdefault(srcname, {})[name] = {"name": rep.get("name"), "id": rep.get("id"), "guidelines": gls} -out = {k: d[k] for k in ("title", "timestamp", "pharmcatVersion", "dataVersion") if k in d} -out["genes"] = genes -out["drugs"] = drugs -out["_fixture_note"] = (f"Trimmed by tests/fixtures/pharmcat/trim_report.py: every gene, and up to {MAX_ANN} matched " - "annotations per drug guideline, with only the fields og pgx reads.") -json.dump(out, open(dst, "w"), separators=(",", ":")) From 7457f1621717f0472580beff12673862a4b23052 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:20:24 +0100 Subject: [PATCH 06/76] docs(changelog): og dev fixture, evidence and renovate-check Bead: pgp-bs9u.39 --- CHANGELOG.md | 5 +++++ rust/og/src/main.rs | 1 + 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd24284ac..506e771c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Python that CI and the fixtures ran outside the pipeline now runs in the Rust `og` binary. The public evidence workflow runs `og dev evidence clashes` and `og dev evidence hg002-gvcf` instead of two inline Python programs; the Renovate dry run checks what Renovate found with `og dev renovate-check`, whose list of non-image pins is now data in `scripts/ci/renovate-pins.tsv`; `og dev fixture comphet` writes the compound-het fixture of the e2e case `comphet-phasing-1-reads`, byte for byte; and `og dev fixture trim-pharmcat` trims a PharmCAT report for `tests/fixtures/pharmcat`. `tests/fixtures/comphet/make_fixture.py`, `tests/fixtures/pharmcat/trim_report.py` and `tests/smoke/vntyper_testdata.py` are gone: VNtyper's two test BAMs now come from `tests/smoke/vntyper_testdata.sh`, one HTTP range request per BAM with curl, checked by gzip against the archive's CRC-32 and size and then against the pinned sha256. On GIAB's HG002 benchmark VCF (4,048,342 records) with the chr20-22 reference, `og dev evidence hg002-gvcf` writes the same gVCF in 3.5 s and 97 MB instead of 11.8 s and 3.8 GB, because it keeps the spans the records cover instead of every covered position. No pipeline output changes. What changes on purpose: + - `og dev evidence hg002-gvcf` skips a panel site past the end of its contig. The Python read the byte after the contig, and at the end of the file wrote a reference block with an empty REF. + - `og dev renovate-check` stops with exit 2 when a pin's file is missing or its glob matches no file. The Python counted such a pin in no file, so it could never fail. + - `og dev fixture trim-pharmcat` names itself in the trimmed report's `_fixture_note`. + - The database re-check and the monthly freshness issue now run in the Rust `og` binary: `og recheck` replaces `bin/owngenome_recheck.py` and `og dev freshness` replaces `scripts/ci/freshness.py`, which is gone. `bin/owngenome_recheck.py` stays until 1.1.0 as a shim that runs `bin/og recheck` with the same arguments and says on stderr that it is deprecated, so a crontab line keeps working; [docs/recheck.md](docs/recheck.md) has the new lines. Each lookup now returns a typed answer (a date, a release, the release a held pin can move to, or "report only"), and the re-check reads that answer, not the text of the report. The lookups run on up to 8 threads with one request at a time per host, and ask the network through the host's `curl`. The freshness report is the same, byte for byte; it took 20 s and 12 MB instead of 109 s and 27 MB on one live run. On a synthetic sample with 5,000 PGS scores the re-check takes 0.05 s and 16 MB instead of 0.56 s and 33 MB, with the same `state.json` by content. What changes on purpose: - The action lines say `bin/og recheck --sample S --rerun` instead of `bin/owngenome_recheck.py --sample S --rerun`. - `--rerun` prints its own lines before the replayed command's output. The Python held its lines in a buffer, so they could come after the child's. diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 7b8f792ab..37a109075 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -69,6 +69,7 @@ const USAGE: &str = "usage: og [args] og exomiser report [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] + og dev fixture|evidence|renovate-check [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash From 8a8cb25ad55e62ca0aa3ac3fe0c8fec65a212800 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:23:02 +0100 Subject: [PATCH 07/76] test(parity): edge cases for og dev fixture trim-pharmcat Every field the trimmer keeps or drops, a non-object diplotype, a null allele, the recommendationDiplotypes fallback, non-ASCII text, and counts of 2, 0 and -1. Bead: pgp-bs9u.39 --- tests/parity/cases.tsv | 4 +++ .../dev-fixture-trim-pharmcat/edge.json | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 692b0f540..8d2905251 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -884,3 +884,7 @@ dev-fixture-trim-pharmcat all-but-last gen:pgx-fixtures.sh pharmcat-docs-example dev-evidence-clashes synthetic dev-evidence-clashes/raw.vcf.gz,dev-evidence-clashes/split.vcf.gz,dev-evidence-clashes/chrA.tsv.gz,dev-evidence-clashes/chrB.tsv.gz,dev-evidence-clashes/chrZ.tsv.gz,dev-evidence-clashes/notes.tsv.gz raw.vcf.gz split.vcf.gz . affected.tsv affected.tsv bytes PYTHON_IMAGE dev-evidence-hg002-gvcf synthetic dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites.tsv truth.vcf.gz truth.bed ref.fasta sites.tsv out.vcf out.vcf bytes PYTHON_IMAGE dev-evidence-hg002-gvcf past-the-end dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth-past-end.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites-past-end.tsv truth.vcf.gz truth-past-end.bed ref.fasta sites-past-end.tsv out.vcf out.vcf bytes PYTHON_IMAGE +dev-fixture-trim-pharmcat edge dev-fixture-trim-pharmcat/edge.json edge.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge2 dev-fixture-trim-pharmcat/edge.json edge.json out.json 2 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge0 dev-fixture-trim-pharmcat/edge.json edge.json out.json 0 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge-all-but-last dev-fixture-trim-pharmcat/edge.json edge.json out.json -1 out.json json:mask=_fixture_note PYTHON_IMAGE diff --git a/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json b/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json new file mode 100644 index 000000000..dd46aeb6b --- /dev/null +++ b/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json @@ -0,0 +1,25 @@ +{"title":"edge é","timestamp":"t","pharmcatVersion":"3.4.0","dataVersion":"d","other":{"x":1}, + "genes":{ + "CYP2D6":{"geneSymbol":"CYP2D6","chr":"chr22","phased":false,"callSource":"OUTSIDE","extra":1, + "relatedDrugs":[{"name":"codeine","id":"PA449088"},{"name":"tramadol"}], + "sourceDiplotypes":[{"gene":"CYP2D6","label":"*1/*4","phenotypes":["Intermediate Metabolizer"],"activityScore":1.0,"outsidePhenotype":true,"lookupKey":["x"], + "allele1":{"gene":"CYP2D6","name":"*1","function":"Normal function","activityValue":"1.0"},"allele2":{"name":"*4","function":"No function"}}, + "not an object"], + "recommendationDiplotypes":[{"label":"ignored because sourceDiplotypes is set"}]}, + "UGT1A1":{"geneSymbol":"UGT1A1","relatedDrugs":null,"sourceDiplotypes":[], + "recommendationDiplotypes":[{"label":"*1/*80","allele1":null,"allele2":{}}]}, + "DPYD":{"geneSymbol":"DPYD","sourceDiplotypes":[]}, + "NAT2":{"geneSymbol":"NAT2","sourceDiplotypes":null,"recommendationDiplotypes":null}}, + "drugs":{ + "CPIC Guideline Annotation":{ + "codeine":{"name":"codeine","id":"PA449088","source":"x","guidelines":[ + {"name":"g1","source":"CPIC","url":"u","annotations":[ + {"phenotypes":{"CYP2D6":"Intermediate Metabolizer"},"classification":"Moderate","drugRecommendation":"Use label-recommended dosing — caution.","implications":["dropped"], + "genotypes":[{"diplotypes":[{"gene":"CYP2D6","label":"*1/*4","x":1},"skip me",{"label":"no gene"}]},{}]}, + {"classification":"Strong"}, + {"classification":"Optional"}]}, + {"name":"g2","annotations":[]}, + {"name":"g3"}]}, + "tramadol":{"name":"tramadol","guidelines":[]}}, + "DPWG Guideline Annotation":{ + "empty":{"name":"empty","guidelines":null}}}} From 63938de53b7bf542246af55ea8ad3a31200c84fb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:23:02 +0100 Subject: [PATCH 08/76] test: plant one fault in each new og dev verb (red control, reverted next) Comphet seeded with 32, the trimmer drops activityScore, clashes needs three records, and renovate-check no longer flags an @ in a package name. Guard rust and parity must go red on this commit. Bead: pgp-bs9u.39 --- rust/crates/og-dev/src/evidence.rs | 2 +- rust/crates/og-dev/src/fixture.rs | 13 ++----------- rust/crates/og-dev/src/renovate.rs | 2 +- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index c9b76d7fc..6971f2f1b 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -119,7 +119,7 @@ fn clash_positions( n += 1; let one = |a: &[u8]| std::str::from_utf8(a).map_or(a.len(), |s| s.chars().count()) == 1; snvs += u32::from(one(c[3]) && one(c[4])); - if n >= 2 && snvs >= 1 { + if n >= 3 && snvs >= 1 { hit.insert(k); } Ok(()) diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs index bb163c9f9..0d5d9f3bf 100644 --- a/rust/crates/og-dev/src/fixture.rs +++ b/rust/crates/og-dev/src/fixture.rs @@ -200,7 +200,7 @@ pub mod comphet { /// The contig: `random.Random(31)` choosing among `ACGT` 80,000 times. pub fn reference() -> Vec { - let mut rng = Mt19937::new(31); + let mut rng = Mt19937::new(32); (0..LENGTH) .map(|_| b"ACGT"[rng.below(4) as usize]) .collect() @@ -423,16 +423,7 @@ fn diplotype(x: &Value) -> Result { let Value::Object(x) = x else { return Ok(x.clone()); }; - let mut out = pick( - x, - &[ - "gene", - "label", - "phenotypes", - "activityScore", - "outsidePhenotype", - ], - ); + let mut out = pick(x, &["gene", "label", "phenotypes", "outsidePhenotype"]); for a in ["allele1", "allele2"] { let Some(v) = x.get(a) else { continue }; let kept = match v { diff --git a/rust/crates/og-dev/src/renovate.rs b/rust/crates/og-dev/src/renovate.rs index f3355c766..4544ccb01 100644 --- a/rust/crates/og-dev/src/renovate.rs +++ b/rust/crates/og-dev/src/renovate.rs @@ -421,7 +421,7 @@ fn check(deps: &[Dep], env_text: &str, pins: &Pins) -> Vec { } else { ":".to_owned() }; - if text.contains([':', '@']) { + if text.contains(':') { fails.push(format!("names: {f}: package name {}", r(name))); } } From d6750f5033bc909f012bccffa5963e9182231e61 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:26:18 +0100 Subject: [PATCH 09/76] fix(ci): run the scanners and version checks through og dev check-personal-data.sh, check-images.sh, gen-containers-config.sh --check-versions, check-links.sh and check-pgs-labels.sh call the og dev verbs instead of their Python heredocs, and each workflow job that runs them has og. New self-test controls: a scan of no text file, a stale image exemption, a versions.env line that is not NAME="value", and a JSON true read as an effect of 1. Bead: pgp-bs9u.37 --- .github/workflows/freshness.yml | 42 +++++---- .github/workflows/guard.yml | 18 +++- .github/workflows/nextflow.yml | 2 + .github/workflows/personal-data.yml | 25 +++++- rust/crates/og-dev/src/personal_data.rs | 10 +-- scripts/ci/check-images.sh | 96 ++++---------------- scripts/ci/check-links.sh | 52 ++++++----- scripts/ci/check-personal-data.sh | 113 +++++++----------------- scripts/ci/check-pgs-labels.sh | 88 ++++++------------ scripts/ci/gen-containers-config.sh | 42 ++------- 10 files changed, 174 insertions(+), 314 deletions(-) diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml index 491342c72..2df8e31e8 100644 --- a/.github/workflows/freshness.yml +++ b/.github/workflows/freshness.yml @@ -129,10 +129,16 @@ jobs: - name: og dev freshness self-test (a bogus image, a missing tag and a bogus URL must exit non-zero) run: rust/target/release/og dev freshness --self-test - - name: check-links.sh self-test (a 404, no answer and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage) + # check-links.sh and check-pgs-labels.sh read with og dev versions + # expand and og dev pgs (rust/crates/og-dev), the og built above. + - name: check-links.sh self-test (a 404, no answer, an unreadable versions.env line and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage) + env: + OG: ${{ github.workspace }}/rust/target/release/og run: scripts/ci/check-links.sh --self-test - - name: check-pgs-labels.sh self-test (a wrong label, an empty answer and no answer must fail) + - name: check-pgs-labels.sh self-test (a wrong label, an empty answer, no answer and an effect of true must fail) + env: + OG: ${{ github.workspace }}/rust/target/release/og run: scripts/ci/check-pgs-labels.sh --self-test pull-request: @@ -158,20 +164,11 @@ jobs: echo "report=$(has 'rust/crates/og-fresh/.*|versions\.env|\.github/workflows/freshness\.yml')" } >> "$GITHUB_OUTPUT" - - name: Every download URL this pull request adds answers - env: - BASE: ${{ github.event.pull_request.base.sha }} - run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Every PGS ID's label matches the PGS Catalog - if: steps.changed.outputs.prs == 'true' - run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY" - - # og dev freshness (rust/crates/og-fresh) writes the report: the - # toolchain rust/rust-toolchain.toml pins, og built without the BAM - # reader, which the report never needs. + # og dev freshness (rust/crates/og-fresh) writes the report, and the + # link and label checks read with og dev versions expand and og dev + # pgs: the toolchain rust/rust-toolchain.toml pins, og built without + # the BAM reader, which none of them needs. - name: Install the toolchain rust/rust-toolchain.toml pins - if: steps.changed.outputs.report == 'true' working-directory: rust run: | channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) @@ -179,16 +176,26 @@ jobs: rustc --version - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - if: steps.changed.outputs.report == 'true' with: workspaces: rust key: freshness - name: Build og - if: steps.changed.outputs.report == 'true' working-directory: rust run: cargo build --release --locked -p og --no-default-features + - name: Every download URL this pull request adds answers + env: + BASE: ${{ github.event.pull_request.base.sha }} + OG: ${{ github.workspace }}/rust/target/release/og + run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Every PGS ID's label matches the PGS Catalog + if: steps.changed.outputs.prs == 'true' + env: + OG: ${{ github.workspace }}/rust/target/release/og + run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY" + - name: Report dry run (no issue is written) if: steps.changed.outputs.report == 'true' env: @@ -236,6 +243,7 @@ jobs: id: checks env: GITHUB_TOKEN: ${{ github.token }} + OG: ${{ github.workspace }}/rust/target/release/og RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | out="${RUNNER_TEMP}" diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 45a57bedd..03ad9d83b 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -33,12 +33,25 @@ jobs: # two files generated from it (conf/containers.config, docs/versions.md) # are up to date. Each script's self-test plants its faults first. images: + needs: og runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Self-test (a literal image in a script, module, workflow or doc, a process with no selector and a stale generated file must be caught) + # The scans and the run-versions check are og dev verbs + # (rust/crates/og-dev): the og the og job built. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: og-unit-tests + path: ${{ runner.temp }}/og + + - name: og for the checks (OG) + run: | + chmod 755 "$RUNNER_TEMP/og/og" + echo "OG=${RUNNER_TEMP}/og/og" >> "$GITHUB_ENV" + + - name: Self-test (a literal image in a script, module, workflow or doc, a stale exemption, a process with no selector and a stale generated file must be caught) run: scripts/ci/check-images.sh --self-test - name: Self-test of the containers.config generator (a stale tag, a missing or stray selector, a module's own container and a wrong version must be caught) @@ -125,7 +138,8 @@ jobs: # og for the unit tests (tests/og_binary.py): the tests of a ported tool # and the demo sample builder run og as a subprocess. Built once here, so - # the three shards do not each spend minutes of their 10 on it. + # the three shards do not each spend minutes of their 10 on it. The images + # job runs its og dev scans with it too. og: runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index 8f5426c6c..64b90c84f 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -363,6 +363,8 @@ jobs: # the image versions.env gives it. - name: Every process ran and states the tag of its image from versions.env if: steps.scope.outputs.run == 'true' + env: + OG: ${{ github.workspace }}/rust/target/release/og # og dev versions check-run, built above run: | cat "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml" scripts/ci/gen-containers-config.sh --check-versions "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml" diff --git a/.github/workflows/personal-data.yml b/.github/workflows/personal-data.yml index 1cc1f5315..81ace8050 100644 --- a/.github/workflows/personal-data.yml +++ b/.github/workflows/personal-data.yml @@ -19,11 +19,32 @@ concurrency: jobs: personal-data: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 20 + env: + OG: ${{ github.workspace }}/rust/target/release/og steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count must be caught) + # The scan is og dev scan personal-data (rust/crates/og-dev): the + # toolchain rust/rust-toolchain.toml pins, og built without the BAM + # reader, which the scan never needs. + - name: Install the toolchain rust/rust-toolchain.toml pins + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + key: personal-data + + - name: Build og + working-directory: rust + run: cargo build --release --locked -p og --no-default-features + + - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count, and a scan of no text file, must be caught) run: scripts/ci/check-personal-data.sh --self-test - name: No tracked file holds a home path, a private address or a genotype diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index a10289f56..520ffb921 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -488,7 +488,7 @@ mod tests { ("rs123: (C;~T)", "rsid"), ("rs4477212\t1\t82154\t~AA", "rsid"), ("HTT CAG repeats: 17/~19", "repeat"), - ("GT:REPCN 0/1:17/~19", "repeat"), + ("GT:REPCN 0~/1:17/~19", "repeat"), ] { let line = p(line); assert!(kinds(&line).contains(&kind), "{line}: {:?}", kinds(&line)); @@ -520,18 +520,18 @@ mod tests { // the number: the line still counts as labelled. assert_eq!(kinds(&p("sample7a *1/~*4")), ["diplotype"]); assert_eq!(kinds(&p("samples: *1/~*4")), ["diplotype"]); - assert_eq!(kinds(&p("10.1.2.3.")), ["address"]); + assert_eq!(kinds(&p("10.1.2.~3.")), ["address"]); assert_eq!(kinds(&p("FMR1 has 29/~31")), ["repeat"]); } #[test] fn an_allowed_example_covers_only_its_own_text() { let rules = builtin(); - let ex = "(`/mnt/user/`, `/home/username/`, etc.)"; + let ex = p("(`/mnt/u~ser/`, `/ho~me/username/`, etc.)"); let mut out = Vec::new(); - assert!(!scan_text(&mut out, &rules, &[ex], "x.md", ex.as_bytes())); + assert!(!scan_text(&mut out, &rules, &[ex.as_str()], "x.md", ex.as_bytes())); let line = p(&format!("{ex} see /home/~bob/x")); - assert!(scan_text(&mut out, &rules, &[ex], "x.md", line.as_bytes())); + assert!(scan_text(&mut out, &rules, &[ex.as_str()], "x.md", line.as_bytes())); let got = String::from_utf8(out).unwrap(); assert!(got.starts_with("FAIL home x.md:1: "), "{got}"); } diff --git a/scripts/ci/check-images.sh b/scripts/ci/check-images.sh index 792504f0b..9620755d1 100755 --- a/scripts/ci/check-images.sh +++ b/scripts/ci/check-images.sh @@ -28,6 +28,9 @@ set -euo pipefail export LC_ALL=C ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# og runs the scan and rule 4 (rust/crates/og-dev; bin/og picks the build +# for this machine); OG names another og binary. +OG=${OG:-${ROOT}/bin/og} MODE=check while [ $# -gt 0 ]; do case "$1" in @@ -55,7 +58,8 @@ tests/fixtures/ provenance notes name the image that wrote # Single literals that stay on purpose: file, image, reason. Each line allows # one occurrence: a change to the image or one more copy of it in the same -# file is reported again. +# file is reported again, and a line whose file no longer names the image +# fails until it is dropped. EXEMPT_PAIRS=' docs/08-hla-typing.md jiachenzdocker/hla-la@sha256:ecca23de6635aa85e60b4ee39dd4e15341b5febb514e5478f2b2a086f05a447c HLA-LA is not a pipeline step docs/stylesheets/theme.css quay.io/biocontainers/expansionhunter:5.0.0 example in a CSS comment @@ -74,90 +78,12 @@ scan() { list=$(cd "$ROOT" && find $SCAN -type f 2>/dev/null || true) fi [ -n "$list" ] || { echo "FAIL: found no file to scan under ${ROOT}"; return 1; } - # The file list goes in as an argument: stdin carries the program. - python3 - "$ROOT" "$EXEMPT_FILES" "$EXEMPT_PAIRS" "$list" <<'PY' -import collections, os, re, sys -root, exempt_files, exempt_pairs = sys.argv[1], sys.argv[2], sys.argv[3] -files = [f for f in sys.argv[4].split("\n") if f] -if not files: - print("FAIL: no file to scan") - sys.exit(1) -skip = [l.split()[0] for l in exempt_files.splitlines() if l.strip()] -# Each line allows one occurrence; list a pair twice to allow two. -pairs = collections.Counter(tuple(l.split()[:2]) for l in exempt_pairs.splitlines() if l.strip()) - -# Docker Hub official images have no slash: those versions.env uses, plus -# the usual base images. -env = open(os.path.join(root, "versions.env")).read() -bare = set(re.findall(r'^[A-Z0-9_]+_IMAGE="([^"/:@]+)[:@]', env, re.M)) -bare |= {"ubuntu", "debian", "alpine", "busybox", "centos", "rockylinux", "fedora", - "python", "node", "perl", "r-base", "openjdk", "eclipse-temurin", "golang", "rust"} - -# Any tag: a version, or a word such as latest or stable. -TAG = r'(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-f]{64})' -LEAD = r'(? 0: - left[(f, image)] -= 1 - continue - print("FAIL: %s:%d names the image %s; use its *_IMAGE variable from versions.env" % (f, n, image)) - bad = 1 -print("scanned %d files" % len(files)) -sys.exit(bad) -PY + "$OG" dev scan images --root "$ROOT" --exempt-files "$EXEMPT_FILES" --exempt-pairs "$EXEMPT_PAIRS" <<<"$list" } # coupled: rule 4. Prints one line per tool, exit 1 if one disagrees. coupled() { - python3 - "${ROOT}/versions.env" <<'PY' -import re, sys -env = open(sys.argv[1]).read() -def line(var): - m = re.search(r'^%s="([^"]*)"(.*)$' % var, env, re.M) - return (m.group(1), m.group(2)) if m else (None, "") -def release(var): - value = line(var)[0] or "" - tag = value.rsplit("/", 1)[-1] - return tag.split(":", 1)[1].split("--")[0] if ":" in tag else None -image, comment = line("MINIMAP2_SAMTOOLS_IMAGE") -if image is None: - print("FAIL: MINIMAP2_SAMTOOLS_IMAGE is not set in versions.env") - sys.exit(1) -held = {k: v.rstrip(".") for k, v in re.findall(r"\b(minimap2|samtools)=([0-9][0-9A-Za-z.]*)", comment)} -bad = 0 -for tool, var in (("minimap2", "MINIMAP2_IMAGE"), ("samtools", "SAMTOOLS_IMAGE")): - want, got = release(var), held.get(tool) - if want and got == want: - print("OK: MINIMAP2_SAMTOOLS_IMAGE holds %s %s, the release of %s" % (tool, got, var)) - else: - print("FAIL: MINIMAP2_SAMTOOLS_IMAGE says %s=%s in its versions.env comment, but %s is %s: pin a" - " BioContainers mulled image with %s %s and name it in that comment (docs/updating.md, Held pins)" - % (tool, got or "(nothing)", var, want or "(unset)", tool, want or "?")) - bad = 1 -sys.exit(bad) -PY + "$OG" dev versions coupled --env "${ROOT}/versions.env" } check() { @@ -191,6 +117,9 @@ self_test() { fi } + # The copies have no bin/.og: they run this tree's og. + export OG + copy clean rc=0; out=$("${tmp}/clean/scripts/ci/check-images.sh" 2>&1) || rc=$? [ "$rc" -eq 0 ] || { echo "self-test: the unchanged copy failed:"; printf '%s\n' "$out"; fail=1; } @@ -230,6 +159,11 @@ self_test() { echo '/* Then quay.io/biocontainers/expansionhunter:5.0.0 again. */' >> "${tmp}/exempt-twice/docs/stylesheets/theme.css" expect exempt-twice '^FAIL: docs/stylesheets/theme\.css:[0-9]+ names the image quay\.io/biocontainers/expansionhunter:5\.0\.0;' + # An exempt literal that left its file: the exemption is stale. + copy exempt-stale + sed -i.bak 's|quay.io/biocontainers/expansionhunter:5.0.0|an example image|' "${tmp}/exempt-stale/docs/stylesheets/theme.css" + expect exempt-stale '^FAIL: docs/stylesheets/theme\.css no longer names quay\.io/biocontainers/expansionhunter:5\.0\.0; drop its line' + # The bwa-mem2 tag scripts/ci/settle-doubts.sh once repeated: it reads # BWAMEM2_IMAGE from versions.env now, so that literal must fail again. copy settle-literal diff --git a/scripts/ci/check-links.sh b/scripts/ci/check-links.sh index 874403fb8..ea234bbe6 100755 --- a/scripts/ci/check-links.sh +++ b/scripts/ci/check-links.sh @@ -31,6 +31,10 @@ set -euo pipefail REPO_ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# og fills the versions.env variables (og dev versions expand, +# rust/crates/og-dev; bin/og picks the build for this machine); OG names +# another og binary. +OG=${OG:-${REPO_ROOT}/bin/og} JOBS=${LINKS_JOBS:-8} # Prints "URLfile:line" for every candidate URL under $1, before variable @@ -58,31 +62,10 @@ raw_urls() { } # Fills ${NAME} and $NAME from the versions.env under $1, reading stdin, and -# drops trailing punctuation. +# drops trailing punctuation. A versions.env line that is not a plain +# NAME="value" fails the check instead of being left out. expand_vars() { - local env=${1}/versions.env - # shellcheck disable=SC2016 # Python source, not shell - python3 -c ' -import re, sys -vals = {} -try: - for line in open(sys.argv[1]): - m = re.match(r"^([A-Z][A-Z0-9_]*)=\"([^\"]*)\"", line) - if m: - vals[m.group(1)] = m.group(2) -except FileNotFoundError: - pass -def sub(m): - name = m.group(1) or m.group(2) - return vals.get(name, m.group(0)) -for line in sys.stdin: - url, loc = line.rstrip("\n").split("\t", 1) - url = re.sub(r"\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)", sub, url) - # Trailing punctuation, and the brace that closes ${VAR:-https://...}. - while url and (url[-1] in ".,;:]" or (url[-1] == "}" and url.count("}") > url.count("{"))): - url = url[:-1] - print(url + "\t" + loc) -' "$env" + "$OG" dev versions expand --env "${1}/versions.env" } # Prints "URLlocations" (locations joined by ", "), one line per URL. @@ -132,7 +115,7 @@ probe() { # ROOT's repository), only the URLs that tree did not already have. check_tree() { local root=$1 since=${2:-} list tmp n=0 nskip=0 nbad=0 nold=0 url loc code - list=$(collect "$root") + list=$(collect "$root") || { echo "ERROR: could not read the URLs under ${root}" >&2; return 1; } tmp=$(mktemp -d) : > "${tmp}/check"; : > "${tmp}/skip"; : > "${tmp}/old" if [ -z "$list" ]; then @@ -147,7 +130,11 @@ check_tree() { rm -rf "$tmp" return 1 fi - collect "${tmp}/base" | cut -f1 > "${tmp}/old" + if ! collect "${tmp}/base" | cut -f1 > "${tmp}/old"; then + echo "ERROR: could not read the URLs of ${since}" >&2 + rm -rf "$tmp" + return 1 + fi fi while IFS=$'\t' read -r url loc; do [ -n "$url" ] || continue @@ -299,13 +286,24 @@ EOF echo "SELF-TEST FAIL: --since did not request exactly the added URL"; fail=1 fi + # A versions.env line that is not NAME="value" fails: the URLs built from + # it would otherwise go unfilled and be skipped. + mkdir -p "${d}/strict/scripts" + echo "wget ${good}/blob/main/README.md" > "${d}/strict/scripts/c.sh" + # shellcheck disable=SC2016 # the ${...} is the text planted in the file + echo 'TESTREPO="${OTHER}"' > "${d}/strict/versions.env" + rc=0; out=$(check_tree "${d}/strict" 2>&1) || rc=$? + if [ "$rc" -eq 0 ] || ! grep -qF 'versions.env:1: not a NAME="value" line' <<<"$out"; then + echo "$out"; echo "SELF-TEST FAIL: a versions.env line with an expansion was read"; fail=1 + fi + # An empty tree is a failure. mkdir -p "${d}/empty" rc=0; out=$(check_tree "${d}/empty" 2>&1) || rc=$? [ "$rc" -ne 0 ] || { echo "SELF-TEST FAIL: an empty tree passed"; fail=1; } if [ "$fail" -eq 0 ]; then - echo "SELF-TEST OK: a 404, no answer and an empty tree fail; comments, prose and scripts/ci are left out; \${VAR} is filled from versions.env; --since requests only added URLs." + echo "SELF-TEST OK: a 404, no answer, an unreadable versions.env line and an empty tree fail; comments, prose and scripts/ci are left out; \${VAR} is filled from versions.env; --since requests only added URLs." fi return "$fail" } diff --git a/scripts/ci/check-personal-data.sh b/scripts/ci/check-personal-data.sh index 534c22dc8..4127d3799 100755 --- a/scripts/ci/check-personal-data.sh +++ b/scripts/ci/check-personal-data.sh @@ -36,6 +36,10 @@ # the repository and point PERSONAL_DATA_PATTERNS at it; each match is # reported as kind "own". # +# The scan is og dev scan personal-data (rust/crates/og-dev); bin/og runs +# the build scripts/setup.sh --og puts in bin/.og for this machine, and OG +# names another og binary. +# # As a pre-commit hook, checking the staged files: # cat > .git/hooks/pre-commit <<'HOOK' # #!/bin/sh @@ -47,6 +51,8 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# og runs the scan (bin/og picks the build for this machine); OG overrides it. +OG=${OG:-${ROOT}/bin/og} # file (exact path or glob) text on the line that makes it a documented example # shellcheck disable=SC2016 # the entries are text, nothing expands @@ -80,89 +86,16 @@ tests/parity/inputs/pgx-refblock-fill/* built from the public PharmCAT data of tests/parity/golden/pgx-refblock-fill/* the output og writes on those inputs ' +# scan [FILE...]: og dev scan personal-data (rust/crates/og-dev) with the +# lists above; with no FILE, every file git ls-files lists under ROOT. A +# scan that read no text file fails: it checked nothing. scan() { - python3 - "$ROOT" "$ALLOW" "$SKIP" "${PERSONAL_DATA_PATTERNS:-}" "${CACHED:-}" "$@" <<'PY' -import fnmatch, os, re, subprocess, sys -root, allow_text, skip_text, own_file, cached, args = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6:] -skip = [l.split()[0] for l in skip_text.splitlines() if l.strip()] -allow = [] -for l in allow_text.splitlines(): - if l.strip(): - path, _, text = l.strip().partition(" ") - allow.append((path, text.strip())) - -if args: - files = args -else: - out = subprocess.run(["git", "-C", root, "ls-files", "-z"], check=True, capture_output=True).stdout - files = [os.path.join(root, f) for f in out.decode().split("\0") if f] -if not files: - print("FAIL: no file to scan") - sys.exit(1) - -# A label may carry a number after a separator: sample7, sample_01, donor-3. -LABEL = r'(?:(?i:(? "${tmp}/binary.bin" + rc=0; out=$(scan "${tmp}/binary.bin" 2>&1) || rc=$? + if [ "$rc" -ne 1 ] || ! grep -q '^FAIL: no text file was scanned' <<<"$out"; then + echo "self-test: a scan of no text file passed (exit ${rc}):"; printf '%s\n' "$out"; fail=1 + else + echo "self-test: caught: $(grep '^FAIL' <<<"$out")" + fi + # ROOT reached through a symbolic link still places the file below it, so + # the file's allowed example applies. + ln -s "${tmp}/tree" "${tmp}/link" + # shellcheck disable=SC2016 # the backticks are text + plant tree/CONTRIBUTING.md 'Personal file paths (`/mnt/user/`, `/home/username/`, etc.)' + rc=0; out=$(ROOT="${tmp}/link" scan "${tmp}/tree/CONTRIBUTING.md" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || { echo "self-test: an allowed example was reported under a linked ROOT:"; printf '%s\n' "$out"; fail=1; } # A pattern of your own, from PERSONAL_DATA_PATTERNS. printf '%s\n' 'Report for Jane Example' > "${tmp}/own.md" printf '%s\n' '\bJane Example\b' > "${tmp}/patterns" diff --git a/scripts/ci/check-pgs-labels.sh b/scripts/ci/check-pgs-labels.sh index ba5f84032..6e719c3d3 100755 --- a/scripts/ci/check-pgs-labels.sh +++ b/scripts/ci/check-pgs-labels.sh @@ -40,6 +40,9 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# og reads the API's answers (og dev pgs, rust/crates/og-dev; bin/og picks +# the build for this machine); OG names another og binary. curl asks. +OG=${OG:-${ROOT}/bin/og} # Prints "IDlabel" for every entry of the map in $1. read_map() { @@ -62,31 +65,12 @@ trait_of() { local id=$1 body api=${PGS_API:-https://www.pgscatalog.org/rest/score} body=$(curl -fsS --connect-timeout 10 --max-time 30 --retry "${PGS_RETRY:-3}" --retry-delay 5 --retry-all-errors \ -H 'Accept: application/json' "${api}/${id}") || return 1 - ID="$id" python3 -c ' -import json, os, sys -try: - d = json.loads(sys.stdin.read()) -except ValueError: - sys.exit(1) -if not isinstance(d, dict) or d.get("id") != os.environ["ID"]: - sys.exit(1) -t = (d.get("trait_reported") or "").strip() -if not t: - sys.exit(1) -print(t) -' <<<"$body" + "$OG" dev pgs trait --id "$id" <<<"$body" } # Exit 0 when every word of label $1 appears in trait $2. label_matches() { - python3 - "$1" "$2" <<'EOF' -import re, sys -def words(s): - s = s.lower().replace("’", "").replace("'", "") - return set(re.sub(r"[^a-z0-9]+", " ", s).split()) -label, trait = words(sys.argv[1]), words(sys.argv[2]) -sys.exit(0 if label and label <= trait else 1) -EOF + "$OG" dev pgs label-match "$1" "$2" } # Prints "IDppm_idancestryeffect" for every row of a score @@ -100,49 +84,15 @@ read_metrics() { } # Exit 0 when evaluation $2 of the PGS Catalog belongs to score $1, was -# measured in samples of ancestry $3 and reports effect $4 ("OR 1.61"). -# Prints what the catalog says, or why it does not match, on stdout. An -# API that fails or answers without that id is exit 2. +# measured in samples of ancestry $3 and reports effect $4 ("OR 1.61"; a +# JSON true is not the number 1). Prints what the catalog says, or why it +# does not match, on stdout. An API that fails or answers without that id +# is exit 2. metric_matches() { local id=$1 ppm=$2 anc=$3 eff=$4 body api=${PGS_PERF_API:-https://www.pgscatalog.org/rest/performance} body=$(curl -fsS --connect-timeout 10 --max-time 30 --retry "${PGS_RETRY:-3}" --retry-delay 5 --retry-all-errors \ -H 'Accept: application/json' "${api}/${ppm}") || { echo "(no answer)"; return 2; } - ID="$id" PPM="$ppm" ANC="$anc" EFF="$eff" python3 -c ' -import json, os, re, sys -# The PGS Catalog ancestry_broad terms of each 1000 Genomes group pgsc_calc -# reports (its ancestry_distribution uses the same codes). -GROUPS = { - "EUR": {"European"}, - "AFR": {"African unspecified", "African American or Afro-Caribbean", "Sub-Saharan African"}, - "AMR": {"Hispanic or Latin American"}, - "EAS": {"East Asian"}, - "SAS": {"South Asian"}, -} -def group(term): - for g, terms in GROUPS.items(): - if term in terms: - return g - return None -try: - d = json.loads(sys.stdin.read()) -except ValueError: - print("(not JSON)"); sys.exit(2) -if not isinstance(d, dict) or d.get("id") != os.environ["PPM"]: - print("(no such evaluation)"); sys.exit(2) -samples = (d.get("sampleset") or {}).get("samples") or [] -terms = [s.get("ancestry_broad") or "" for s in samples] -found = {group(t) for t in terms} -effects = [(e.get("name_short"), e.get("estimate")) for e in (d.get("performance_metrics") or {}).get("effect_sizes") or []] -said = "%s, %s, %s" % (d.get("associated_pgs_id"), "; ".join(terms) or "no samples", - "; ".join("%s %s" % e for e in effects) or "no effect size") -print(said) -m = re.fullmatch(r"(OR|HR) ([0-9]+(?:\.[0-9]+)?)", os.environ["EFF"]) -ok = (d.get("associated_pgs_id") == os.environ["ID"] - and os.environ["ANC"] in GROUPS and found == {os.environ["ANC"]} - and m is not None - and any(n == m.group(1) and isinstance(v, (int, float)) and abs(v - float(m.group(2))) < 1e-9 for n, v in effects)) -sys.exit(0 if ok else 1) -' <<<"$body" + "$OG" dev pgs metric-match --id "$id" --ppm "$ppm" --ancestry "$anc" --effect "$eff" <<<"$body" } # The evaluations table of file $1; exit 1 when one is wrong. A score's @@ -308,8 +258,24 @@ self_test() { echo "SELF-TEST FAIL: an evaluation was judged wrongly:"; echo "$out"; fail=1 fi + # Control 7: an evaluation whose effect is the JSON value true is not an + # effect of 1. The answers are files (curl reads file:// URLs), so this + # control needs no network. + mkdir -p "${d}/perf" + printf '%s' '{"id": "PPM000001", "associated_pgs_id": "PGS000018", "sampleset": {"samples": [{"ancestry_broad": "European"}]}, "performance_metrics": {"effect_sizes": [{"name_short": "OR", "estimate": true}]}}' > "${d}/perf/PPM000001" + rc=0; out=$(PGS_PERF_API="file://${d}/perf" metric_matches PGS000018 PPM000001 EUR 'OR 1') || rc=$? + if [ "$rc" -ne 1 ]; then + echo "SELF-TEST FAIL: an effect of true was read as OR 1 (exit ${rc}): ${out}"; fail=1 + fi + sed 's/true/1.0/' "${d}/perf/PPM000001" > "${d}/perf/PPM000002" + sed -i.bak 's/PPM000001/PPM000002/' "${d}/perf/PPM000002" + rc=0; out=$(PGS_PERF_API="file://${d}/perf" metric_matches PGS000018 PPM000002 EUR 'OR 1') || rc=$? + if [ "$rc" -ne 0 ]; then + echo "SELF-TEST FAIL: an effect of 1.0 was not read as OR 1 (exit ${rc}): ${out}"; fail=1 + fi + if [ "$fail" -eq 0 ]; then - echo "SELF-TEST OK: wrong label (list and map formats), old format, empty answer, no answer, empty map, and a wrong PPM id, ancestry or effect or an empty entry are all caught." + echo "SELF-TEST OK: wrong label (list and map formats), old format, empty answer, no answer, empty map, a wrong PPM id, ancestry or effect, an empty entry and an effect of true are all caught." fi return "$fail" } diff --git a/scripts/ci/gen-containers-config.sh b/scripts/ci/gen-containers-config.sh index 572001f2a..743f763a7 100755 --- a/scripts/ci/gen-containers-config.sh +++ b/scripts/ci/gen-containers-config.sh @@ -172,6 +172,10 @@ PRS PGSC_CALC_VERSION sha256=PGSC_CALC_SHA256 pgscatalog_utils=PGSC_UTILS_IMAG ' ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# og reads a run's versions (rust/crates/og-dev; bin/og picks the build for +# this machine); OG names another og binary. Set before --root, so a check +# of another tree still runs this tree's og. +OG=${OG:-${ROOT}/bin/og} MODE="write" ARGS=() while [ $# -gt 0 ]; do @@ -272,43 +276,7 @@ check() { # its versions.yml block, the tag of the image it ran in (task.container). # Compare that line with the image conf/containers.config gives the process. check_versions() { - python3 - "$CONFIG" "$@" <<'PY' -import re, sys -config, yml, expect = sys.argv[1], sys.argv[2], sys.argv[3:] -text = open(config).read() -images = dict(re.findall(r"withName: '(\w+)' \{ container = '([^']+)' \}", text)) -# A process that runs on the host states the release of the pipeline it starts. -images.update(re.findall(r"withName: '(\w+)' \{ ext\.pipeline_version = '([^']+)';", text)) -def tag(image): - return re.sub(r"^[^:@]+[:@]", "", image, count=1) -first, proc = {}, None -for line in open(yml): - m = re.match(r'^"([^"]+)":\s*$', line) - if m: - proc = m.group(1).split(":")[-1] - continue - m = re.match(r"^\s+([\w.-]+):\s*(.*?)\s*$", line) - if m and proc and proc not in first: - first[proc] = (m.group(1), m.group(2)) -fail = 0 -if not first: - print("FAIL: %s has no process block" % yml) - fail = 1 -for p in sorted(set(expect or images) | set(first)): - if p not in images: - print("FAIL %-20s has no selector in conf/containers.config" % p) - fail = 1 - elif p not in first: - if p in (expect or images): - print("FAIL %-20s missing from %s" % (p, yml)) - fail = 1 - elif first[p][1] != tag(images[p]): - print("FAIL %-20s %s: %s, want %s (%s)" % (p, first[p][0], first[p][1], tag(images[p]), images[p])) - fail = 1 - else: - print("OK %-20s %s: %s" % (p, first[p][0], first[p][1])) -sys.exit(fail) -PY + "$OG" dev versions check-run --config "$CONFIG" "$@" } # self_test: plant each kind of fault in a copy of this tree and require From 1d6e3b3dbab629dd0567ab9fcc681415c5fba97c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:29:25 +0100 Subject: [PATCH 10/76] style(og): rustfmt the og dev scanners Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 63 +++++++++++++++---- rust/crates/og-dev/src/lookaround.rs | 11 ++-- rust/crates/og-dev/src/personal_data.rs | 44 ++++++++++--- rust/crates/og-dev/src/pgs.rs | 82 ++++++++++++++++++++----- rust/crates/og-dev/src/versions.rs | 59 ++++++++++++++---- 5 files changed, 205 insertions(+), 54 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 91ded0e7f..34f2d540d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -73,8 +73,10 @@ static URL: LazyLock = LazyLock::new(|| Regex::new(r"[a-z][a-z0-9+.-]*://[^\s\x1c-\x1f]+").expect("a valid pattern")); /// A path to a file, such as modules/local/vep/main.nf:23, is not an image. static FILE: LazyLock = LazyLock::new(|| { - Regex::new(r"\.(nf|sh|md|py|config|ya?ml|json|txt|tsv|csv|vcf|gz|html|css|js|R|r):[0-9]+(-[0-9]+)?$") - .expect("a valid pattern") + Regex::new( + r"\.(nf|sh|md|py|config|ya?ml|json|txt|tsv|csv|vcf|gz|html|css|js|R|r):[0-9]+(-[0-9]+)?$", + ) + .expect("a valid pattern") }); fn bare_names(pins: &[Pin]) -> BTreeSet { @@ -99,8 +101,11 @@ fn patterns(bare: &BTreeSet) -> [Rule; 2] { Some(lead), ) .expect("a valid pattern"), - Rule::new(&format!(r"((?:{}){TAG}){AHEAD}", names.join("|")), Some(lead)) - .expect("a valid pattern"), + Rule::new( + &format!(r"((?:{}){TAG}){AHEAD}", names.join("|")), + Some(lead), + ) + .expect("a valid pattern"), ] } @@ -219,7 +224,11 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", - if *n > 1 { format!(" {n} more times") } else { String::new() } + if *n > 1 { + format!(" {n} more times") + } else { + String::new() + } ); bad = true; } @@ -267,16 +276,25 @@ static HELD: LazyLock = LazyLock::new(|| { /// The check on parsed pins; prints one line per tool. pub fn coupled(pins: &[Pin], out: &mut dyn Write) -> bool { let Some(image) = versions_env::get(pins, "MINIMAP2_SAMTOOLS_IMAGE") else { - let _ = writeln!(out, "FAIL: MINIMAP2_SAMTOOLS_IMAGE is not set in versions.env"); + let _ = writeln!( + out, + "FAIL: MINIMAP2_SAMTOOLS_IMAGE is not set in versions.env" + ); return false; }; let mut held: BTreeMap<&str, &str> = BTreeMap::new(); for c in HELD.captures_iter(&image.rest) { - let (tool, v) = (c.get(1).map_or("", |m| m.as_str()), c.get(2).map_or("", |m| m.as_str())); + let (tool, v) = ( + c.get(1).map_or("", |m| m.as_str()), + c.get(2).map_or("", |m| m.as_str()), + ); held.insert(tool, v.trim_end_matches('.')); } let mut ok = true; - for (tool, var) in [("minimap2", "MINIMAP2_IMAGE"), ("samtools", "SAMTOOLS_IMAGE")] { + for (tool, var) in [ + ("minimap2", "MINIMAP2_IMAGE"), + ("samtools", "SAMTOOLS_IMAGE"), + ] { let want = versions_env::get(pins, var).and_then(|p| release(&p.value)); let got = held.get(tool).copied(); match (want, got) { @@ -337,7 +355,15 @@ mod tests { String::from_utf8(out) .unwrap() .lines() - .map(|l| l.split(" names the image ").nth(1).unwrap().split(';').next().unwrap().to_owned()) + .map(|l| { + l.split(" names the image ") + .nth(1) + .unwrap() + .split(';') + .next() + .unwrap() + .to_owned() + }) .collect() } @@ -349,12 +375,21 @@ mod tests { #[test] fn literals_are_found_as_python_finds_them() { - assert_eq!(found(&p("docker run staphb/bcftools:~1.21 x")), [p("staphb/bcftools:~1.21")]); - assert_eq!(found(&p("runs staphb/samtools:~1.19.")), [p("staphb/samtools:~1.19")]); + assert_eq!( + found(&p("docker run staphb/bcftools:~1.21 x")), + [p("staphb/bcftools:~1.21")] + ); + assert_eq!( + found(&p("runs staphb/samtools:~1.19.")), + [p("staphb/samtools:~1.19")] + ); assert_eq!(found(&p("// container python:~3.12")), [p("python:~3.12")]); assert_eq!(found(&p("uses: docker://alpine:~3.8")), [p("alpine:~3.8")]); // A tag followed by a slash backs off to the part before a dot. - assert_eq!(found(&p("x staphb/samtools:~1.2/x")), [p("staphb/samtools:~1")]); + assert_eq!( + found(&p("x staphb/samtools:~1.2/x")), + [p("staphb/samtools:~1")] + ); for clean in [ "see https://quay.io/repository/biocontainers/samtools:1.0", "modules/local/vep/main.nf:23", @@ -386,7 +421,9 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); assert!(!ok); assert!( - out.contains(&p("FAIL: a.md no longer names quay.io/x/y:~3; drop its line")), + out.contains(&p( + "FAIL: a.md no longer names quay.io/x/y:~3; drop its line" + )), "{out}" ); assert!(!out.contains(&p("y:~2;")), "{out}"); diff --git a/rust/crates/og-dev/src/lookaround.rs b/rust/crates/og-dev/src/lookaround.rs index d40d83b0d..d8dc7cce9 100644 --- a/rust/crates/og-dev/src/lookaround.rs +++ b/rust/crates/og-dev/src/lookaround.rs @@ -38,9 +38,9 @@ impl Rule { let caps = self.re.captures_at(line, pos)?; let m = caps.get(1).or_else(|| caps.get(0))?; let (start, end) = (m.start(), m.end()); - let refused = self.not_after.is_some_and(|bad| { - line[..start].chars().next_back().is_some_and(bad) - }); + let refused = self + .not_after + .is_some_and(|bad| line[..start].chars().next_back().is_some_and(bad)); if refused { pos = next_char(line, start); continue; @@ -64,7 +64,10 @@ impl Rule { /// The byte offset of the character after `i`, or one past the end. fn next_char(s: &str, i: usize) -> usize { - s[i..].chars().next().map_or(s.len() + 1, |c| i + c.len_utf8()) + s[i..] + .chars() + .next() + .map_or(s.len() + 1, |c| i + c.len_utf8()) } #[cfg(test)] diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 520ffb921..33cf3ef5b 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -138,12 +138,18 @@ fn builtin() -> Vec { }, Check { kind: "rsid", - rule: rule(r"\brs[0-9]+\t[0-9XYMT]{1,2}\t[0-9]+\t[ACGTDI-]{1,2}\b", None), + rule: rule( + r"\brs[0-9]+\t[0-9XYMT]{1,2}\t[0-9]+\t[ACGTDI-]{1,2}\b", + None, + ), cond: None, }, Check { kind: "repeat", - rule: rule(r"([0-9]{1,4}/[0-9]{1,4})(?:[^0-9/]|$)", Some(digit_dot_slash)), + rule: rule( + r"([0-9]{1,4}/[0-9]{1,4})(?:[^0-9/]|$)", + Some(digit_dot_slash), + ), cond: Some(&*REPCN), }, // Python: \b(?:GENES)\b[^/\n]{0,40}?(? Result, String> { /// The staged blob of each path (relative to `root`), from one /// `git cat-file --batch`. A path with no blob in the index is absent. fn staged(root: &Path, rels: &[String]) -> Result>, String> { - let wanted: Vec<&String> = rels.iter().filter(|r| !r.starts_with('/') && !r.contains('\n')).collect(); + let wanted: Vec<&String> = rels + .iter() + .filter(|r| !r.starts_with('/') && !r.contains('\n')) + .collect(); let mut blobs = HashMap::new(); if wanted.is_empty() { return Ok(blobs); @@ -345,7 +354,9 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da .flat_map(|t| line.match_indices(*t).map(|(a, s)| (a, a + s.len()))) .collect(); for c in rules { - if c.cond.is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) { + if c.cond + .is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) + { continue; } let leaked = c @@ -385,7 +396,10 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { .filter(|(_, rel)| !a.skip.iter().any(|g| fnmatch(rel, g))) .collect(); let blobs = if a.cached { - staged(&root, &todo.iter().map(|(_, r)| r.clone()).collect::>())? + staged( + &root, + &todo.iter().map(|(_, r)| r.clone()).collect::>(), + )? } else { HashMap::new() }; @@ -458,7 +472,9 @@ mod tests { let rules = builtin(); let mut out = Vec::new(); for c in &rules { - if c.cond.is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) { + if c.cond + .is_some_and(|cs| !cs.iter().any(|r| r.is_match(line))) + { continue; } if c.rule.is_match(line) { @@ -529,9 +545,21 @@ mod tests { let rules = builtin(); let ex = p("(`/mnt/u~ser/`, `/ho~me/username/`, etc.)"); let mut out = Vec::new(); - assert!(!scan_text(&mut out, &rules, &[ex.as_str()], "x.md", ex.as_bytes())); + assert!(!scan_text( + &mut out, + &rules, + &[ex.as_str()], + "x.md", + ex.as_bytes() + )); let line = p(&format!("{ex} see /home/~bob/x")); - assert!(scan_text(&mut out, &rules, &[ex.as_str()], "x.md", line.as_bytes())); + assert!(scan_text( + &mut out, + &rules, + &[ex.as_str()], + "x.md", + line.as_bytes() + )); let got = String::from_utf8(out).unwrap(); assert!(got.starts_with("FAIL home x.md:1: "), "{got}"); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 3bc7acd3c..14639f36f 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -107,16 +107,21 @@ static EFFECT: LazyLock = /// What the catalog says of an evaluation, and whether it matches. Err is /// the reason the answer is not an evaluation with that id. -pub fn metric(body: &str, id: &str, ppm: &str, anc: &str, eff: &str) -> Result<(String, bool), String> { +pub fn metric( + body: &str, + id: &str, + ppm: &str, + anc: &str, + eff: &str, +) -> Result<(String, bool), String> { let d = loads(body).map_err(|_| "(not JSON)".to_owned())?; if !matches!(d, Value::Object(_)) || !is_str(field(&d, "id"), ppm) { return Err("(no such evaluation)".into()); } let odd = || "(not an evaluation the check can read)".to_owned(); let empty = Value::Null; - let or_empty = |v: Option<&'_ Value>| -> Value { - v.filter(|v| truthy(v)).cloned().unwrap_or(Value::Null) - }; + let or_empty = + |v: Option<&'_ Value>| -> Value { v.filter(|v| truthy(v)).cloned().unwrap_or(Value::Null) }; let samples = match or_empty(field(&or_empty(field(&d, "sampleset")), "samples")) { Value::Null => Vec::new(), Value::Array(a) => a, @@ -134,7 +139,10 @@ pub fn metric(body: &str, id: &str, ppm: &str, anc: &str, eff: &str) -> Result<( } } let found: std::collections::BTreeSet> = terms.iter().map(|t| group(t)).collect(); - let sizes = match or_empty(field(&or_empty(field(&d, "performance_metrics")), "effect_sizes")) { + let sizes = match or_empty(field( + &or_empty(field(&d, "performance_metrics")), + "effect_sizes", + )) { Value::Null => Vec::new(), Value::Array(a) => a, _ => return Err(odd()), @@ -159,8 +167,16 @@ pub fn metric(body: &str, id: &str, ppm: &str, anc: &str, eff: &str) -> Result<( let said = format!( "{}, {}, {}", show(pgs)?, - if terms_said.is_empty() { "no samples" } else { &terms_said }, - if effects_said.is_empty() { "no effect size" } else { &effects_said }, + if terms_said.is_empty() { + "no samples" + } else { + &terms_said + }, + if effects_said.is_empty() { + "no effect size" + } else { + &effects_said + }, ); let ok = is_str(Some(pgs), id) && group_named(anc) @@ -259,21 +275,40 @@ mod tests { #[test] fn trait_needs_the_same_id_and_a_trait() { assert_eq!( - trait_of(r#"{"id": "PGS000018", "trait_reported": " Coronary artery disease "}"#, "PGS000018").as_deref(), + trait_of( + r#"{"id": "PGS000018", "trait_reported": " Coronary artery disease "}"#, + "PGS000018" + ) + .as_deref(), Some("Coronary artery disease") ); assert_eq!(trait_of("{}", "PGS000018"), None); - assert_eq!(trait_of(r#"{"id": "PGS000020", "trait_reported": "x"}"#, "PGS000018"), None); - assert_eq!(trait_of(r#"{"id": "PGS000018", "trait_reported": ""}"#, "PGS000018"), None); + assert_eq!( + trait_of(r#"{"id": "PGS000020", "trait_reported": "x"}"#, "PGS000018"), + None + ); + assert_eq!( + trait_of(r#"{"id": "PGS000018", "trait_reported": ""}"#, "PGS000018"), + None + ); assert_eq!(trait_of("not json", "PGS000018"), None); } #[test] fn labels_match_by_words() { - assert!(label_matches("Type 2 diabetes (T2D)", "Type 2 diabetes (T2D)")); + assert!(label_matches( + "Type 2 diabetes (T2D)", + "Type 2 diabetes (T2D)" + )); assert!(label_matches("type_2_diabetes", "Type 2 diabetes (T2D)")); - assert!(label_matches("Late-onset Alzheimer\u{2019}s disease", "Late-onset Alzheimer's disease")); - assert!(!label_matches("inflammatory_bowel_disease", "Type 2 diabetes (T2D)")); + assert!(label_matches( + "Late-onset Alzheimer\u{2019}s disease", + "Late-onset Alzheimer's disease" + )); + assert!(!label_matches( + "inflammatory_bowel_disease", + "Type 2 diabetes (T2D)" + )); assert!(!label_matches("", "anything")); } @@ -294,10 +329,19 @@ mod tests { ("PGS000018", "XYZ", "HR 1.53"), ("PGS000018", "EUR", "HR"), ] { - assert!(!metric(ANSWER, id, "PPM000591", anc, eff).unwrap().1, "{id} {anc} {eff}"); + assert!( + !metric(ANSWER, id, "PPM000591", anc, eff).unwrap().1, + "{id} {anc} {eff}" + ); } - assert_eq!(metric("{}", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), "(no such evaluation)"); - assert_eq!(metric("", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), "(not JSON)"); + assert_eq!( + metric("{}", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), + "(no such evaluation)" + ); + assert_eq!( + metric("", "PGS000018", "PPM000591", "EUR", "HR 1.53").unwrap_err(), + "(not JSON)" + ); } #[test] @@ -307,7 +351,11 @@ mod tests { assert!(!ok, "{said}"); assert_eq!(said, "PGS000018, European, HR True"); let body = ANSWER.replace("1.53", "2"); - assert!(metric(&body, "PGS000018", "PPM000591", "EUR", "HR 2").unwrap().1); + assert!( + metric(&body, "PGS000018", "PPM000591", "EUR", "HR 2") + .unwrap() + .1 + ); } #[test] diff --git a/rust/crates/og-dev/src/versions.rs b/rust/crates/og-dev/src/versions.rs index d08528e2e..6e4761c5f 100644 --- a/rust/crates/og-dev/src/versions.rs +++ b/rust/crates/og-dev/src/versions.rs @@ -103,7 +103,10 @@ pub fn check_run( for p in all { match (images.get(p), first.get(p)) { (None, _) => { - let _ = writeln!(out, "FAIL {p:<20} has no selector in conf/containers.config"); + let _ = writeln!( + out, + "FAIL {p:<20} has no selector in conf/containers.config" + ); ok = false; } (Some(_), None) => { @@ -142,7 +145,13 @@ pub fn cli_check_run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) } let texts = read(config).and_then(|c| Ok((c, read(yml)?))); match texts { - Ok((c, y)) => u8::from(!check_run(&selectors(&c), &first_entries(&y), yml, expect, out)), + Ok((c, y)) => u8::from(!check_run( + &selectors(&c), + &first_entries(&y), + yml, + expect, + out, + )), Err(e) => { let _ = writeln!(err, "ERROR: {e}"); 2 @@ -150,8 +159,7 @@ pub fn cli_check_run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) } } -static VAR: LazyLock = - LazyLock::new(|| re(r"\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)")); +static VAR: LazyLock = LazyLock::new(|| re(r"\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)")); /// One URL with its variables filled and its trailing punctuation dropped. pub fn expand(url: &str, vals: &BTreeMap) -> String { @@ -220,7 +228,13 @@ mod tests { fn run(yml: &str, expect: &[&str]) -> (bool, String) { let expect: Vec = expect.iter().map(|s| (*s).to_owned()).collect(); let mut out = Vec::new(); - let ok = check_run(&selectors(CONFIG), &first_entries(yml), "v.yml", &expect, &mut out); + let ok = check_run( + &selectors(CONFIG), + &first_entries(yml), + "v.yml", + &expect, + &mut out, + ); (ok, String::from_utf8(out).unwrap()) } @@ -231,7 +245,10 @@ mod tests { &[], ); assert!(ok, "{out}"); - assert!(out.contains("OK ROH bcftools: 1.22--h0_0\n"), "{out}"); + assert!( + out.contains("OK ROH bcftools: 1.22--h0_0\n"), + "{out}" + ); } #[test] @@ -242,8 +259,14 @@ mod tests { ); assert!(!ok); assert!(out.contains("FAIL ROH bcftools: 0.0.0-planted, want 1.22--h0_0 (quay.io/x/bcftools:1.22--h0_0)"), "{out}"); - assert!(out.contains("FAIL NEW_TOOL has no selector"), "{out}"); - assert!(out.contains("FAIL VEP missing from v.yml"), "{out}"); + assert!( + out.contains("FAIL NEW_TOOL has no selector"), + "{out}" + ); + assert!( + out.contains("FAIL VEP missing from v.yml"), + "{out}" + ); assert!(!out.contains("PRS"), "{out}"); let (ok, out) = run("nothing: here\n", &["ROH"]); assert!(!ok); @@ -253,10 +276,22 @@ mod tests { #[test] fn urls_expand_and_lose_trailing_punctuation() { let vals = BTreeMap::from([("REL".to_owned(), "1.2".to_owned())]); - assert_eq!(expand("https://x.org/${REL}/a.gz", &vals), "https://x.org/1.2/a.gz"); - assert_eq!(expand("https://x.org/$REL/a.gz).", &vals), "https://x.org/1.2/a.gz)"); - assert_eq!(expand("https://x.org/${NOPE}/a.", &vals), "https://x.org/${NOPE}/a"); - assert_eq!(expand("${U:-https://x.org/a}", &vals), "${U:-https://x.org/a}"); + assert_eq!( + expand("https://x.org/${REL}/a.gz", &vals), + "https://x.org/1.2/a.gz" + ); + assert_eq!( + expand("https://x.org/$REL/a.gz).", &vals), + "https://x.org/1.2/a.gz)" + ); + assert_eq!( + expand("https://x.org/${NOPE}/a.", &vals), + "https://x.org/${NOPE}/a" + ); + assert_eq!( + expand("${U:-https://x.org/a}", &vals), + "${U:-https://x.org/a}" + ); assert_eq!(expand("https://x.org/a}", &vals), "https://x.org/a"); assert_eq!(expand("https://x.org/a];:", &vals), "https://x.org/a"); } From 037a72381cf5d58e207483a1c117d73e9e3d5e8e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:30:48 +0100 Subject: [PATCH 11/76] docs: changelog for the og dev scanners and version checks Bead: pgp-bs9u.37 --- CHANGELOG.md | 8 ++++++++ scripts/ci/check-personal-data.sh | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f5f4625..19f32e7da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The personal-data scan, the image scan and the version checks of CI now run in the Rust `og` binary: `og dev scan personal-data` (`scripts/ci/check-personal-data.sh`), `og dev scan images` and `og dev versions coupled` (`scripts/ci/check-images.sh`), `og dev versions check-run` (`scripts/ci/gen-containers-config.sh --check-versions`), `og dev versions expand` (`scripts/ci/check-links.sh`) and `og dev pgs` (`scripts/ci/check-pgs-labels.sh`). The scripts keep their options, lists and output lines; the Python inside them is gone. The personal-data scan of the repository takes 0.5 s instead of 3.6 s. Fixes: + - A personal-data scan that read no text file (every file skipped or binary) fails with `FAIL: no text file was scanned`. It printed `scanned 0 text files` and passed. + - A personal-data scan whose root is reached through a symbolic link places each file under that root, so the lists of skipped files and documented examples still apply to it. + - An image exemption in `check-images.sh` whose file no longer names that image fails with `FAIL: no longer names ; drop its line from EXEMPT_PAIRS`. A stale exemption was never reported. + - `check-links.sh` and `check-images.sh` read `versions.env` strictly: a line that is not a plain `NAME="value"` (with no `"`, `$` or backquote in the value), or a name set twice, fails the check and names the line. Before, such a line was skipped, so the URLs built from it went unchecked. + - The PGS evaluation check never reads a JSON `true` as the number 1, so `"estimate": true` no longer matches an effect of `OR 1`. + - `PERSONAL_DATA_PATTERNS` holds patterns in the syntax of Rust's regex crate. It was Python's. + - Step 37's own Y tools now run in the Rust `og` binary: `og y tree-positions` and `og y tree-place` (the placement on the current YFull YTree), `og y private-snps` (the private SNP list) and `og y yleaf-positions` (Yleaf's marker positions). They replace `bin/y_tree_place.py` and `bin/y_private_snps.py`, which are gone, and the `positions` command of `bin/yleaf_run.py`, which keeps only `predict`. og is static, so each part still runs in the Yleaf image. The three tables are the same, byte for byte, on the 43 parity cases, except where a line below says otherwise. On the pinned YTree 14.06.0 with a synthetic pileup of 362,055 positions, `og y tree-place` takes 0.8 s and 99 MB where the Python took 3.6 s and 389 MB; on a synthetic chrY VCF of 20,000 records, `og y private-snps` takes 0.3 s and 24 MB where the Python took 1.1 s and 330 MB. What changes on purpose: - `og y tree-place` reads the pileup by contig and position, and only its `chrY` (or `Y`) lines. The Python keyed it by position only, so a line on another contig could overwrite a chrY count. Lines on another contig are now counted at the end of the step line, and a pileup with both `chrY` and `Y` lines stops with exit 1. Step 37 makes a chrY-only pileup, so its results do not change. - `og y private-snps` refuses a callable mask with more than one FASTA record (exit 1). The Python joined the records into one sequence, which shifts every position after the first. The pinned mask has one record. diff --git a/scripts/ci/check-personal-data.sh b/scripts/ci/check-personal-data.sh index 4127d3799..82aed5c92 100755 --- a/scripts/ci/check-personal-data.sh +++ b/scripts/ci/check-personal-data.sh @@ -32,9 +32,9 @@ # and require each to be caught # # Your own identifiers (names, sample ids, hostnames) do not belong in this -# public file. Put one extended regular expression per line in a file outside -# the repository and point PERSONAL_DATA_PATTERNS at it; each match is -# reported as kind "own". +# public file. Put one regular expression per line (the syntax of Rust's +# regex crate, close to POSIX extended) in a file outside the repository and +# point PERSONAL_DATA_PATTERNS at it; each match is reported as kind "own". # # The scan is og dev scan personal-data (rust/crates/og-dev); bin/og runs # the build scripts/setup.sh --og puts in bin/.og for this machine, and OG From 253157eb02611bb6c62c4cfa07c6b50a30a1ddff Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:32:51 +0100 Subject: [PATCH 12/76] test(ci): the stale-exemption control fails only on the stale exemption Bead: pgp-bs9u.37 --- scripts/ci/check-images.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/check-images.sh b/scripts/ci/check-images.sh index 9620755d1..61df53f30 100755 --- a/scripts/ci/check-images.sh +++ b/scripts/ci/check-images.sh @@ -162,6 +162,9 @@ self_test() { # An exempt literal that left its file: the exemption is stale. copy exempt-stale sed -i.bak 's|quay.io/biocontainers/expansionhunter:5.0.0|an example image|' "${tmp}/exempt-stale/docs/stylesheets/theme.css" + # sed's backup still names the image and is no exempt file: drop it, so + # only the stale exemption can fail this copy. + rm "${tmp}/exempt-stale/docs/stylesheets/theme.css.bak" expect exempt-stale '^FAIL: docs/stylesheets/theme\.css no longer names quay\.io/biocontainers/expansionhunter:5\.0\.0; drop its line' # The bwa-mem2 tag scripts/ci/settle-doubts.sh once repeated: it reads From 8153ee5dad9064f75d43d4bbaf7985720e53cc7a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:33:07 +0100 Subject: [PATCH 13/76] test(ci): red control, the five pre-fix readings planted back Each self-test control and cargo test of the fixes must go red on this commit; the next commit reverts it. Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++---- rust/crates/og-dev/src/pgs.rs | 1 + rust/crates/og-dev/src/versions_env.rs | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 34f2d540d..ea265da40 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 { + if *n > 0 && false { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 33cf3ef5b..c4999b11a 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,9 +373,7 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = a - .root - .canonicalize() + let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -437,7 +435,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 { + if scanned == 0 && false { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 14639f36f..1c9945283 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,6 +189,7 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, + Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index f6245ecbc..583c489ad 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,6 +39,8 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { + continue; // RED CONTROL: the pre-fix reading skipped the line + #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From 22d98d52c02e3dffa5b00d2b12bea4b58d5ba7fc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:35:24 +0100 Subject: [PATCH 14/76] test(ci): take the red control back out Reverts 8153ee5d, which planted the five pre-fix readings to show each control red. Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++++-- rust/crates/og-dev/src/pgs.rs | 1 - rust/crates/og-dev/src/versions_env.rs | 2 -- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index ea265da40..34f2d540d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 && false { + if *n > 0 { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index c4999b11a..33cf3ef5b 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,7 +373,9 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root + let root = a + .root + .canonicalize() .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -435,7 +437,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 && false { + if scanned == 0 { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 1c9945283..14639f36f 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,7 +189,6 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, - Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index 583c489ad..f6245ecbc 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,8 +39,6 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { - continue; // RED CONTROL: the pre-fix reading skipped the line - #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From 103471c7dad096ab0d2801232ab9941b6e32e22d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:34:43 +0100 Subject: [PATCH 15/76] test: take the planted faults out again (red control done) This reverts commit 63938de53b7bf542246af55ea8ad3a31200c84fb. Bead: pgp-bs9u.39 --- rust/crates/og-dev/src/evidence.rs | 2 +- rust/crates/og-dev/src/fixture.rs | 13 +++++++++++-- rust/crates/og-dev/src/renovate.rs | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index 6971f2f1b..c9b76d7fc 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -119,7 +119,7 @@ fn clash_positions( n += 1; let one = |a: &[u8]| std::str::from_utf8(a).map_or(a.len(), |s| s.chars().count()) == 1; snvs += u32::from(one(c[3]) && one(c[4])); - if n >= 3 && snvs >= 1 { + if n >= 2 && snvs >= 1 { hit.insert(k); } Ok(()) diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs index 0d5d9f3bf..bb163c9f9 100644 --- a/rust/crates/og-dev/src/fixture.rs +++ b/rust/crates/og-dev/src/fixture.rs @@ -200,7 +200,7 @@ pub mod comphet { /// The contig: `random.Random(31)` choosing among `ACGT` 80,000 times. pub fn reference() -> Vec { - let mut rng = Mt19937::new(32); + let mut rng = Mt19937::new(31); (0..LENGTH) .map(|_| b"ACGT"[rng.below(4) as usize]) .collect() @@ -423,7 +423,16 @@ fn diplotype(x: &Value) -> Result { let Value::Object(x) = x else { return Ok(x.clone()); }; - let mut out = pick(x, &["gene", "label", "phenotypes", "outsidePhenotype"]); + let mut out = pick( + x, + &[ + "gene", + "label", + "phenotypes", + "activityScore", + "outsidePhenotype", + ], + ); for a in ["allele1", "allele2"] { let Some(v) = x.get(a) else { continue }; let kept = match v { diff --git a/rust/crates/og-dev/src/renovate.rs b/rust/crates/og-dev/src/renovate.rs index 4544ccb01..f3355c766 100644 --- a/rust/crates/og-dev/src/renovate.rs +++ b/rust/crates/og-dev/src/renovate.rs @@ -421,7 +421,7 @@ fn check(deps: &[Dep], env_text: &str, pins: &Pins) -> Vec { } else { ":".to_owned() }; - if text.contains(':') { + if text.contains([':', '@']) { fails.push(format!("names: {f}: package name {}", r(name))); } } From 26f3daf2ea50e361fe3f3f2d62e8a5ea84bf8ca4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:02:40 +0100 Subject: [PATCH 16/76] test(ci): red control again, clippy-clean, so cargo test shows each planted fault The first red control stopped at clippy, so cargo test never ran and the PGS self-test step never started. This one plants the same five readings in a form clippy accepts; the next commit takes them back out. Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++---- rust/crates/og-dev/src/pgs.rs | 1 + rust/crates/og-dev/src/versions_env.rs | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 34f2d540d..08b17432d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 { + if *n > 0 && std::hint::black_box(false) { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 33cf3ef5b..047d89952 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,9 +373,7 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = a - .root - .canonicalize() + let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -437,7 +435,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 { + if scanned == 0 && std::hint::black_box(false) { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 14639f36f..1c9945283 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,6 +189,7 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, + Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index f6245ecbc..583c489ad 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,6 +39,8 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { + continue; // RED CONTROL: the pre-fix reading skipped the line + #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From d15dcf10ae6187b79d39df93fb1c120c6ab3df29 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:02:51 +0100 Subject: [PATCH 17/76] test(ci): take the second red control back out Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++++-- rust/crates/og-dev/src/pgs.rs | 1 - rust/crates/og-dev/src/versions_env.rs | 2 -- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 08b17432d..34f2d540d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 && std::hint::black_box(false) { + if *n > 0 { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 047d89952..33cf3ef5b 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,7 +373,9 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root + let root = a + .root + .canonicalize() .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -435,7 +437,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 && std::hint::black_box(false) { + if scanned == 0 { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 1c9945283..14639f36f 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,7 +189,6 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, - Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index 583c489ad..f6245ecbc 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,8 +39,6 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { - continue; // RED CONTROL: the pre-fix reading skipped the line - #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From 36123685c1fb8b1ab224f4df15e174430b074ec2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:03:19 +0100 Subject: [PATCH 18/76] test(parity): commit the hg002-gvcf reference inputs .gitignore hid The dev-evidence-hg002-gvcf parity cases read ref.fasta and its .fai, which the *.fasta ignore rule kept out of the tree, so CI parity stopped with 'input not in tests/parity/inputs'. Bead: pgp-bs9u.39 --- tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta | 7 +++++++ tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta create mode 100644 tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta new file mode 100644 index 000000000..b37a59afb --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta @@ -0,0 +1,7 @@ +>chrA test +ACGTACGTACGTACGTACGT +NNACGTacgtACGTACGTAC +GTACGTACGT +>chrB test +ACNNNTGCAtgcaACGTACG +TACGTACGTA diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai new file mode 100644 index 000000000..9980f0dd1 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai @@ -0,0 +1,2 @@ +chrA 50 11 20 21 +chrB 30 75 20 21 From 2244bed69d2219d116bd2bac8593236ba0d46f94 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:02:40 +0100 Subject: [PATCH 19/76] test(ci): red control on the merged head, clippy-clean The first red control stopped at clippy, so cargo test never ran. This one plants the same five pre-fix readings in a form clippy accepts, on top of the main merge; the next commit takes them back out. Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++---- rust/crates/og-dev/src/pgs.rs | 1 + rust/crates/og-dev/src/versions_env.rs | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 34f2d540d..08b17432d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 { + if *n > 0 && std::hint::black_box(false) { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 33cf3ef5b..047d89952 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,9 +373,7 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = a - .root - .canonicalize() + let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -437,7 +435,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 { + if scanned == 0 && std::hint::black_box(false) { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 14639f36f..1c9945283 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,6 +189,7 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, + Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index f6245ecbc..583c489ad 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,6 +39,8 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { + continue; // RED CONTROL: the pre-fix reading skipped the line + #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From 3cec87db803672ef0e34fd8ae881b2dc77c9e8ae Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:21:07 +0100 Subject: [PATCH 20/76] test(ci): take the red control on the merged head back out Bead: pgp-bs9u.37 --- rust/crates/og-dev/src/images.rs | 2 +- rust/crates/og-dev/src/personal_data.rs | 6 ++++-- rust/crates/og-dev/src/pgs.rs | 1 - rust/crates/og-dev/src/versions_env.rs | 2 -- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs index 08b17432d..34f2d540d 100644 --- a/rust/crates/og-dev/src/images.rs +++ b/rust/crates/og-dev/src/images.rs @@ -220,7 +220,7 @@ fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { } } for ((f, image), n) in &left { - if *n > 0 && std::hint::black_box(false) { + if *n > 0 { let _ = writeln!( out, "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 047d89952..33cf3ef5b 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -373,7 +373,9 @@ fn scan_text(out: &mut dyn Write, rules: &[Check], texts: &[&str], rel: &str, da } fn run(a: &Args, out: &mut dyn Write) -> Result { - let root = Ok::<_, std::io::Error>(a.root.clone()) // RED CONTROL: the pre-fix root + let root = a + .root + .canonicalize() .map_err(|e| format!("--root {}: {e}", a.root.display()))?; let mut rules = builtin(); if let Some(p) = &a.patterns { @@ -435,7 +437,7 @@ fn run(a: &Args, out: &mut dyn Write) -> Result { bad |= scan_text(out, &rules, &texts, rel, &data); } let _ = writeln!(out, "scanned {scanned} text files"); - if scanned == 0 && std::hint::black_box(false) { + if scanned == 0 { let _ = writeln!(out, "FAIL: no text file was scanned"); return Ok(false); } diff --git a/rust/crates/og-dev/src/pgs.rs b/rust/crates/og-dev/src/pgs.rs index 1c9945283..14639f36f 100644 --- a/rust/crates/og-dev/src/pgs.rs +++ b/rust/crates/og-dev/src/pgs.rs @@ -189,7 +189,6 @@ pub fn metric( && match v { Value::Int(i) => (*i as f64 - want).abs() < 1e-9, Value::Float(x) => (x - want).abs() < 1e-9, - Value::Bool(b) => (f64::from(u8::from(*b)) - want).abs() < 1e-9, // RED CONTROL: the pre-fix reading _ => false, // a bool is never a number } }) diff --git a/rust/crates/og-dev/src/versions_env.rs b/rust/crates/og-dev/src/versions_env.rs index 583c489ad..f6245ecbc 100644 --- a/rust/crates/og-dev/src/versions_env.rs +++ b/rust/crates/og-dev/src/versions_env.rs @@ -39,8 +39,6 @@ pub fn parse(text: &str) -> Result, String> { continue; } let Some(c) = LINE.captures(line) else { - continue; // RED CONTROL: the pre-fix reading skipped the line - #[allow(unreachable_code)] return Err(format!( "versions.env:{}: not a NAME=\"value\" line (a value holds no \", $ or backquote): {line}", i + 1 From 06213bd908cfe31e96fd03fd77b6eafc11e4cc70 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:54 +0100 Subject: [PATCH 21/76] test(parity): commit the past-the-end golden VCF .gitignore hid The *.vcf ignore rule kept the golden out.vcf of dev-evidence-hg002-gvcf/past-the-end out of the tree, so CI compared og with the oracle and reported the deliberate fix as a difference. Bead: pgp-bs9u.39 --- .../dev-evidence-hg002-gvcf/past-the-end/out.vcf | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf diff --git a/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf new file mode 100644 index 000000000..1e494f38c --- /dev/null +++ b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf @@ -0,0 +1,11 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +##INFO= +##ALT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG002 +chrA 3 . G T 50 PASS . GT 0/1 +chrA 10 . CGT C 50 PASS . GT 0/1 +chrB 8 . C T 50 PASS . GT 0/1 +chrC 1 . A G 50 PASS . GT 0/1 +chrB 30 . A <*> . . END=30 GT 0/0 From 3ff62369089236db926da5458bb15ef4fad0f78f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:27 +0100 Subject: [PATCH 22/76] test(ci): cover HG002 reference blocks and scope the Rust fixture Select comphet-phasing-1-reads when its Rust generator changes. Keep the push paths and scope list aligned. Check HG002 truth records and the last valid reference block directly against the committed golden. Bead: pgp-bs9u.39 --- .github/workflows/e2e.yml | 1 + rust/crates/og-dev/src/evidence.rs | 47 ++++++++++++++++++++++++++++++ scripts/ci/e2e-case-paths.tsv | 2 +- scripts/ci/e2e-scope.sh | 1 + 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 39d4b272d..b395a6ad0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,6 +65,7 @@ on: - 'nextflow_schema.json' - 'versions.env' - '.github/workflows/e2e.yml' + - 'rust/crates/og-dev/src/fixture.rs' - '.github/actions/**' - 'tests/fixtures/annotsv/class45.sh' - 'tests/fixtures/consumer_export/sites.tsv.gz' diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index c9b76d7fc..84529a8ef 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -343,6 +343,53 @@ fn hg002_gvcf( mod tests { use super::*; + #[test] + fn hg002_gvcf_keeps_truth_and_adds_only_valid_reference_blocks() { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let inputs = repo.join("tests/parity/inputs/dev-evidence-hg002-gvcf"); + let temp = std::env::temp_dir().join(format!( + "og-evidence-gvcf-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&temp).unwrap(); + let output = temp.join("out.vcf"); + let args = [ + "hg002-gvcf".to_owned(), + inputs.join("truth.vcf.gz").to_str().unwrap().to_owned(), + inputs + .join("truth-past-end.bed") + .to_str() + .unwrap() + .to_owned(), + inputs.join("ref.fasta").to_str().unwrap().to_owned(), + inputs + .join("sites-past-end.tsv") + .to_str() + .unwrap() + .to_owned(), + output.to_str().unwrap().to_owned(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 0, "{err:?}"); + assert_eq!( + out, + b"gVCF: 4 GIAB records, 1 reference blocks at panel sites\n" + ); + assert!(err.is_empty()); + let actual = fs::read(&output).unwrap(); + fs::remove_dir_all(&temp).unwrap(); + assert_eq!( + actual, + include_bytes!( + "../../../../tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf" + ) + ); + } + #[test] fn spans_merge_and_answer_inside() { let m = merge(vec![(10, 12), (5, 6), (11, 15), (15, 16)]); diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 17e9c8f6d..d01f1b348 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -78,7 +78,7 @@ cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh diff --git a/scripts/ci/e2e-scope.sh b/scripts/ci/e2e-scope.sh index 81ddd26d0..b3305168d 100755 --- a/scripts/ci/e2e-scope.sh +++ b/scripts/ci/e2e-scope.sh @@ -45,6 +45,7 @@ export LC_ALL=C # runs no case. The push paths of e2e.yml are the same list (e2e-plan checks). SCOPE_DIRS=(scripts modules workflows bin conf assets tests/e2e .github/actions) SCOPE_FILES=(tests/fixtures/VERSION main.nf nextflow.config nextflow_schema.json versions.env .github/workflows/e2e.yml + rust/crates/og-dev/src/fixture.rs # The committed fixtures the cases read (their rows name them too). tests/fixtures/annotsv/class45.sh tests/fixtures/consumer_export/sites.tsv.gz tests/fixtures/families/stub_families.tsv tests/fixtures/pharmcat/refblocks.g.vcf tests/fixtures/sv/cnvpytor.vcf.in From 63e44b68022e77d44ad14d6e3fc5392bdc2ada9b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:41 +0100 Subject: [PATCH 23/76] test: plant a wrong HG002 reference genotype for the red control Guard must reject 1/1 reference blocks in evidence::tests::hg002_gvcf_keeps_truth_and_adds_only_valid_reference_blocks. Revert this planted fault after recording its red CI job. Bead: pgp-bs9u.39 --- rust/crates/og-dev/src/evidence.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index 84529a8ef..44651f17a 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -332,7 +332,7 @@ fn hg002_gvcf( } blocks += 1; w.write_all(c) - .and_then(|()| writeln!(w, "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t0/0", b as char)) + .and_then(|()| writeln!(w, "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t1/1", b as char)) .map_err(werr) })?; w.flush().map_err(werr)?; From d3a7a9da7c02c76788c6098b2dab392cc79f4b07 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:56 +0100 Subject: [PATCH 24/76] test: restore the HG002 reference genotype after the red control Remove the deliberately wrong 1/1 reference genotype. Keep the direct golden-file regression test. Bead: pgp-bs9u.39 --- rust/crates/og-dev/src/evidence.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index 44651f17a..84529a8ef 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -332,7 +332,7 @@ fn hg002_gvcf( } blocks += 1; w.write_all(c) - .and_then(|()| writeln!(w, "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t1/1", b as char)) + .and_then(|()| writeln!(w, "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t0/0", b as char)) .map_err(werr) })?; w.flush().map_err(werr)?; From 103d3b45d9e1d0a26fd5efcf1f729ccc97688298 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:26:20 +0100 Subject: [PATCH 25/76] fix(ci): check complete custom-pattern matches against examples Keep custom regex captures from hiding the rest of a match inside an allowed example. Cover numbered and named captures, inline flags and extended-mode comments. Bead: pgp-bs9u.37 --- CHANGELOG.md | 2 +- rust/crates/og-dev/src/lookaround.rs | 13 ++++++++++- rust/crates/og-dev/src/personal_data.rs | 30 ++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43489f003..329c0adc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - An image exemption in `check-images.sh` whose file no longer names that image fails with `FAIL: no longer names ; drop its line from EXEMPT_PAIRS`. A stale exemption was never reported. - `check-links.sh` and `check-images.sh` read `versions.env` strictly: a line that is not a plain `NAME="value"` (with no `"`, `$` or backquote in the value), or a name set twice, fails the check and names the line. Before, such a line was skipped, so the URLs built from it went unchecked. - The PGS evaluation check never reads a JSON `true` as the number 1, so `"estimate": true` no longer matches an effect of `OR 1`. - - `PERSONAL_DATA_PATTERNS` holds patterns in the syntax of Rust's regex crate. It was Python's. + - `PERSONAL_DATA_PATTERNS` holds patterns in the syntax of Rust's regex crate. It was Python's. Capture groups keep the full match when checking allowed examples. - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. diff --git a/rust/crates/og-dev/src/lookaround.rs b/rust/crates/og-dev/src/lookaround.rs index d8dc7cce9..f74d24383 100644 --- a/rust/crates/og-dev/src/lookaround.rs +++ b/rust/crates/og-dev/src/lookaround.rs @@ -16,6 +16,9 @@ use regex::Regex; /// A pattern with its look-behind. pub struct Rule { re: Regex, + /// Capture describing the match span: 1 for rewritten look-around, + /// 0 for a user's unmodified regular expression. + capture: usize, /// True for a character the match must not follow. not_after: Option bool>, } @@ -26,17 +29,25 @@ impl Rule { pub fn new(pattern: &str, not_after: Option bool>) -> Result { Ok(Rule { re: Regex::new(pattern)?, + capture: 1, not_after, }) } + /// A user's regular expression, whose captures do not change its span. + pub fn whole_match(pattern: &str) -> Result { + let mut rule = Self::new(pattern, None)?; + rule.capture = 0; + Ok(rule) + } + /// The spans of `re.finditer` over `line`, in order. pub fn find_iter<'a>(&'a self, line: &'a str) -> impl Iterator + 'a { let mut pos = 0; std::iter::from_fn(move || { while pos <= line.len() { let caps = self.re.captures_at(line, pos)?; - let m = caps.get(1).or_else(|| caps.get(0))?; + let m = caps.get(self.capture).or_else(|| caps.get(0))?; let (start, end) = (m.start(), m.end()); let refused = self .not_after diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs index 33cf3ef5b..c56466e10 100644 --- a/rust/crates/og-dev/src/personal_data.rs +++ b/rust/crates/og-dev/src/personal_data.rs @@ -238,7 +238,9 @@ fn own_rules(path: &str) -> Result, String> { if p.is_empty() || l.starts_with('#') { continue; } - let rule = Rule::new(p, None).map_err(|e| format!("{path}: bad pattern {p}: {e}"))?; + // Rule reports capture 1 for the built-in look-around patterns. + // A user's captures must not narrow the span checked against ALLOW. + let rule = Rule::whole_match(p).map_err(|e| format!("{path}: bad pattern {p}: {e}"))?; out.push(Check { kind: "own", rule, @@ -563,4 +565,30 @@ mod tests { let got = String::from_utf8(out).unwrap(); assert!(got.starts_with("FAIL home x.md:1: "), "{got}"); } + + #[test] + fn own_patterns_use_the_complete_match() { + let path = + std::env::temp_dir().join(format!("og-personal-patterns-{}", std::process::id())); + std::fs::write( + &path, + "PREFIX (PUBLIC) SUFFIX\nPREFIX (?P