diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 89b54d8dd..091f86063 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -313,7 +313,7 @@ jobs: grep -q 'summary.json validates (og dev schema)' <<< "$out" \ || { echo "ERROR: reports-stale-step did not run og dev schema"; exit 1; } - # The tests of the og filters steps 04, 05 and 07 run, og chip's (step 49), og family's (step 38), og run delta's, step 26b's og ancestry-fine, + # The tests of the og filters steps 04, 05 and 07 run, og chip's (step 49), og cn's (steps 35, 41 and 46), og family's (step 38), og run delta's, step 26b's og ancestry-fine, # the opt-in step reports' (45, 47, 48, 50 to 53), the # og tools setup.sh runs (og str catalog, og setup unzip and genetic-map), and the step 25 # tests that compare og prs format with the folders they write by hand: @@ -334,7 +334,9 @@ jobs: tests/test_family_report.py tests/test_family_segments.py \ tests/test_vntyper_report.py tests/test_locityper_report.py tests/test_mocha_report.py \ tests/test_mei_report.py tests/test_viral_report.py tests/test_ystr_report.py \ - tests/test_exomiser_report.py; do + tests/test_exomiser_report.py \ + tests/test_copy_number.py tests/test_hba_copy_number.py \ + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in @@ -346,6 +348,15 @@ jobs: done exit "$fail" + # The fake-docker job builds no og, so its cn-engine-calls case says it + # is skipped there; here bin/og is built and the case must run. + - name: The fake-docker case cn-engine-calls (steps 35, 41 and 46 through og cn) with og + working-directory: . + run: | + out=$(scripts/ci/run-fake-docker-suite.sh cn-engine-calls 2>&1) || { echo "$out"; exit 1; } + echo "$out" + if grep -q '^skip: bin/og is not built' <<<"$out"; then echo "ERROR: the case was skipped, with og built"; exit 1; fi + # A Python file kept only for what other Python imports from it holds # those names and nothing else (tests/parity/remnants.tsv). - name: og dev remnants diff --git a/.gitleaksignore b/.gitleaksignore index 3bc7bbfcc..cb0ae8e24 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -4,3 +4,8 @@ # A test string in the Parascopy freshness self-test ("GRCh37_v9.9.tar.gz"), # not a key; the line was rewritten in 7d6dad4. 63e12d6c2b29078a49e14de8ef4c4e57a9ebd8b9:scripts/ci/freshness.py:generic-api-key:940 +# A Rust constant name compared with a marker's name in og-cn, not a key; the +# comparison was turned round in the same pull request. The second line is this +# file's own first comment about it, which quoted the expression. +0bc60efda244e5361c54fe411f46db9514e5e8c3:rust/crates/og-cn/src/cli.rs:generic-api-key:590 +63b5a9273c444970e469d04c8da52aa8890726b5:.gitleaksignore:generic-api-key:7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3043350..5fab1af08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The copy-number engine now runs in the Rust `og` binary, as `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read`. That covers step 35's depth counts (HBA, the gene-dosage rows, CYP21A2, the interval rows) and its SMN1 marker, step 46's KIV-2 count and 4925G>A reading, the RHD regions of step 41 and the ALT-contig check of steps 32 and 35. They replace `bin/hba_copy_number.py` and `bin/lpa_kiv2.py`, which are gone, the commands of `bin/copy_number.py` and the marker reader of `bin/traits_report.py`. Two parts stay in Python for now: step 41's report makes the RhD call with the catalog reader and `assess()` of `bin/copy_number.py`, and step 32's CYP2D6 depth check is its own tool. `bin/traits_report.py` keeps `MARKERS` for the summary. Every table, file name and option a step uses stays the same; the parity cases in `tests/parity` hold every output that changed on purpose: + - Step 46: the KIV-2 4925G>A reading is made in the same call as the count and judges the aligner on the count's own MAPQ 0 share. Before, it read the share the count table prints, rounded to three places, so at a share just under 0.20 the count said not assessed while the marker still read `present`. Both now say not assessed. + - Step 35: the SMN1 marker message takes the SMN1 psCN from the first row of the SMN table that covers exon 7, the rule of the Paraphase comparison. It took the last such row, so a table with two rows over exon 7 made the two read different counts. + - Steps 35 and 46: a status the HBA or LPA table layout did not name (a catalog edit could give one) now writes `not assessed: `. The HBA table wrote `called` with no count, and the LPA table stopped with an error. + - A catalog row with a `more=` reading and no numbered reading is refused when the catalog loads, naming the row. Before, a call that reached it failed without naming the row. + - `og cn call --sex` reads a gene row on chrX or chrY against the sample's count there. The Python call had no sex, so such a row could only read `sex_unknown`. No row of the shipped catalog is a gene row on chrX or chrY. + - The tables are written to a temporary file and renamed, so a failed run leaves no half-written table. + - `og cn bed` and `og cn marker-regions` exit 2 when they cannot write the region list, so a step never reads a truncated list. + - `copy_number.py call --json` is gone with no replacement: no step passed it. + - Error messages about a wrong regions table say to make it with `og cn bed`. + - The opt-in steps 45, 47, 48 and 50 to 53 now read their tool's output with the Rust `og` binary: `og y str-bed` and `og y str-report` (step 45), `og viral report` (47), `og exomiser prepare` and `og exomiser report` (48), `og mei report` and `og mei annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53). They replace `bin/ystr_report.py`, `bin/viral_report.py`, `bin/exomiser_report.py`, `bin/mei_report.py`, `bin/mocha_report.py`, `bin/vntyper_report.py` and `bin/locityper_report.py`, which are gone. og is static, so each step still runs in the image it ran in. The tables are the same, byte for byte, on the 123 parity cases, except where a line below says otherwise. Every output is written whole or not at all, and an input a step cannot read stops it with one `ERROR:` line, not a Python traceback. - Step 48: the summary printed at the end of `og exomiser report` has one line per rank. The Python cut it by variant count, so a rank with several variants could push the closing note in or out. The text file is unchanged. - Step 50: the depth of each catalog window is its own. The Python looked it up by gene and chromosome, so two windows of one gene on one chromosome both got the last one's depth. The shipped catalog has no such pair, so its results do not change. diff --git a/bin/bloodtype_report.py b/bin/bloodtype_report.py index a16874c84..73eaf4236 100755 --- a/bin/bloodtype_report.py +++ b/bin/bloodtype_report.py @@ -26,8 +26,8 @@ --depth is samtools bedcov -Q 20 -j over the table's sites and the RHD row of bin/cn_loci.tsv (scripts/41-bloodtype.sh and the BLOODTYPE_DEPTH process make -it from the BAM; `copy_number.py bed --locus RHD` prints that row's regions, -and bin/copy_number.py turns their depth into the copy number). It gives: +it from the BAM; `og cn bed --locus RHD` prints that row's regions, and the +catalog code of bin/copy_number.py turns their depth into the copy number). It gives: - RhD. The common RhD-negative allele (RHD*01N.01) deletes the whole RHD gene, which a VCF does not show. RHD depth over the depth of its two flanks gives the copy number (0, 1, 2); exons 3 to 7 without RHD reads in a gene that is diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 23d8ed5c2..a890177ae 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -128,7 +128,7 @@ "Variants of uncertain significance (not interpreted)", ] -# Added to NOT_ASSESSED unless step 35's depth count (bin/hba_copy_number.py, +# Added to NOT_ASSESSED unless step 35's depth count (og cn call --locus HBA, # short reads) or step 35b (Paraphase, PacBio HiFi, both genes) made a call. HBA_NOT_ASSESSED = ("HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count " "(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call") @@ -910,13 +910,13 @@ def sec_parascopy(d, s): "smn1_marker": read_marker_table(os.path.join(d, "paralogs", f"{s}_smn1_marker.tsv"), "35", "SMN1_27134"), "rows": len(rows), - # HBA1/HBA2 from depth (bin/hba_copy_number.py); None when + # HBA1/HBA2 from depth (og cn call --locus HBA); None when # step 35 wrote no HBA table (a run from before the check) "hba": read_hba(hp) if os.path.isfile(hp) else None, # the interval rows of bin/cn_loci.tsv (recurrent CNV regions, # carrier deletions, AZF); None without step 35's table "cn_intervals": read_cn_intervals(ip) if os.path.isfile(ip) else None, - # CYP21A2 from depth and PSV sites (bin/copy_number.py, the + # CYP21A2 from depth and PSV sites (og cn call, the # CYP21A2 row): copies, module total, finding; None without it "cyp21a2": read_hba(cp) if os.path.isfile(cp) else None, # the gene-dosage rows (HP, LCE3BC, CCL3L1): total copies from @@ -931,7 +931,7 @@ def sec_parascopy(d, s): def read_cn_intervals(path): - """Step 35's interval table (bin/copy_number.py intervals): the rows with + """Step 35's interval table (og cn intervals): the rows with a deletion or a duplication, and a count of the rows by status.""" rows = read_tsv(path) cols = ("locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change") @@ -2523,7 +2523,7 @@ def sec_archaic(d, s): def sec_lpa(d, s): - """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, bin/lpa_kiv2.py), with + """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, og cn call --locus LPA_KIV2), with step 42's two LPA risk variants when the traits table is there, and whether either points to a genetically high Lp(a).""" p = first_existing(d, [f"lpa/{s}_lpa_kiv2.tsv"]) diff --git a/bin/copy_number.py b/bin/copy_number.py index 84374c59c..e56144990 100755 --- a/bin/copy_number.py +++ b/bin/copy_number.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 -"""copy_number.py: copy number at any locus of bin/cn_loci.tsv from short-read depth. +"""copy_number.py: the catalog reader and the depth-ratio engine of +bin/cn_loci.tsv, as step 41's report (bin/bloodtype_report.py) imports them. + +The commands are og cn's (rust/crates/og-cn, docs/cn-loci.md): `og cn bed` +prints a locus's segments, `og cn call` turns `samtools bedcov` over them +into the step's table, `og cn intervals` reads the interval rows and `og cn +check-bam` refuses a BAM aligned with ALT contigs. This file keeps what +bloodtype_report.py reads the RHD row with until it moves to og too: +load_catalog, assess, cut_copies, BrokenControl and CatalogError. One method, one catalog row per locus: the depth over the locus's target segments against the depth over single-copy control segments, as @@ -9,73 +17,11 @@ then guarded by a minimum control depth, flank agreement and the share of the target depth from reads of mapping quality 0, and called as the row's `call` column says. docs/cn-loci.md describes the columns and how to add a locus. - - copy_number.py bed [--catalog TSV] [--locus L ...] - print the segments of the loci (all bedcov loci by default) as a BED - (chrom, 0-based start, end, name), sorted by position within a locus - copy_number.py call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] - [--json OUT.json] [--table OUT.tsv] - read `samtools bedcov` over those segments (chrom, start, end, name, - summed depth; the row's first bedcov pass) and, for a row with a MAPQ 0 - guard, the MAPQ >= 1 pass; write the locus's JSON block and, with - --table, the step's metricvalue table (bin/hba_copy_number.py and - bin/lpa_kiv2.py lay those out). Exit 0 whatever the status; 2 on - unreadable input; 3 (BROKEN_CONTROL) when a control segment has no - read while the other controls have the depth the row needs: the row's - coordinates are wrong, or the sample lacks that stretch, and no number - is given. - - copy_number.py bed --kind interval - the segments of every interval row (kind interval, one bedcov pass: the - recurrent CNV regions, the carrier deletion alleles and AZF) in one BED - copy_number.py intervals [--catalog TSV] --bedcov T --sex male|female|unknown - --out OUT.tsv - copy_number.py intervals --failed --out OUT.tsv - call every interval row from one `samtools bedcov` over that BED; write - one line per row (locus, name, chrom, start, end, status, copies, - expected, change, estimate, reason). A row on chrX or chrY is read - against the sample's own count there: 1 on a male, 2 on a female's X; - a chrY row on a female is 'not applicable', and without a sex both are - 'not assessed' (sex_unknown). A row whose control has no read is 'not - assessed' (broken_control) and the others still run. --failed writes - every row as 'not assessed' (check_failed), for a step whose depth - pass failed. - - copy_number.py bed --psv --locus L - the paralogous sequence variant (PSV) sites of a psv row (call psv:D): - each site's position on the gene and on its pseudogene, one base each - (name and name_pseudo), for `samtools mpileup -l` - copy_number.py call --locus L --bedcov T --pileup P ... - a psv row (CYP21A2) also reads `samtools mpileup` without a reference - over those sites (P): the module total from depth as above, then at - each site the share of reads with the gene's base, pooled over the gene - and pseudogene positions, times that total gives the copies that carry - the gene's sequence there. The table names a deletion (one module and - one gene copy fewer), a possible conversion (fewer gene copies at some - sites than at the others, or fewer than two) or none; it never says - which chromosome carries the change. - - copy_number.py check-bam --header H - refuse a BAM aligned to a reference with ALT or HLA contigs. H is - `samtools view -H` of the BAM, or its contig names one per line ('-' - reads stdin). On such a BAM a read that fits a primary locus and its - ALT copy equally gets MAPQ 0 or lands on the ALT copy, so the depth of - a duplicated gene drops and a depth-based count reads a deletion that - is not there. Exit 0 and no output without such a contig; exit 4 with - one REFUSED line on stderr with one. `call --bam-header H` does the - same check first. - -Two empty tables mean the BAM has no reads on that contig at all (bedcov is -skipped there): status 'not assessed'. Standard library only: it runs in the -Python and Parascopy images and under tests/test_copy_number.py. +Standard library only. """ -import argparse import csv -import importlib -import json import math import os -import sys CATALOG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cn_loci.tsv") COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", @@ -86,22 +32,9 @@ # scripts/ci/build-cn-intervals.py normalises it, and a row with any other # spelling is refused here). CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} -SEXES = ("male", "female", "unknown") -INTERVAL_COLUMNS = ["locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", - "estimate", "reason"] INPUTS = ("bedcov", "cnvpytor-level") Z95 = 1.96 -# The step table of a locus, by the module that lays it out (same folder). -TABLES = {"HBA": "hba_copy_number", "LPA_KIV2": "lpa_kiv2"} -# A psv row: the reads a site needs (gene and pseudogene bases pooled) before -# its copies are read, and the findings a reading must exist for. -PSV_MIN_READS = 30 -# A site reads as converted when its copies estimate lies this far under the -# others' count (or under the usual count) or more. At 30x, about 70 reads at -# a site, read sampling alone gives an estimate of 2 copies a standard -# deviation of about 0.25; HG002's six sites read 1.62 to 2.82 (35x), so the -# real spread is wider and 0.6 is a screen setting, not a measured error rate. -PSV_MARGIN = 0.6 +# The findings a psv row needs a reading for. PSV_FINDINGS = ("none", "deletion", "conversion", "pseudogene_deletion") BASES = "ACGT" @@ -110,43 +43,6 @@ class CatalogError(ValueError): pass -class AltContigs(ValueError): - """The BAM was aligned to a reference with ALT or HLA contigs.""" - - -def alt_contigs(lines): - """The ALT or HLA contigs of `samtools view -H` lines or of contig names - one per line (the rule of scripts/validate-setup.sh: a name ending in - _alt, or starting with HLA-).""" - out = [] - for line in lines: - line = line.rstrip("\n") - if line.startswith("@"): - if not line.startswith("@SQ"): - continue - name = next((f[3:] for f in line.split("\t") if f.startswith("SN:")), "") - else: - name = line.split("\t")[0].strip() - if name.endswith("_alt") or name.startswith("HLA-"): - out.append(name) - return out - - -def check_bam(path): - """Raise AltContigs, with the refusal line, when the header at path - (or '-' for stdin) names an ALT or HLA contig.""" - if path == "-": - found = alt_contigs(sys.stdin) - else: - with open(path, errors="replace") as f: - found = alt_contigs(f) - if found: - raise AltContigs(f"REFUSED: the BAM was aligned to a reference with {len(found)} ALT or HLA contigs " - f"(first: {found[0]}). Reads that fit a primary locus and its ALT copy equally get MAPQ 0 " - "or land on the ALT copy, so depth-based copy number reads deletions that are not there. " - "Realign to the default no-ALT reference (docs/realignment.md).") - - class BrokenControl(ValueError): """A control segment with no read while the other controls are covered.""" @@ -303,29 +199,6 @@ def regions(row): return sorted(((row["chrom"], s, e, n) for n, s, e, _ in segs), key=lambda r: (r[1], r[2])) -def read_bedcov(path, row): - """{name: mean depth} from `samtools bedcov` output over the row's - segments, which must all be there and nothing else; {} for an empty table.""" - means, seen = {}, set() - with open(path, errors="replace") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5: - raise ValueError(f"{path}: expected chrom, start, end, name, depth sum; got {line.strip()!r}") - chrom, start, end, name = c[0], int(c[1]), int(c[2]), c[3] - if end <= start: - raise ValueError(f"{path}: region {name} {chrom}:{start}-{end} is empty or reversed") - if (chrom, start, end, name) in seen: - raise ValueError(f"{path}: region {name} {chrom}:{start}-{end} appears twice") - seen.add((chrom, start, end, name)) - means[name] = float(c[4]) / (end - start) - if seen and seen != set(regions(row)): - raise ValueError(f"{path}: its regions differ from the ones `copy_number.py bed --locus {row['locus']}` prints") - return means - - def _combine(mode, segs, m): if mode == "sum": return sum(m[n] for n, *_ in segs) @@ -445,7 +318,7 @@ def assess(row, m, q1=None, sex=None): b["ci95"] = (max(0.0, total * math.exp(-Z95 * rel) - row["_offset"]), total * math.exp(Z95 * rel) - row["_offset"]) if mode == "psv": - # the module total (gene plus pseudogene copies); psv_assess reads the sites + # the module total (gene plus pseudogene copies); og cn reads the sites n = round(est) if abs(est - n) > vals[0]: b["reason"] = "between_counts" @@ -467,387 +340,3 @@ def assess(row, m, q1=None, sex=None): b["reading"] = reading(row, b["class"]) b["status"] = "called" return b - - -def count_bases(bases, ref="N"): - """{A, C, G, T: reads} of one `samtools mpileup` bases column: read - starts (^ and its MAPQ character), read ends, indels (+N and -N with - their N bases), deletions and skips are not bases; '.' and ',' are the - reference base when there is one.""" - n = dict.fromkeys(BASES, 0) - i = 0 - while i < len(bases): - ch = bases[i] - if ch == "^": - i += 2 - continue - if ch in "+-": - j = i + 1 - while j < len(bases) and bases[j].isdigit(): - j += 1 - i = j + int(bases[i + 1:j] or 0) - continue - b = ref.upper() if ch in ".," else ch.upper() - if b in n: - n[b] += 1 - i += 1 - return n - - -def read_pileup(path, row): - """{site: (gene-base reads, pseudogene-base reads, other reads)} of a psv - row from `samtools mpileup` over `bed --psv`, each site's two positions - pooled: a read that carries the gene's base counts for the gene wherever - the aligner put it. A position missing from the pileup has no read.""" - pos = {} - for name, g, ps, gb, pb, _ in row["_psv"]: - pos[g + 1] = pos[ps + 1] = name - counts = {name: dict.fromkeys(BASES, 0) for name, *_ in row["_psv"]} - with open(path, errors="replace") as f: - for line in f: - if not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5 or not c[1].isdigit(): - raise ValueError(f"{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); " - f"got {line.strip()[:80]!r}") - if c[0] != row["chrom"] or int(c[1]) not in pos: - continue - for k, v in count_bases(c[4], c[2]).items(): - counts[pos[int(c[1])]][k] += v - out = {} - for name, g, ps, gb, pb, _ in row["_psv"]: - n = counts[name] - gene, pseudo = sum(n[x] for x in gb), sum(n[x] for x in pb) - out[name] = (gene, pseudo, sum(n.values()) - gene - pseudo) - return out - - -def psv_assess(row, b, counts): - """A psv row's block (from assess) with its sites read. With the module - total called, each site of PSV_MIN_READS reads or more gets the copies - carrying the gene's base there (total x gene reads / gene and pseudogene - reads) and its ratio to the median of the sites. A site is low when its - estimate is more than PSV_MARGIN under the larger of the median's count - and the usual. cyp21a2 copies is the least count over the low sites, or - the median's count when none is low. finding: deletion when a site is low - and the module total under the usual (the 30 kb chimeric deletion leaves - one module and one gene copy fewer); conversion at the low sites when the - module total is kept; pseudogene_deletion when the module total is under - the usual with no low site; else none. Which chromosome carries a change - is never said.""" - sites = {} - for name, *_ in row["_psv"]: - g, p, o = counts.get(name, (0, 0, 0)) - sites[name] = {"gene_reads": g, "pseudo_reads": p, "other_reads": o, "copies": None, - "estimate": None, "ratio": None} - b.update(sites=sites, finding=None, conversion_sites=[]) - if b["status"] != "called" or b.get("module_total") is None: - b["status"] = "not assessed" - return b - usual = b["expected"] // len(row["_targets"]) - read = [] - for name, s in sites.items(): - n = s["gene_reads"] + s["pseudo_reads"] - if n >= PSV_MIN_READS: - s["estimate"] = b["total"] * s["gene_reads"] / n - s["copies"] = round(s["estimate"]) - read.append(s) - if not read: - b.update(status="not assessed", reason="psv_low_depth", copies=None) - return b - est = sorted(s["estimate"] for s in read) - mid = len(est) // 2 - median = est[mid] if len(est) % 2 else (est[mid - 1] + est[mid]) / 2 - for s in read: - s["ratio"] = s["estimate"] / median if median > 0 else None - floor = max(round(median), usual) - conv = [name for name, s in sites.items() if s["estimate"] is not None and s["estimate"] < floor - PSV_MARGIN] - # Only a site past the margin lowers the count, so one noisy site never - # gives fewer copies than the finding says. - b["copies"] = min(sites[n]["copies"] for n in conv) if conv else round(median) - if conv and b["module_total"] < b["expected"]: - b["finding"] = "deletion" - elif conv: - b.update(finding="conversion", conversion_sites=conv) - elif b["module_total"] < b["expected"]: - b["finding"] = "pseudogene_deletion" - else: - b["finding"] = "none" - text = {name: t for name, *_, t in row["_psv"]} - b["reading"] = reading(row, b["finding"]).replace( - "{sites}", ", ".join(f"{n} ({text[n]})" for n in conv)) - return b - - -def psv_message(row, b): - """'.' for a called psv row, else 'not assessed: ' and why. A low MAPQ 0 - share is spelled out: the reports print this message, and the reason - code alone would not say that the BAM, not the genome, stopped the call.""" - if b["status"] == "called": - return "." - if b["reason"] == "low_mapq0_share": - targets = " and ".join(n.removesuffix("_BODY") for n, *_ in row["_targets"]) - return (f"not assessed: the BAM has few or no multi-mapping reads here (reads of mapping quality 0 make " - f"up {100 * b['mapq0_share']:.1f}% of the depth over {targets}, under " - f"{100 * row['_mapq0_min']:g}%), a possible sign of an aligner that leaves out the reads that fit both " - "copies (as novoalign does by default); the module total may then come out low and read as a " - "deletion; realign with minimap2 (step 02) or BWA-MEM") - return f"not assessed: {b['reason'] or '.'}" - - -def psv_table(row, b): - """([(metric, value)], one line) of a psv row's step table.""" - f = (lambda x: "." if x is None else f"{x:.2f}") - d = (lambda x: "." if x is None else str(x)) - rows = [("status", b["status"]), ("cyp21a2_copies", d(b["copies"])), ("module_total", d(b.get("module_total"))), - ("expected_total", d(b["expected"])), ("estimate", f(b["estimate"])), ("finding", b.get("finding") or "."), - ("conversion_sites", ",".join(b.get("conversion_sites") or []) or ".")] - for name, s in (b.get("sites") or {}).items(): - rows.append((f"psv_{name}", f"{s['gene_reads']} gene, {s['pseudo_reads']} pseudogene, {s['other_reads']} other" - f" reads; copies {d(s['copies'])} ({f(s['estimate'])}); ratio {f(s['ratio'])}")) - rows += [("target_depth", f(b["target_depth"])), ("control_depth", f(b["control_depth"])), - ("reason", b["reason"] or "."), ("reading", b["reading"] or "."), - ("mapq0_share", "." if b.get("mapq0_share") is None else f"{b['mapq0_share']:.3f}"), - ("message", psv_message(row, b))] - line = (f"{row['locus']}: {b['status']} ({b['reason'] or b.get('finding')}; {d(b['copies'])} gene copies, " - f"module total {d(b.get('module_total'))} of {d(b['expected'])})") - return rows, line - - -def interval_rows(cat): - """The rows `intervals` calls: kind interval with one bedcov pass (LCE3BC, - an interval row with a MAPQ >= 1 pass, stays a gene-dosage row).""" - return [r for r in cat.values() if r["kind"] == "interval" and r["input"] == "bedcov" and r["_passes"] == ["."]] - - -def _bed(args): - cat = load_catalog(args.catalog) - if args.psv: - if not args.locus or args.kind: - raise CatalogError("bed --psv needs --locus (and no --kind)") - for loc in args.locus: - if loc not in cat or not cat[loc]["_psv"]: - raise CatalogError(f"no psv locus {loc!r} in the catalog") - r = cat[loc] - pos = [(s, f"{n}") for n, s, *_ in r["_psv"]] + [(p, f"{n}_pseudo") for n, _, p, *_ in r["_psv"]] - for s, n in sorted(pos): - print(f"{r['chrom']}\t{s}\t{s + 1}\t{n}") - return 0 - if args.kind == "interval": - if args.locus: - raise CatalogError("bed: --kind interval or --locus, not both") - loci = [r["locus"] for r in interval_rows(cat)] - else: - loci = args.locus or [k for k, r in cat.items() if r["input"] == "bedcov"] - for loc in loci: - if loc not in cat or cat[loc]["input"] != "bedcov": - raise CatalogError(f"no bedcov locus {loc!r} in the catalog") - for r in regions(cat[loc]): - print("\t".join(map(str, r))) - return 0 - - -def _call_cmd(args): - try: - if args.bam_header: - check_bam(args.bam_header) - except AltContigs as e: - print(e, file=sys.stderr) - return 4 - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - cat = load_catalog(args.catalog) - if args.locus not in cat or cat[args.locus]["input"] != "bedcov": - raise CatalogError(f"no bedcov locus {args.locus!r} in the catalog") - row = cat[args.locus] - m = read_bedcov(args.bedcov, row) - guarded = row["_mapq0_min"] is not None or row["_mapq0_max"] is not None - if guarded and not args.bedcov_mapq1: - raise ValueError(f"{args.locus} has a MAPQ 0 guard: --bedcov-mapq1 is needed") - q1 = read_bedcov(args.bedcov_mapq1, row) if args.bedcov_mapq1 else None - if q1 is not None and bool(m) != bool(q1): - raise ValueError(f"{args.bedcov_mapq1 if m else args.bedcov} is empty but the other table is not: " - "both must cover the regions, or both be empty (no reads on that contig)") - block = assess(row, m, q1) - if row["_psv"]: - if not args.pileup: - raise ValueError(f"{args.locus} is a psv row: --pileup (samtools mpileup over `bed --psv`) is needed") - block = psv_assess(row, block, read_pileup(args.pileup, row) if m else {}) - except BrokenControl as e: - print(f"ERROR: {e}", file=sys.stderr) - return 3 - except (OSError, ValueError, KeyError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - rows, line = table(row, block) - try: - if args.json: - with open(args.json, "w") as f: - json.dump(block, f, indent=1, sort_keys=True) - f.write("\n") - if args.table: - with open(args.table, "w") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - print(line) - return 0 - - -def read_bedcov_all(path): - """[(chrom, start, end, name, depth sum)] of a `samtools bedcov` table.""" - out = [] - with open(path, errors="replace") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5: - raise ValueError(f"{path}: expected chrom, start, end, name, depth sum; got {line.strip()!r}") - out.append((c[0], int(c[1]), int(c[2]), c[3], float(c[4]))) - return out - - -def interval_line(row, block): - """One line of the interval table for a row and its block.""" - tg = row["_targets"] - f = (lambda x: "." if x is None else f"{x:.2f}") - copies, exp = block["copies"], block["expected"] - change = "." if copies is None or exp is None else \ - "deletion" if copies < exp else "duplication" if copies > exp else "no change" - return [row["locus"], tg[0][3] or row["locus"], row["chrom"], str(min(s for _, s, _, _ in tg)), - str(max(e for _, _, e, _ in tg)), block["status"], "." if copies is None else str(copies), - "." if exp is None else str(exp), change, f(block["estimate"]), block["reason"] or "."] - - -def call_intervals(cat, rows_in, sex): - """[line] of every interval row from one bedcov table (rows_in, as - read_bedcov_all gives it); a broken control makes that row 'not - assessed' (broken_control), never the whole table.""" - by_name = {} - for c, s, e, n, d in rows_in: - if n in by_name: - raise ValueError(f"segment {n} appears twice in the bedcov table") - by_name[n] = (c, s, e, d) - out = [] - for row in interval_rows(cat): - want = regions(row) - have = [w for w in want if w[3] in by_name] - if len(have) != len(want): - raise ValueError(f"the bedcov table lacks some of {row['locus']}'s segments: " - f"its regions differ from `copy_number.py bed --kind interval`") - m = {} - for c, s, e, n in have: - if by_name[n][:3] != (c, s, e): - raise ValueError(f"segment {n} of {row['locus']} has other coordinates in the bedcov table") - m[n] = by_name[n][3] / (e - s) - try: - block = assess(row, m, None, sex) - except BrokenControl as e: - print(f"WARNING: {e}", file=sys.stderr) - block = assess(row, {}, None, sex) - if block["status"] == "not assessed": - block["reason"] = "broken_control" - out.append(interval_line(row, block)) - return out - - -def _intervals(args): - try: - cat = load_catalog(args.catalog) - if args.failed: - lines = [interval_line(r, {"status": "not assessed", "copies": None, "estimate": None, - "reason": "check_failed", "expected": None}) for r in interval_rows(cat)] - else: - if not args.bedcov or not args.sex: - raise ValueError("intervals needs --bedcov and --sex (or --failed)") - lines = call_intervals(cat, read_bedcov_all(args.bedcov), args.sex) - except (OSError, ValueError, KeyError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - with open(args.out + ".tmp", "w") as f: - f.write("\t".join(INTERVAL_COLUMNS) + "\n") - for ln in lines: - f.write("\t".join(ln) + "\n") - os.replace(args.out + ".tmp", args.out) - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - n = {} - for ln in lines: - n[ln[5]] = n.get(ln[5], 0) + 1 - print(f"interval rows: {len(lines)} ({', '.join(f'{v} {k}' for k, v in sorted(n.items()))})") - return 0 - - -def table(row, block): - """([(metric, value)], one summary line) of a locus's step table.""" - if row["_psv"]: - return psv_table(row, block) - mod = TABLES.get(row["locus"]) - if mod: - return importlib.import_module(mod).table(block) - f = (lambda x: "." if x is None else f"{x:.2f}") - rows = [("status", block["status"]), ("copies", "." if block["copies"] is None else block["copies"]), - ("estimate", f(block["estimate"])), ("target_depth", f(block["target_depth"])), - ("control_depth", f(block["control_depth"])), ("reason", block["reason"] or "."), - ("reading", block["reading"] or ".")] - return rows, f"{row['locus']}: {block['status']} ({block['reason'] or block['reading']})" - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="copy_number.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - b = sub.add_parser("bed", help="print the segments of the loci") - b.add_argument("--catalog") - b.add_argument("--locus", action="append") - b.add_argument("--kind", choices=["interval"], help="every interval row (as `intervals` calls them)") - b.add_argument("--psv", action="store_true", help="the PSV sites of a psv row (gene and pseudogene positions)") - c = sub.add_parser("call", help="the copy number of one locus from samtools bedcov over its segments") - c.add_argument("--catalog") - c.add_argument("--locus", required=True) - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", help="samtools bedcov with the row's MAPQ >= 1 pass") - c.add_argument("--json") - c.add_argument("--table") - c.add_argument("--bam-header", help="refuse a BAM with ALT or HLA contigs first (as check-bam)") - c.add_argument("--pileup", help="samtools mpileup over `bed --psv` (a psv row)") - iv = sub.add_parser("intervals", help="call every interval row from one samtools bedcov table") - iv.add_argument("--catalog") - iv.add_argument("--bedcov", help="samtools bedcov over `copy_number.py bed --kind interval`") - iv.add_argument("--sex", choices=SEXES) - iv.add_argument("--failed", action="store_true", help="every row not assessed (check_failed)") - iv.add_argument("--out", required=True) - k = sub.add_parser("check-bam", help="refuse a BAM aligned to a reference with ALT or HLA contigs") - k.add_argument("--header", required=True, help="samtools view -H of the BAM, or contig names; - for stdin") - args = ap.parse_args(argv) - if args.cmd == "check-bam": - try: - check_bam(args.header) - except AltContigs as e: - print(e, file=sys.stderr) - return 4 - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - if args.cmd == "intervals": - return _intervals(args) - if args.cmd == "bed": - try: - return _bed(args) - except (OSError, CatalogError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return _call_cmd(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/couple_report.py b/bin/couple_report.py index 9cd5d3c45..b41a24d44 100755 --- a/bin/couple_report.py +++ b/bin/couple_report.py @@ -442,7 +442,7 @@ def cyp21a2_finding(path, genes): copy left.""" t = cyp21a2_table(path) if t.get("status") not in ("called", "not assessed"): - die(["%s: not step 35's CYP21A2 table (bin/copy_number.py call --locus CYP21A2)" % path]) + die(["%s: not step 35's CYP21A2 table (og cn call --locus CYP21A2)" % path]) if t["status"] != "called": return [], "CYP21A2 not assessed (%s)" % t.get("reason", ".") what = "CYP21A2 %s copies, RCCX module total %s of %s" % (t.get("cyp21a2_copies", "."), t.get("module_total", "."), @@ -489,7 +489,7 @@ def read_cn_intervals(path): lines = [ln.rstrip("\n").split("\t") for ln in f if ln.strip()] if (not lines or lines[0][:6] != ["locus", "name", "chrom", "start", "end", "status"] or not {"copies", "expected", "change", "reason"} <= set(lines[0])): - die(["%s: not step 35's interval table (bin/copy_number.py intervals)" % path]) + die(["%s: not step 35's interval table (og cn intervals)" % path]) head = lines[0] return {r[0]: dict(zip(head, r)) for r in lines[1:] if len(r) == len(head)} diff --git a/bin/hba_copy_number.py b/bin/hba_copy_number.py deleted file mode 100755 index 302e35a50..000000000 --- a/bin/hba_copy_number.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -"""hba_copy_number.py: HBA1 and HBA2 copy number from short-read depth (step 35). - -Parascopy's pinned models have none for the alpha-globin genes, so step 35 -counts them from read depth instead. HBA1 and HBA2 are near-identical, so -short reads cannot be placed on one of them; but an aligner that reports -every read puts each read from either on one of the two (with mapping -quality 0 where it fits both equally), so the depth over both gene bodies -together, against -single-copy sequence on each side of the cluster, counts the alpha-globin -genes: four on a genome without a deletion (aa/aa), three with one gene lost -on one chromosome (-a3.7 or -a4.2), two with both lost on one chromosome -(--SEA and other large deletions) or one on each (-a/-a). - -The method and the regions are the HBA row of bin/cn_loci.tsv, run by -bin/copy_number.py (steps 35 and the PARASCOPY module call that engine); -this file lays out the step's table and keeps its own command line. - - hba_copy_number.py bed - print the regions (GRCh38, 0-based BED, name in column 4) to stdout - hba_copy_number.py check --bedcov BEDCOV.tsv --bedcov-mapq1 BEDCOV_Q1.tsv --out HBA.tsv - read `samtools bedcov` over those regions (chrom, start, end, name, - summed depth; all MAPQ, duplicates left out) and `samtools bedcov -Q 1` - over the same regions (reads of MAPQ 1 or more only), and write HBA.tsv - (metricvalue: status, alpha_globin_genes, estimate, the depths, - mapq0_share and the message). Exit 0 whatever the status; 2 on - unreadable input. - -status is 'called' or 'not assessed'. The estimate is -2 x (HBA2 depth + HBA1 depth) / flank depth. It is called only when it lies -within 0.3 of a whole number, the flanks reach MIN_FLANK_DEPTH and agree -with each other, and (unless the estimate rounds to 0) reads of MAPQ 0 make -up at least MIN_MAPQ0_SHARE of the depth over the two genes; otherwise the -message says why not. Standard library only: -it runs in the Parascopy image and under tests/test_hba_copy_number.py. -""" -import argparse -import sys - -import copy_number as cn - -LOCUS = "HBA" -_ROW = [] - - -def row(): - """The HBA row of the catalog, read once.""" - if not _ROW: - _ROW.append(cn.load_catalog()[LOCUS]) - return _ROW[0] - - -def __getattr__(name): - # The regions and thresholds come from the catalog row, read on first use. - r = row() - vals = {"REGIONS": cn.regions(r), "MIN_FLANK_DEPTH": r["_min_control"], "MAX_FLANK_RATIO": r["_max_ratio"], - "MAX_OFF": r["_call"][1][0], "MIN_MAPQ0_SHARE": r["_mapq0_min"]} - if name in vals: - return vals[name] - raise AttributeError(name) - - -def read_bedcov(path): - """{'flank_left': mean, 'HBA2': mean, ...} from `samtools bedcov` output - over the regions, which must all be there and nothing else.""" - m = cn.read_bedcov(path, row()) - if not m: - raise ValueError(f"{path}: its regions differ from the ones `hba_copy_number.py bed` prints (empty)") - return m - - -def table(b): - """([(metric, value)], summary line) of the engine's HBA block.""" - r, m = row(), b["depths"] - if b["reason"] == "no_data": - out = {k: "." for k in ORDER} - out.update(status="not assessed", message="not assessed: the BAM has no chr16 reads, so HBA1 and HBA2 have none") - return [(k, out[k]) for k in ORDER], f"HBA1/HBA2: {out['message']}" - left, right = m.get("flank_left"), m.get("flank_right") - share = b["mapq0_share"] - out = {"hba2_depth": round(m["HBA2"], 2), "hba1_depth": round(m["HBA1"], 2), - "flank_left_depth": round(left, 2), "flank_right_depth": round(right, 2), - "mapq0_share": f"{share:.3f}", - "estimate": ".", "alpha_globin_genes": ".", "status": "not assessed"} - reason, est = b["reason"], b["estimate"] - if est is not None: - out["estimate"] = f"{est:.2f}" - if reason == "low_control_depth": - out["message"] = (f"not assessed: flank depth under {r['_min_control']:g}x " - f"({left:.1f}x and {right:.1f}x), too low to compare") - elif reason == "controls_disagree": - out["message"] = (f"not assessed: the two flanks disagree ({left:.1f}x and {right:.1f}x); " - "a deletion larger than --SEA, or uneven coverage, would skew the count") - elif reason == "low_mapq0_share": - out["message"] = (f"not assessed: reads of mapping quality 0 make up {100 * share:.1f}% of the depth " - "over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as " - "novoalign does by default) and the count would come out low; realign with " - "minimap2 (step 02) or BWA-MEM") - elif reason == "between_counts": - out["message"] = (f"not assessed: the estimate {est:.2f} lies between {int(est)} and {int(est) + 1} " - "alpha-globin genes") - else: - out.update(status="called", alpha_globin_genes=str(b["copies"]), message=b["reading"]) - line = (f"HBA1/HBA2: {out['message']} (estimate {out['estimate']}; HBA2 {out['hba2_depth']}x, " - f"HBA1 {out['hba1_depth']}x, flanks {out['flank_left_depth']}x and {out['flank_right_depth']}x; " - f"MAPQ 0 share over the genes {out['mapq0_share']})") - return [(k, out[k]) for k in ORDER], line - - -def assess(m, q1): - """The result dict for the mean depths of read_bedcov: m over every - read, q1 over reads of MAPQ 1 or more.""" - return dict(table(cn.assess(row(), m, q1))[0]) - - -ORDER = ("status", "alpha_globin_genes", "estimate", "hba2_depth", "hba1_depth", - "flank_left_depth", "flank_right_depth", "mapq0_share", "message") - - -def check(args): - try: - rows, line = table(cn.assess(row(), read_bedcov(args.bedcov), read_bedcov(args.bedcov_mapq1))) - except cn.BrokenControl as e: - print(f"ERROR: {e}", file=sys.stderr) - return 3 - except (OSError, ValueError, KeyError, ZeroDivisionError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - with open(args.out, "w") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - except OSError as e: - print(f"ERROR: cannot write {args.out}: {e}", file=sys.stderr) - return 2 - print(line) - return 0 - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="hba_copy_number.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - sub.add_parser("bed", help="print the regions") - c = sub.add_parser("check", help="judge samtools bedcov over the regions") - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", required=True, help="samtools bedcov -Q 1 over the same regions") - c.add_argument("--out", required=True) - args = ap.parse_args(argv) - if args.cmd == "bed": - for r in cn.regions(row()): - print("\t".join(map(str, r))) - return 0 - return check(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/lpa_kiv2.py b/bin/lpa_kiv2.py deleted file mode 100755 index ff8183e63..000000000 --- a/bin/lpa_kiv2.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 -"""lpa_kiv2.py: the LPA KIV-2 repeat count from short-read depth (step 46). - -Lp(a) is a mostly inherited cardiovascular risk factor, and the number of -kringle IV type 2 (KIV-2) repeats in LPA sets the size of apo(a): fewer -repeats, a smaller isoform and, on average, a higher Lp(a) level. The KIV-2 -repeat is a 5.5 kb unit with two exons; a Danish population study counted 6 -to 99 on both chromosomes together (Kamstrup et al. 2009, PMID 19509380). -GRCh38 holds six units, and their exons are near-identical, so an aligner -that reports every read places each KIV-2 read on one of the six (with -mapping quality 0). The depth over those exons, against the depth over -single-copy LPA exons, counts the units the sample carries: the approach of -the published short-read estimators (Zekavat et al. 2018, PMID 29973585; -Behera et al. 2024, PMID 39449055, which also gives the GRCh38 region and the -six reference units). - -The regions and the method are the LPA_KIV2 row of bin/cn_loci.tsv, run by -bin/copy_number.py (step 46 and the LPA processes call that engine); this -file lays out the step's table and keeps its own command line. - - lpa_kiv2.py bed - print the regions (GRCh38, 0-based BED, name in column 4) to stdout - lpa_kiv2.py estimate --bedcov BEDCOV.tsv --bedcov-mapq1 BEDCOV_Q1.tsv --out LPA.tsv - read `samtools bedcov -j` over those regions (chrom, start, end, name, - summed depth; all MAPQ, duplicates left out) and `samtools bedcov -j -Q 1` - over the same regions, and write LPA.tsv (metricvalue). Exit 0 - whatever the status; 2 on unreadable input. Two empty tables mean the - BAM has no chr6: status 'not assessed'. - lpa_kiv2.py null-allele --pileup PILEUP.tsv --table LPA.tsv --out NULL.tsv - read `samtools mpileup` (no reference, every MAPQ) at the seven copies - of the KIV-2 4925G>A site (`traits_report.py --marker-regions - LPA_KIV2_4925`) and write its reading to NULL.tsv (metricvalue), - with the count N of LPA.tsv (written by `estimate` or `copy_number.py - call --locus LPA_KIV2`, left as it is). Exit 0 whatever the reading (an - unreadable pileup reads 'not assessed'); 2 when NULL.tsv cannot be - written. - -The regions are exons of the MANE Select transcript ENST00000316300.10, -numbered from its first exon: - kiv2_e03 to kiv2_e16 the six KIV-2 units of GRCh38 (exons 4 to 15), plus - exon 3 (the last exon of KIV-1) and exon 16 (the first - of KIV-3), which are near-identical to KIV-2 exons and - take some of its reads. Seven 160 bp and seven 182 bp - exons, 2,394 bp: six KIV-2 units plus one single-copy - pair per chromosome. - flank_e19 to flank_e33 single-copy kringle exons downstream (the second - exon of KIV-4 to KV), under 90% identical to any KIV-2 - exon. Exons 17 and 18 (about 95% identical) are in - neither group, nor are exons 1 and 2 or the protease - exons, which resemble plasminogen (PLG). - -With D the mean depth over the flank exons (two copies) and M the mean depth -over the KIV-2 block, the block holds N + 2 copies of a 342 bp pair spread -over seven reference pairs, so M = (N + 2) x D / 14 and - N = 14 x M / D - 2 (the total over both chromosomes) -The 95% interval comes from the spread of the per-exon depths in each group: -relative standard error sqrt(cv_kiv2^2 / 14 + cv_flank^2 / 15), applied to -N + 2 on the log scale. It covers sampling noise only, not a systematic bias -of an aligner or a library. - -status is 'estimated', 'too little depth' or 'not assessed'; the message says -why. - -The KIV-2 4925G>A splice variant (Coassin et al. 2017, PMID 28444229) sits at -the donor splice site of the second KIV-2 exon: carriers had a much lower -Lp(a) than non-carriers with the same isoform size, most of all with small -isoforms, and it is usually in one or two of a person's KIV-2 repeats. GRCh38 -holds the site in each of its six units and in KIV-1, so null-allele adds up -the reads at all seven (bin/traits_report.py MARKERS) and reads the share -with the variant as a fraction of copies: status is 'present', 'not seen', -'uncertain', 'too little depth' or 'not assessed', and with a count, copies -is the fraction x (N + 2), the copies that carry it. -A short-read fraction cannot say which chromosome or which repeat carries -it, so it cannot say whether it sits on the small or the large isoform. Research estimate, not a diagnostic test: the actionable test is a blood -Lp(a) measurement. Standard library only: it runs in the Python image and -under tests/test_lpa_kiv2.py. -""" -import argparse -import os -import sys - -import copy_number as cn - -LOCUS = "LPA_KIV2" -_ROW = [] -ADVICE = ("A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 " - "European Atherosclerosis Society consensus advises measuring it at least once in every adult.") - - -def row(): - """The LPA_KIV2 row of the catalog, read once.""" - if not _ROW: - _ROW.append(cn.load_catalog()[LOCUS]) - return _ROW[0] - - -def __getattr__(name): - # The regions and thresholds come from the catalog row, read on first use. - r = row() - if name == "REGIONS": # in catalog order: the KIV-2 block, then the single-copy exons - return [(r["chrom"], s, e, n) for n, s, e, _ in r["_targets"] + r["_controls"]] - vals = {"MIN_FLANK_DEPTH": r["_min_control"], "MIN_MAPQ0_SHARE": r["_mapq0_min"], - "LOW": r["_call"][1][0], "HIGH": r["_call"][1][1], "READINGS": r["_readings"]} - if name in vals: - return vals[name] - raise AttributeError(name) - - -def bed_lines(): - return ["\t".join(map(str, r)) for r in cn.regions(row())] - - -def read_bedcov(path): - """{name: mean depth} from `samtools bedcov` output over the regions, - which must all be there and nothing else; {} for an empty table (no chr6).""" - return cn.read_bedcov(path, row()) - - -def table(b): - """([(metric, value)], summary line) of the engine's LPA_KIV2 block.""" - r = row() - out = {"status": "not assessed", "kiv2_copies": ".", "ci95_low": ".", "ci95_high": ".", "class": ".", - "kiv2_depth": ".", "flank_depth": ".", "mapq0_share": ".", "message": "", "advice": ADVICE} - reason = b["reason"] - if reason == "no_data": - out["message"] = "not assessed: the BAM has no chr6, so LPA has no reads" - else: - kdepth, fdepth, share = b["target_depth"], b["control_depth"], b["mapq0_share"] - out.update(kiv2_depth=f"{kdepth:.2f}", flank_depth=f"{fdepth:.2f}", mapq0_share=f"{share:.3f}") - if reason == "low_control_depth": - out["status"] = "too little depth" - out["message"] = (f"too little depth: {fdepth:.1f}x over the single-copy LPA exons, under the " - f"{r['_min_control']:g}x the count needs") - elif reason == "low_mapq0_share": - out["message"] = (f"not assessed: reads of mapping quality 0 make up {100 * share:.1f}% of the depth " - "over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 " - "copies (as novoalign does by default) and the count would come out low; realign " - "with minimap2 (step 02) or BWA-MEM") - elif reason == "below_floor": - out["message"] = (f"not assessed: the KIV-2 exons have {kdepth:.1f}x against {fdepth:.1f}x over the " - "single-copy exons, too little for any repeat count (a deletion or a mapping problem)") - else: - est, (lo, hi) = b["estimate"], b["ci95"] - out.update(status="estimated", kiv2_copies=f"{est:.1f}", ci95_low=f"{lo:.1f}", ci95_high=f"{hi:.1f}", - **{"class": b["class"]}, - message=(f"about {est:.0f} KIV-2 repeats in total (95% interval {lo:.1f} to {hi:.1f}): " - f"{b['reading']}")) - line = (f"LPA KIV-2: {out['message']} (KIV-2 exons {out['kiv2_depth']}x, single-copy exons " - f"{out['flank_depth']}x, MAPQ 0 share {out['mapq0_share']})") - return [(k, out[k]) for k in ORDER], line - - -def assess(m, q1): - """The result dict for the mean depths of read_bedcov: m over every read, - q1 over reads of MAPQ 1 or more.""" - out = dict(table(cn.assess(row(), m, q1))[0]) - del out["advice"] - return out - - -ORDER = ("status", "kiv2_copies", "ci95_low", "ci95_high", "class", "kiv2_depth", "flank_depth", - "mapq0_share", "message", "advice") - - -NULL_LIMIT = ("A short-read fraction cannot say which chromosome or which repeat carries it, so it cannot say " - "whether it sits on the small or the large isoform.") - - -def null_allele(counts, kiv2_copies=".", why_not=None): - """[(metric, value)] of the KIV-2 4925G>A table from - traits_report.read_marker_pileup's counts (None: unreadable), with the - count N of step 46's table ('.' when there is none). why_not: the reason - the reads cannot be trusted (the aligner left out the reads of MAPQ 0), - which makes it 'not assessed' whatever the counts.""" - import traits_report as tr # bin/: the copy-aware marker catalog - m = tr.MARKERS["LPA_KIV2_4925"] - a = tr.assess_marker("LPA_KIV2_4925", None if why_not else counts) - st = a["status"] - try: - total = float(kiv2_copies) + 2 - except ValueError: - total = None - copies = "." - if st in ("present", "uncertain") and total is not None: - copies = f"{float(a['fraction']) * total:.1f}" - name = f"{m['name']} (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017)" - if st == "not assessed": - msg = f"{name}: not assessed, " + (why_not or "the reads at its seven GRCh38 copies could not be read.") - else: - msg = (f"{name}: {st}, {a['alt_reads']} of {a['depth']} reads at its seven GRCh38 copies carry it" - + (f" (about {copies} of the {total:.0f} copies counted)" if copies != "." else "") - + (f", under the {m['min_depth']} reads a reading needs" if st == "too little depth" else "") - + ".") - if st == "present": - msg += (" Carriers had a lower Lp(a) than non-carriers with the same number of repeats, most of " - "all with few repeats. " + NULL_LIMIT) - elif st == "not seen": - msg += (" Not seen in the reads is not proof of absence: one carrying repeat among many gives few " - "reads, and a short-read fraction never says which chromosome or repeat holds a variant.") - else: - msg += " " + NULL_LIMIT - return [("marker", m["name"]), ("rsid", m["dbsnp"][0] + " (first KIV-2 copy)"), ("status", st), - ("alt_reads", a["alt_reads"]), ("depth", a["depth"]), ("fraction", a["fraction"]), ("copies", copies), - ("kiv2_copies", kiv2_copies), ("message", msg)] - - -def write_null_allele(table_path, pileup_path, out_path): - """Write the KIV-2 4925G>A table beside step 46's table (read for the - count N; a table that is missing or unreadable gives no N). Returns the message.""" - import traits_report as tr - t = {} - try: - with open(table_path, errors="replace") as f: - t = dict(ln.rstrip("\n").split("\t", 1) for ln in f if "\t" in ln) - except OSError: - pass - n = t.get("kiv2_copies", ".") - why_not = None - try: - share = float(t.get("mapq0_share", ".")) - except ValueError: - share = None - if share is not None and share < row()["_mapq0_min"]: - # The reads that fit several copies are the ones a carrying repeat - # gives; without them the share at the seven copies is biased. - why_not = (f"reads of mapping quality 0 make up {100 * share:.1f}% of the KIV-2 depth, so the aligner " - "left out the reads that fit several copies, the ones a carrying repeat gives.") - try: - counts = tr.read_marker_pileup(pileup_path, "LPA_KIV2_4925") - except (OSError, ValueError) as e: - print(f"WARNING: {e}; KIV-2 4925G>A is not assessed", file=sys.stderr) - counts = None - rows = null_allele(counts, n, why_not) - tr.write_metric_table(out_path, rows) - return dict(rows)["message"] - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="lpa_kiv2.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - sub.add_parser("bed", help="print the regions") - c = sub.add_parser("estimate", help="count the KIV-2 repeats from samtools bedcov over the regions") - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", required=True, help="samtools bedcov -j -Q 1 over the same regions") - c.add_argument("--out", required=True) - n = sub.add_parser("null-allele", help="the KIV-2 4925G>A table beside the count") - n.add_argument("--pileup", required=True, help="samtools mpileup at traits_report.py --marker-regions LPA_KIV2_4925") - n.add_argument("--table", required=True, help="the table estimate wrote (for the count)") - n.add_argument("--out", required=True, help="the 4925G>A table") - args = ap.parse_args(argv) - if args.cmd == "null-allele": - try: - print("LPA " + write_null_allele(args.table, args.pileup, args.out)) - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - if args.cmd == "bed": - print("\n".join(bed_lines())) - return 0 - return cn.main(["call", "--locus", LOCUS, "--bedcov", args.bedcov, "--bedcov-mapq1", args.bedcov_mapq1, - "--table", args.out]) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/traits_report.py b/bin/traits_report.py index cd4fbac98..6b984f1cb 100755 --- a/bin/traits_report.py +++ b/bin/traits_report.py @@ -40,27 +40,15 @@ Copy-aware markers (MARKERS): two well-sourced markers of the rule catalog's list sit in sequence the genome holds more than once, so a gVCF genotype at one position cannot read them and they are not rule lines. Each is read from -the BAM instead, by the step that counts its copies: the reads at every +the BAM instead, by the step that counts its copies (og cn marker-regions, +marker-read and call --marker, in rust/crates/og-cn): the reads at every GRCh38 copy of the site are added up, and the share that carries the marker's -base is a fraction of copies, not a diploid genotype. +base is a fraction of copies, not a diploid genotype. MARKERS stays here for +the reports (bin/collect_summary.py reads the marker tables with it). LPA_KIV2_4925 the LPA KIV-2 4925G>A splice variant (Coassin et al. 2017, PMID 28444229), read by step 46 beside the KIV-2 count - (laid out by bin/lpa_kiv2.py) SMN1_27134 SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID 23788250), read by step 35 beside the SMN1 psCN - - traits_report.py --marker-regions NAME - print the marker's copy sites as samtools regions (chrom:pos-pos) - traits_report.py --marker LPA_KIV2_4925 --pileup P --copy-table LPA.tsv --marker-out OUT.tsv - the 4925G>A table beside step 46's table LPA.tsv, whose count gives - the copies that carry it (laid out by bin/lpa_kiv2.py); exit 2 when - OUT.tsv cannot be written - traits_report.py --marker SMN1_27134 --pileup P [--copy-table SMN.tsv] --marker-out OUT.tsv - read `samtools mpileup` at those sites (no reference, every MAPQ) and - write OUT.tsv (metricvalue). SMN.tsv is step 35's SMN table: the - SMN1 psCN at exon 7 and the agCN at the marker site come from it. Exit - 0 whatever the status (an unreadable pileup is 'not assessed'); 2 when - OUT.tsv cannot be written. """ import argparse import itertools @@ -538,7 +526,8 @@ def report(rows, recs, rules=None, phased=None, lines=None): # Ensembl GRCh38 agree, 2026-11-07): the same 21 bases at each copy, which is # why the reads cannot be placed on one copy. tests/test_traits_report.py # checks every copy against the context and the record against the marker, and -# tests/e2e/genotype-rules-2-smn1-marker.sh compares the records with dbSNP. +# that og cn reads the same copies; tests/e2e/genotype-rules-2-smn1-marker.sh +# compares the records with dbSNP. # min_depth fewer reads over all copies together: too little depth # min_alt reads with the marker's base for 'present' ... # min_fraction ... and a share of the reads at least this @@ -551,7 +540,7 @@ def report(rows, recs, rules=None, phased=None, lines=None): # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, - # the block bin/lpa_kiv2.py counts as N + 2 copies (N KIV-2 repeats and + # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. "LPA_KIV2_4925": { "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", @@ -571,9 +560,6 @@ def report(rows, recs, rules=None, phased=None, lines=None): }, } MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") -# SMN1 exon 7 (c.840, chr5:70,951,946): where step 35's report reads the SMN1 psCN. -SMN1_EXON7 = ("chr5", 70951946) -SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." def marker_errors(name, m=None, dbsnp=None): @@ -602,159 +588,6 @@ def marker_errors(name, m=None, dbsnp=None): return errs -def marker_regions(name): - """The copy sites as samtools regions, chrom:pos-pos, in the marker's order.""" - m = MARKERS[name] - return [f"{m['chrom']}:{p}-{p}" for p in m["copies"]] - - -def read_marker_pileup(path, name): - """(ref reads, marker reads, other reads) over every copy of the marker, - from `samtools mpileup` lines (chrom, pos, ref, depth, bases, ...) at its - sites, without a reference: a read that carries the marker's base counts - wherever the aligner put it. Lines at other positions are left out; a - copy with no line has no read. Raises ValueError on a line that is not - an mpileup line.""" - import copy_number # bin/: its mpileup base counter, shared with the CYP21A2 sites - m = MARKERS[name] - want = set(m["copies"]) - n = dict.fromkeys("ACGT", 0) - with open(path, errors="replace") as f: - for line in f: - if not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5 or not c[1].isdigit(): - raise ValueError(f"{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); " - f"got {line.strip()[:80]!r}") - if c[0] != m["chrom"] or int(c[1]) not in want: - continue - for k, v in copy_number.count_bases(c[4], c[2]).items(): - n[k] += v - ref, alt = n[m["ref"]], n[m["alt"]] - return ref, alt, sum(n.values()) - ref - alt - - -def assess_marker(name, counts): - """{status, alt_reads, depth, fraction} of a marker from read_marker_pileup's - counts (None: the pileup could not be read). depth counts the reads with - the ref or the marker's base.""" - m = MARKERS[name] - if counts is None: - return {"status": "not assessed", "alt_reads": ".", "depth": ".", "fraction": "."} - ref, alt, _ = counts - depth = ref + alt - out = {"alt_reads": str(alt), "depth": str(depth), "fraction": f"{alt / depth:.4f}" if depth else "."} - if depth < m["min_depth"]: - out["status"] = "too little depth" - elif alt >= m["min_alt"] and alt / depth >= m["min_fraction"]: - out["status"] = "present" - elif alt <= m["max_absent"]: - out["status"] = "not seen" - else: - out["status"] = "uncertain" - return out - - -def smn_copy_numbers(path): - """(SMN1 psCN at exon 7 or None, its text, agCN at the marker site or None) - from step 35's SMN table. A copy number counts only with filter PASS and - quality 20 or more, the threshold of step 35's reading.""" - def regions(r): - out = [(r["chrom"], int(r["start"]) + 1, int(r["end"]))] - for h in (r.get("homologous_regions") or "").split(","): - mm = re.match(r"^([^:]+):(\d+)-(\d+)", h.strip()) - if mm: - out.append((mm.group(1), int(mm.group(2)), int(mm.group(3)))) - return out - - def value(r, kind, idx): - vals = (r.get(kind) or "").split(",") - quals = (r.get(kind + "_qual") or "").split(",") - v = vals[idx] if idx < len(vals) else "." - q = quals[idx] if idx < len(quals) else "." - try: - ok = (r.get(kind + "_filter") == "PASS" and float(q) >= 20 and v.isdigit()) - except ValueError: - ok = False - return ((int(v) if ok else None), - f"{v} (quality {q}, filter {r.get(kind + '_filter') or '.'})" + ("" if ok else ", not resolved")) - - sm = MARKERS["SMN1_27134"] - ps, ps_text, ag = None, "not read (no SMN table from step 35)", None - if not path or not os.path.exists(path): - return ps, ps_text, ag - ps_text = "not read (no row of the SMN table covers SMN1 exon 7)" - with open(path, errors="replace") as f: - header = f.readline().rstrip("\n").split("\t") - for line in f: - r = dict(zip(header, line.rstrip("\n").split("\t"))) - try: - regs = regions(r) - except (KeyError, ValueError): - continue - for i, (c, s, e) in enumerate(regs): - if c == SMN1_EXON7[0] and s <= SMN1_EXON7[1] <= e: - ps, ps_text = value(r, "psCN", i) - # agCN is one value per row: the row whose region, or a - # homologous region, holds a copy of the marker site - if c == sm["chrom"] and any(s <= p <= e for p in sm["copies"]): - ag = value(r, "agCN", 0)[0] - return ps, ps_text, ag - - -def smn1_marker(counts, smn_table=None): - """[(metric, value)] of step 35's SMN1 marker table: the marker's reading - beside the SMN1 psCN. The message always says the marker does not prove - 2+0 (SMN1_NOT_PROOF).""" - m = MARKERS["SMN1_27134"] - a = assess_marker("SMN1_27134", counts) - try: - ps, ps_text, ag = smn_copy_numbers(smn_table) - except OSError as e: - print(f"WARNING: {e}; the SMN copy numbers are not read", file=sys.stderr) - ps, ps_text, ag = None, "not read (the SMN table could not be read)", None - copies = "." - if a["status"] in ("present", "uncertain") and ag is not None and a["fraction"] != ".": - copies = f"{float(a['fraction']) * ag:.1f}" - st = a["status"] - if st == "not assessed": - head = f"{m['name']} ({m['dbsnp'][0]}): not assessed, the reads at the marker could not be read." - else: - head = (f"{m['name']} ({m['dbsnp'][0]}): {st}, {a['alt_reads']} of {a['depth']} reads carry G at the " - "SMN1 and SMN2 copies of the site together" - + (f" (about {copies} of the {ag} SMN1 and SMN2 copies)" if copies != "." else "") - + (f", under the {m['min_depth']} reads a reading needs" if st == "too little depth" else "") - + ". The reads cannot tell whether a G sits on SMN1 or SMN2.") - if ps == 2: - body = {"present": " With two SMN1 copies, this marker goes with a higher chance that both sit on one " - "chromosome (2+0, a silent carrier): in Luo 2014 it raised the residual carrier risk of a " - "two-copy result (in African Americans from about 1 in 121 to 1 in 34); the figures " - "depend on ancestry.", - "not seen": " With two SMN1 copies and no marker, the chance of 2+0 is lower than for a two-copy " - "result alone (in Luo 2014, in African Americans, about 1 in 396 against 1 in 121), " - "not zero; the figures depend on ancestry."}.get( - st, " With two SMN1 copies the marker would refine the chance of 2+0; here it gives no reading.") - elif ps is not None: - body = (f" SMN1 has {ps} cop{'y' if ps == 1 else 'ies'} here; the marker refines the carrier risk of a " - "two-copy result only.") - else: - body = f" SMN1 copy number: {ps_text}; the marker refines the carrier risk of a two-copy result only." - return [("marker", m["name"]), ("rsid", m["dbsnp"][0]), ("status", st), ("alt_reads", a["alt_reads"]), - ("depth", a["depth"]), ("fraction", a["fraction"]), ("copies", copies), - ("smn1_pscn", ps_text), ("agcn", "." if ag is None else str(ag)), - ("message", head + body + " " + SMN1_NOT_PROOF)] - - -def write_metric_table(path, rows): - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - os.replace(tmp, path) - - def write_tsv(path, out, columns=OUT_COLUMNS): tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as f: @@ -775,41 +608,7 @@ def main(argv=None): ap.add_argument("--rules-out", help="the sample's genotype rules (TSV)") ap.add_argument("--phase-sites", metavar="VCF", help="write the sites WhatsHap must phase and exit") ap.add_argument("--check-table", metavar="TABLE", help="check the table's schema (and --rules') and exit") - ap.add_argument("--marker-regions", metavar="NAME", choices=sorted(MARKERS), - help="print a copy-aware marker's sites as samtools regions and exit") - ap.add_argument("--marker", choices=sorted(MARKERS), help="read a copy-aware marker from --pileup") - ap.add_argument("--pileup", help="samtools mpileup at the --marker-regions sites") - ap.add_argument("--copy-table", help="SMN1_27134: step 35's SMN table (the SMN1 psCN and the agCN); " - "LPA_KIV2_4925: step 46's table (the KIV-2 count)") - ap.add_argument("--marker-out", help="the marker's table (metricvalue)") a = ap.parse_args(argv) - if a.marker_regions: - print("\n".join(marker_regions(a.marker_regions))) - return 0 - if a.marker: - if not (a.pileup and a.marker_out): - ap.error("--marker needs --pileup and --marker-out") - if a.marker == "LPA_KIV2_4925": - import lpa_kiv2 # bin/: step 46's table layout - try: - print("LPA " + lpa_kiv2.write_null_allele(a.copy_table or "", a.pileup, a.marker_out)) - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - try: - counts = read_marker_pileup(a.pileup, a.marker) - except (OSError, ValueError) as e: - print(f"WARNING: {e}; the marker is not assessed", file=sys.stderr) - counts = None - rows = smn1_marker(counts, a.copy_table) - try: - write_metric_table(a.marker_out, rows) - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - print(dict(rows)["message"]) - return 0 try: if a.check_table: rows = read_table(a.check_table) diff --git a/docs/35-paralogs.md b/docs/35-paralogs.md index 295da9f1d..87a01cfa2 100644 --- a/docs/35-paralogs.md +++ b/docs/35-paralogs.md @@ -62,11 +62,11 @@ With Nextflow: `--tools ...,parascopy --parascopy_data ${GENOME_DIR}/reference/p 1. `parascopy depth`: background read depth of the sample, over Parascopy's GRCh38 windows (or `PARASCOPY_DEPTH_BED`). 2. For each row of the catalog: `samtools view -c` counts the reads over the row's region, then `parascopy cn-using models_GRCh38_1KGP//` gives the copy number of the locus with those model parameters. A row whose model is not installed, whose region has no read (or whose reads `samtools` could not count), or on which Parascopy fails or writes no result file is not assessed: its table holds one row that says so, with the reason, and the other rows still run. Before any of this the step stops on a catalog row it cannot use: a malformed model, region or table name, a table name used twice, or the table names `hba` and `paraphase`, which other tables use. 3. Writes one row per region of each locus profile to `${SAMPLE}__copy_number.tsv` (`${SAMPLE}_smn_copy_number.tsv` for the SMN1 row). -4. `samtools bedcov` (in the Parascopy image) sums the depth over HBA2, HBA1 and two flanks, once over every read and once over reads of mapping quality 1 or more (`-Q 1`), and `bin/copy_number.py call --locus HBA` (the copy-number engine, on the HBA row of `bin/cn_loci.tsv`; see [Copy number from depth](cn-loci.md)) writes `${SAMPLE}_hba_copy_number.tsv`, laid out by `bin/hba_copy_number.py`. A failure here prints a warning and writes a table that says not assessed; the SMN result stands. +4. `samtools bedcov` (in the Parascopy image) sums the depth over HBA2, HBA1 and two flanks, once over every read and once over reads of mapping quality 1 or more (`-Q 1`), and `og cn call --locus HBA` (the copy-number engine, on the HBA row of `bin/cn_loci.tsv`; see [Copy number from depth](cn-loci.md)) writes `${SAMPLE}_hba_copy_number.tsv`. A failure here prints a warning and writes a table that says not assessed; the SMN result stands. 5. The gene-dosage rows (HP, LCE3BC, CCL3L1) the same way, into `${SAMPLE}_cn_dosage.tsv` ([the gene-dosage rows](cn-loci.md#the-gene-dosage-rows)). -6. `samtools bedcov` over the CYP21A2 row's segments (two passes, all reads and mapping quality 1 or more, as HBA), `samtools mpileup` without a reference at its PSV sites, then `bin/copy_number.py call --locus CYP21A2 --bedcov-mapq1 ... --pileup ...` writes `${SAMPLE}_cyp21a2_psv.tsv`. A failure prints a warning and writes a table that says not assessed (`check_failed`). +6. `samtools bedcov` over the CYP21A2 row's segments (two passes, all reads and mapping quality 1 or more, as HBA), `samtools mpileup` without a reference at its PSV sites, then `og cn call --locus CYP21A2 --bedcov-mapq1 ... --pileup ...` writes `${SAMPLE}_cyp21a2_psv.tsv`. A failure prints a warning and writes a table that says not assessed (`check_failed`). 7. The interval rows into `${SAMPLE}_cn_intervals.tsv` ([the interval rows](cn-loci.md#the-interval-rows)). -8. SMN1 g.27134T>G beside the SMN1 psCN (before the HBA count): `samtools mpileup` without a reference at its SMN1 and SMN2 positions, then `bin/traits_report.py --marker SMN1_27134` with the SMN table writes `${SAMPLE}_smn1_marker.tsv` ([the SMN1 silent-carrier marker](#the-smn1-silent-carrier-marker)). A failure prints a warning and writes a table that says not assessed. +8. SMN1 g.27134T>G beside the SMN1 psCN (before the HBA count): `samtools mpileup` without a reference at its SMN1 and SMN2 positions, then `og cn marker-read SMN1_27134` with the SMN table writes `${SAMPLE}_smn1_marker.tsv` ([the SMN1 silent-carrier marker](#the-smn1-silent-carrier-marker)). A failure prints a warning and writes a table that says not assessed. ## Output @@ -96,7 +96,7 @@ SMN2 holds the same 21 bases 875,420 bases upstream, so the reads cannot tell an | `too little depth` | under 10 reads over the two positions | | `not assessed` | the pileup could not be read, or the check failed | -The message reads the marker beside the SMN1 psCN at exon 7 (filter `PASS` and quality 20 or more, the rule of the report's SMN1 line): with two SMN1 copies it says whether the marker raises or lowers the chance of 2+0; with another count, or a psCN that is not resolved, it says the marker refines a two-copy result only. Every message ends with "This marker does not prove 2+0, and its absence does not rule 2+0 out." Both reports and the live person page print the marker right under the SMN1 copies: its state (`present` or `uncertain` with the reads that carry G and the copies, `not seen` with the reads, `too little depth`, or `not assessed`), the reader's message, and the caveat that the marker raises the chance that two SMN1 copies sit on one chromosome, does not prove 2+0, and that a carrier test in a clinical laboratory confirms it. Without the table (a run from before this reader) or with a table that has no known status, the line says `not assessed` with the reason. The live card also lists the SMN1 copies and the marker state as facts. The Nextflow `HTML_REPORT` gets the table from the `PARASCOPY` task the same way. +The message reads the marker beside the SMN1 psCN at exon 7 (filter `PASS` and quality 20 or more, from the first row of the SMN table whose region or homologous regions hold exon 7: the rule of the report's SMN1 line and of the Paraphase comparison): with two SMN1 copies it says whether the marker raises or lowers the chance of 2+0; with another count, or a psCN that is not resolved, it says the marker refines a two-copy result only. Every message ends with "This marker does not prove 2+0, and its absence does not rule 2+0 out." Both reports and the live person page print the marker right under the SMN1 copies: its state (`present` or `uncertain` with the reads that carry G and the copies, `not seen` with the reads, `too little depth`, or `not assessed`), the reader's message, and the caveat that the marker raises the chance that two SMN1 copies sit on one chromosome, does not prove 2+0, and that a carrier test in a clinical laboratory confirms it. Without the table (a run from before this reader) or with a table that has no known status, the line says `not assessed` with the reason. The live card also lists the SMN1 copies and the marker state as facts. The Nextflow `HTML_REPORT` gets the table from the `PARASCOPY` task the same way. ## Runtime @@ -187,7 +187,7 @@ A new locus is a new row for a model already in the pinned data; it needs no cod ## HBA1 and HBA2 from Depth -HBA1 and HBA2 are near-identical, so a short read from one may be placed on the other. But an aligner that reports every read puts each read from either on one of the two, with mapping quality 0 where it fits both equally. So the depth over both gene bodies together, against single-copy sequence on each side of the cluster, counts the genes: the HBA row of `bin/cn_loci.tsv`, run by `bin/copy_number.py` ([the locus catalog](cn-loci.md)), computes +HBA1 and HBA2 are near-identical, so a short read from one may be placed on the other. But an aligner that reports every read puts each read from either on one of the two, with mapping quality 0 where it fits both equally. So the depth over both gene bodies together, against single-copy sequence on each side of the cluster, counts the genes: the HBA row of `bin/cn_loci.tsv`, run by `og cn call` ([the locus catalog](cn-loci.md)), computes estimate = 2 x (depth over HBA2 + depth over HBA1) / mean flank depth @@ -237,7 +237,7 @@ The ratio of a site is its estimate over the median of the sites, so one copy co - `tests/test_paralog_loci.py` checks the model list (167 genes in `assets/parascopy_models_v1.7.txt`, each with one main region) and every catalog row (a model on the list, a GRCh38 region inside the model's main region, a known tag and cross-check, a table name used once, the SMN1 row), and that planted faults are refused: a model not on the list (`GBA1.gz`, and `CYP21A2.gz`, which the data has no model for), a bad region, a region outside its model, a repeated table name, a catalog without the SMN1 row. It checks one owner per locus: no catalog model overlaps a depth-engine row of `bin/cn_loci.tsv` but RHCE (without that exemption RHCE is refused, and a planted model over HBA is refused), and every catalog gene `assets/callability_genes.tsv` lists has the owner `parascopy` there (a planted `.` is refused). The catalog sets owners in that file but adds no rows to it: its rows come from GIAB's lists, gene body by gene body, and a model can cover only part of a gene. So 17 catalog genes are not there: seven pseudogenes (CELP, GBAP1, NCF1B, NCF1C, OTOAP1, PMS2CL, STRCP1), which it leaves out as not protein-coding, and ten genes to which none of its reasons applies (ABCC6, CFH, CFHR4, CHRNA7, DEFA3, KRT86, NEB, PIK3CA, PLG, VWF). This section reads them all. The test holds that list and fails when it changes. It also runs the step's own lines that write a not-assessed SMN table and checks that the report, the Paraphase side-by-side and the couple screen each read it as not assessed, with the reason, and that a catalog table without `agCN_filter` or `agCN_qual` reads as unreadable, not as not resolved. - `tests/fake-docker/paralog-catalog-loop.sh` runs the step on a catalog of three rows against a fake Docker: three `parascopy cn-using` calls with the right model files, in order, one table each; another population's files with `PARASCOPY_POPULATION`; no call and a not-assessed table for a region without reads, a model that is not installed and a Parascopy run that fails, while the other rows run; a rerun with fewer rows removes the tables of the dropped rows and keeps the HBA and Paraphase tables; a bad row, or a region off chr1-22, X and Y or with its start past its end, stops the step before any container. - The e2e case `paralog-catalog-1-gba` adds a GBA row (`GBA.gz`) to a copy of the catalog and runs the step on the fixture's slice plus GIAB's HG002 reads over GBA1 and GBAP1, sampled to about 30x: the GBA table must hold an integer agCN with its quality. On the fixture's slice alone, which has no read at GBA1, the same row must be not assessed. The case also checks that the committed models are the names in the installed `list.txt` of each population, that the SMN table is byte for byte what one `parascopy cn-using` of `SMN1.gz` writes, as the step did before the catalog, and that the loop of the Nextflow module, cut from the file, writes the same two tables in the same image. -- `tests/test_copy_number.py` checks the HBA row and that these depth tables give, through the engine, the same tables as the code before it. `tests/test_hba_copy_number.py` gives `bin/hba_copy_number.py` made-up depth tables: four genes at even depth, three with HBA2 at half depth, two, a triplication, an estimate between two whole numbers, low flanks, flanks that disagree, and no read of mapping quality 0 over the genes (the depths of the novoalign BAM, and a clean three), which must be not assessed. +- `tests/test_copy_number.py` checks the HBA row and that these depth tables give, through the engine, the same tables as the code before it. `tests/test_hba_copy_number.py` gives `og cn call --locus HBA` made-up depth tables: four genes at even depth, three with HBA2 at half depth, two, a triplication, an estimate between two whole numbers, low flanks, flanks that disagree, and no read of mapping quality 0 over the genes (the depths of the novoalign BAM, and a clean three), which must be not assessed. - The e2e case `paralog-checks-and-doc-fixes-1-hba` writes synthetic 30x reads over chr16:95,001-235,000 for two samples, one with every second read over HBA2 left out (a planted one-gene loss), runs step 35 on each and checks four and three genes. A synthetic read whose bases occur more than once in that stretch gets mapping quality 0, as an aligner gives it. The HBA block of `modules/local/parascopy/main.nf`, cut from the file, then runs on the same BAMs in the same image and must write the same table. Last, the block runs on HG002's real Illumina reads at the cluster, three ways: GIAB's 60x novoalign BAM sampled to about 30x must be not assessed, naming the missing mapping quality 0 reads; Google's 35x NovaSeq BWA-MEM BAM must count four; and the same reads realigned with `minimap2 -a -x sr` to chr16:1-300,000 must count four. The truth is four, from Paraphase on HG002's HiFi reads. The run's notes record each estimate, the four depths and the share of mapping quality 0. - The e2e case `paralog-rows-1` runs the whole catalog on the fixture's slice plus GIAB's HG002 reads over every catalog locus on the fixture's chromosomes, sampled to about 30x, timed (it fails past 30 minutes): every row's table must hold an integer agCN or say not assessed with the reason, and a header-only table fails that check. It then runs the step with a catalog that has no SMN1 row and checks that both reports still show the HBA count and the other loci. `tests/test_collect_summary.py` holds the same case on made-up tables, with the loci read in the catalog's order. - `tests/test_copy_number.py` section 10 runs the CYP21A2 row on made-up bedcov and mpileup tables (`tests/fixtures/cn/CYP21A2_*`): unmodified, 2 copies and no finding; a planted chimeric deletion, 1 copy with the module total 3; a planted In2G conversion on one copy, a ratio of 0.5 at In2G and a possible conversion there. The same depths with every site even give no finding. The deletion's tables with no MAPQ 0 read (a share under 0.02) read not assessed (`low_mapq0_share`), and with a share of 0.021 or HG002's 0.1 still give the deletion. The CYP21A2 block of step 35, cut out, writes those tables, a not-assessed one when mpileup fails and a `low_mapq0_share` one when its MAPQ 1 pass equals the first, and the couple screen and the report read them. diff --git a/docs/46-lpa.md b/docs/46-lpa.md index b1ead3ceb..ab6bb4010 100644 --- a/docs/46-lpa.md +++ b/docs/46-lpa.md @@ -22,7 +22,7 @@ A VCF cannot show the repeat count or the 4925G>A variant: the six KIV-2 units i ## Tool - samtools `bedcov -j` for the read depth over the exons, `mpileup` for the reads at the 4925G>A sites -- `bin/copy_number.py` (standard library Python), the copy-number engine, on the LPA_KIV2 row of `bin/cn_loci.tsv` for the count, its interval and the reading; `bin/lpa_kiv2.py` lays out the table. See [Copy number from depth](cn-loci.md) +- `og cn` (OwnGenome's Rust binary, [docs/rust.md](rust.md)), the copy-number engine, on the LPA_KIV2 row of `bin/cn_loci.tsv` for the count, its interval, the reading and the table, and in the same call the 4925G>A reading. See [Copy number from depth](cn-loci.md) ## Docker Image @@ -48,7 +48,7 @@ With `run-all.sh`: `TOOLS=...,lpa`. It is opt-in and needs no data, so a run lis The method is the one published short-read estimators use: depth over the KIV-2 repeat against depth over single-copy sequence (Zekavat et al., Nat Commun 2018, PMID 29973585; Behera et al., BMC Med Genomics 2024, PMID 39449055). Behera et al. give the KIV-2 region in GRCh38 as chr6:160,611,568-160,646,868, six units, and its reads reach into the last exon of KIV-1 and the first exon of KIV-3, which resemble KIV-2. -The regions are exons of the MANE Select transcript ENST00000316300.10 (Ensembl, checked 2026-10-25), numbered from its first exon. They are the LPA_KIV2 row of `bin/cn_loci.tsv`, and `bin/copy_number.py bed --locus LPA_KIV2` prints them: +The regions are exons of the MANE Select transcript ENST00000316300.10 (Ensembl, checked 2026-10-25), numbered from its first exon. They are the LPA_KIV2 row of `bin/cn_loci.tsv`, and `og cn bed --locus LPA_KIV2` prints them: | Group | Exons | What they are | |---|---|---| @@ -84,7 +84,7 @@ The count is not made when: ## How the 4925G>A variant is read -The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds that base at the end of each of its six KIV-2 units (exons 5, 7, ..., 15; `NM_005577.4:c.733G>A` at the first, the one dbSNP names as rs1484919544) and at the near-identical last exon of KIV-1 (exon 3). The 21 bases around it are the same at all seven, so a read from any copy can land on any of them. `samtools mpileup` without a reference reads every read at the seven positions, every mapping quality, duplicates left out, bases of quality 13 or more, and `bin/traits_report.py --marker LPA_KIV2_4925` adds them up into its own table (`bin/lpa_kiv2.py` lays it out). The share of reads with T is a fraction of the copies, not a genotype. With a count, the copies that carry it are that fraction times N + 2 (the KIV-2 repeats plus one KIV-1 per chromosome). +The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds that base at the end of each of its six KIV-2 units (exons 5, 7, ..., 15; `NM_005577.4:c.733G>A` at the first, the one dbSNP names as rs1484919544) and at the near-identical last exon of KIV-1 (exon 3). The 21 bases around it are the same at all seven, so a read from any copy can land on any of them. `samtools mpileup` without a reference reads every read at the seven positions, every mapping quality, duplicates left out, bases of quality 13 or more, and `og cn call --locus LPA_KIV2 ... --marker LPA_KIV2_4925` adds them up into its own table, in the same call that makes the count. The share of reads with T is a fraction of the copies, not a genotype. With a count, the copies that carry it are that fraction times N + 2 (the KIV-2 repeats plus one KIV-1 per chromosome). | `status` | When | |---|---| @@ -92,7 +92,7 @@ The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand | `not seen` | 2 reads with T or fewer (one or two can be sequencing errors) | | `uncertain` | between the two | | `too little depth` | under 100 reads over the seven positions together | -| `not assessed` | the pileup could not be read (also when `samtools mpileup` failed; the count still runs), or the count found that the aligner left out the reads of MAPQ 0, which are the reads a carrying repeat gives | +| `not assessed` | the pileup could not be read (also when `samtools mpileup` failed; the count still runs), or the count found that the aligner left out the reads of MAPQ 0, which are the reads a carrying repeat gives. The marker reads the count's own MAPQ 0 share, not the rounded value in the count table, so the two never disagree at the 20% line | What a short-read fraction cannot say: which chromosome or which repeat carries the variant, so whether it sits on the small or the large isoform. A variant in one repeat among many gives few reads, so `not seen` is not proof of absence at low depth. diff --git a/docs/cn-loci.md b/docs/cn-loci.md index cada3a10b..279ab7c6f 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -1,6 +1,6 @@ # Copy Number from Depth (the locus catalog) -Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one engine, `bin/copy_number.py`, and one table, `bin/cn_loci.tsv`, with one row per locus. A new locus is a new row, not a new script. +Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one table, `bin/cn_loci.tsv`, with one row per locus, and one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)). Step 41 is the one exception for now: `og cn bed` gives it the RHD regions, and its report makes the RhD call in Python, with the catalog reader and `assess()` that `bin/copy_number.py` keeps. A new locus is a new row, not a new script. The table sits in `bin/` because `bin/` is the one repository folder that every container mounts: the bash steps mount it as `/pgp-bin`, and Nextflow puts it on every task's PATH. @@ -28,7 +28,7 @@ Then the row's `call` column decides: - `psv:D`: the module total of a gene and its pseudogene as `round:D` does, then the reads at the row's PSV sites ([the CYP21A2 row](#the-cyp21a2-row)); - `class:LOW,HIGH`: low, typical or high from the 95% interval (LPA KIV-2, 30 and 45). With `interval` `cv`, that interval comes from the spread of the per-segment depths: relative standard error sqrt(cv_target^2 / n_target + cv_control^2 / n_control), applied on the log scale. It covers sampling noise only. -`copy_number.py call --json` writes the result as one JSON block per locus: status, reason, estimate, copies, interval, class, the depths and the reading. +`og cn call --table OUT` writes the result as the step's table (status, the estimate, the copies or class, the depths and the reading), and prints one summary line. ## The columns @@ -81,7 +81,7 @@ CYP21A2 and its pseudogene CYP21A1P lie in the duplicated RCCX module, about 98% The call has two parts: 1. **The module total**: `scale` 2 times the two targets' depths added up over the flank depth, a whole number within 0.4, else not assessed (`between_counts`). Two modules per chromosome give 4. The tolerance is wider than HBA's 0.3 because the depth here runs high: HG002's 35x reads give 4.22, and a deletion planted in the same reads 3.32. -2. **The PSV sites**: `copy_number.py bed --psv --locus CYP21A2` prints each site on CYP21A2 and on CYP21A1P, and `samtools mpileup -A -B -q 0 -Q 13 -d 100000` without a reference counts the bases there (every read whatever its mapping quality, bases of quality 13 or more; read starts, ends and indels are skipped). A read with CYP21A2's base counts for CYP21A2 at either position, since a converted copy's reads may land on the pseudogene. A site with 30 such reads or more gets `estimate = module total x CYP21A2 reads / (CYP21A2 + CYP21A1P reads)` and its ratio to the median of the sites. +2. **The PSV sites**: `og cn bed --psv --locus CYP21A2` prints each site on CYP21A2 and on CYP21A1P, and `samtools mpileup -A -B -q 0 -Q 13 -d 100000` without a reference counts the bases there (every read whatever its mapping quality, bases of quality 13 or more; read starts, ends and indels are skipped). A read with CYP21A2's base counts for CYP21A2 at either position, since a converted copy's reads may land on the pseudogene. A site with 30 such reads or more gets `estimate = module total x CYP21A2 reads / (CYP21A2 + CYP21A1P reads)` and its ratio to the median of the sites. The sites, each CYP21A2's base in the reference with CYP21A1P's base at the matching position (dbSNP positions, the CYP21A1P bases read from the reference, checked on 2026-11-04): In2G c.293-13A/C>G (rs6467, chr6:32,039,081; CYP21A2 carries A or C there, CYP21A1P G), I172N c.518T>A (rs6475), V237E c.713T>A (rs12530380, the exon 6 cluster), V281L c.844G>T (rs6471), Q318X c.955C>T (rs7755898) and R356W c.1069C>T (rs7769409). @@ -89,7 +89,7 @@ A site is low when its estimate is more than 0.6 copies under the larger of the ### The interval rows -`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `copy_number.py bed --kind interval`, one `samtools bedcov` over every segment of every row, then `copy_number.py intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. +`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. - **The ClinGen recurrent regions**: every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0, pinned by `CLINGEN_RECURRENT_CNV_URL` and its sha256 in `versions.env`). The file's breakpoint and LCR rows are the segmental duplications that bound the regions. They are cut out of the targets, as are GIAB v3.6 segmental duplications, assembly gaps, satellites and the regions of every pinned Parascopy model (CHRNA7 in 15q13.3 stays with Parascopy). Targets are cut to pieces of at most 500 kb, and their depth is pooled. Controls are two 100 kb windows, one on each side of the cluster of ClinGen rows the region belongs to, clear of all of that and of the targets of the hand rows below (so no row's control sits on another row's target); a side with no clear window within 3 Mb (a chromosome end, as next to 22q11.2 and 15q11.2) is left out. The BED spells its chrX rows `chrx`; the script normalises them, and the engine refuses any contig name that is not a GRCh38 chromosome. The classification of a CNV found here follows the ACMG and ClinGen technical standards (Riggs 2020, PMID 31690835); a call from this step is a lead for a clinical test, not a classification. - **Two more ClinGen curations**: the 1p36 terminal region (ISCA-37434, haploinsufficiency score 3) and the Xq28 region that holds MECP2 (ISCA-46304, triplosensitivity score 3: the MECP2 duplication), from ClinGen's region curation list, read on 2026-11-04. @@ -119,7 +119,7 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. 1. Check that no pinned caller owns the gene: Parascopy (`assets/parascopy_models_v1.7.txt`), pypgx, Paraphase. 2. Choose single-copy controls near the gene, outside any common deletion or duplication there and outside segmental duplications. 3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to `scripts/ci/build-cn-intervals.py` and rerun the script. -4. Run it from a step: `copy_number.py bed --locus NAME`, `samtools bedcov` with the row's passes, then `copy_number.py call --locus NAME --bedcov ... --json ...`. +4. Run it from a step: `og cn bed --locus NAME`, `samtools bedcov` with the row's passes, then `og cn call --locus NAME --bedcov ... --table ...`. 5. Add made-up bedcov tables for the row to `tests/test_copy_number.py`, at every count it can call and at each guard. ## How it is tested @@ -128,7 +128,8 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. - The e2e case `cn-intervals-1-catalog` rebuilds the interval rows from the pinned downloads and fails on any difference with the committed catalog. `cn-intervals-2-hg002` runs the interval blocks of step 35 and of the PARASCOPY module on HG002's real reads at the AZF genes, their controls and 22q11.2: a male with one copy of AZFa and AZFb, AZFc_DAZ not assessed (its estimate, between 3.3 and 3.9, is no whole count of the four DAZ copies, so the row says `between_counts` rather than a call), no change at 22q11.2 A-B, no call on a row off chrY and chr22 (they have no read; the chr22 rows share A-B's control window, so they are left out of that check), and the AZF rows not applicable when the same reads are read as female. - The e2e case `followups-cn-1-couple-report` runs step 39, COUPLE_CARRIER and HTML_REPORT on a made-up couple, plus a stub run of `main.nf`, and checks that the interval table and the CNVpytor calls reach the couple report and `summary.json`. Each wiring line has a red control: the case runs again without it and checks that the rows are gone. -- `tests/test_copy_number.py` checks every row and refuses broken ones. For the three gene-dosage rows it calls made-up depths at 0 to 4 copies, cuts the dosage block out of step 35 and runs it on the made-up bedcov tables in `tests/fixtures/cn/` (HP 3, LCE3BC 1, CCL3L1 5 copies), checks that a row shifted by 1 kb no longer matches its table, and that the PARASCOPY module runs the same list. It runs the owner check with planted AMY1C, C4B and DEFB4 rows. It feeds made-up bedcov tables at 0, 1, 2, 3 and 4 copies and a MAPQ 0 share over the ceiling. It checks that a control with no read stops the run with `BROKEN_CONTROL`. It also checks that the depth tables of `tests/test_hba_copy_number.py` and `tests/test_lpa_kiv2.py` give, through the engine, the same tables the code before the engine wrote. -- The fake-docker case `cn-engine-calls` runs steps 35, 41 and 46 with made-up depths. Each step calls `copy_number.py` once for its regions, step 35 and 46 once more for the call, and none calls the old per-gene scripts. A control exon with no read fails step 46 with `BROKEN_CONTROL`. +- `tests/test_copy_number.py` checks every row and refuses broken ones. For the three gene-dosage rows it calls made-up depths at 0 to 4 copies, cuts the dosage block out of step 35 and runs it on the made-up bedcov tables in `tests/fixtures/cn/` (HP 3, LCE3BC 1, CCL3L1 5 copies), checks that a row shifted by 1 kb no longer matches its table, and that the PARASCOPY module runs the same list. It runs the owner check with planted AMY1C, C4B and DEFB4 rows. It feeds made-up bedcov tables at 0, 1, 2, 3 and 4 copies and a MAPQ 0 share over the ceiling. It checks that a control with no read stops the run with `BROKEN_CONTROL`. With `OG` set (Guard's rust job sets it) it also checks that the depth tables of `tests/test_hba_copy_number.py` and `tests/test_lpa_kiv2.py` give, through `og cn`, the same tables the Python code before it wrote, and that `og cn` refuses the broken rows with the same message as the catalog reader. +- The fake-docker case `cn-engine-calls` runs steps 35, 41 and 46 with made-up depths. Each step calls `og cn` once for its regions, step 35 and 46 once more for the call, step 46 reads the KIV-2 4925G>A marker in the same call, and no step calls a Python copy-number script. A control exon with no read fails step 46 with `BROKEN_CONTROL`. The case needs `og`, so Guard's rust job runs it after the build. - `tests/test_copy_number.py` section 10 checks the CYP21A2 row: its six sites, the pileup parser, and the fixtures `tests/fixtures/cn/CYP21A2_{normal,deletion,conversion}` (2 copies and no finding; 1 copy with the module total 3; an In2G ratio of 0.5 and a possible conversion), with the conversion's depths read with every site even as the red control (no finding). The e2e case `cyp21a2-short-reads-1-hg002` runs the step's and the module's blocks on HG002's reads, unmodified and with a planted conversion and a planted deletion. The same section feeds the deletion's tables with MAPQ 0 shares of 0, 0.019, 0.021 and 0.1 (not assessed twice, then the deletion), and `cn-guard-1-mapq0` runs both blocks on HG002's reads with every MAPQ 0 read removed: not assessed (`low_mapq0_share`), never a deletion. +- `tests/parity` runs `og cn` against the Python engine it replaced on every command and the tables of the three per-gene loci (`scripts/ci/oracle-diff.sh`, [docs/rust.md](rust.md)); the deliberate fixes are its goldens. - The e2e case `cn-engine-1-parity` runs steps 41 and 46 and the HBA block of step 35 on HG002's real reads at the three loci, at full depth and at an eighth of it. The code before the engine is fetched by its commit, 6056452, so the comparison still means something after this change is on main. The LPA and HBA tables and the RhD rows of step 41's table and evidence file must be byte for byte what it wrote. diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index 066860c5b..71c5e7d8a 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -627,6 +627,15 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted - **Why:** HBA had the guard (a second `samtools bedcov -Q 1` pass and `mapq0_min`); the CYP21A2 row was added later without it. On HG002's 35x reads, 9.9% of the depth over the two targets comes from MAPQ 0 reads, with BWA-MEM and with minimap2 `-ax sr` alike, so a share near 0 means the aligner dropped them. - **Fix:** the row runs both bedcov passes with `mapq0_min` 0.02. Under that share it is not assessed (`low_mapq0_share`), and the table's `message` row, which both reports print, says the BAM has few or no multi-mapping reads there. `tests/test_copy_number.py` and the e2e case `cn-guard-1-mapq0` check it. +### The KIV-2 count and its 4925G>A reading disagreed at a MAPQ 0 share of 0.1996 +- **Failed:** step 46's count judged the MAPQ 0 share on the raw value, and the 4925G>A reader judged it on the value the count table printed, rounded to three places. At a share of 0.1996 the count said not assessed (under 0.20) while the marker read 0.200 and said present, a reading from the very reads the count had refused. +- **Why:** the marker ran as a second program that could only see the count's table, not the count's numbers. +- **Fix:** `og cn call --marker LPA_KIV2_4925` reads the marker in the same call as the count and passes it the raw share, so both judge the same number. The parity case `marker-lpa-share1996-kiv-present` holds the fixed output. + +### Two rows of the SMN table over exon 7 gave two different SMN1 counts +- **Failed:** the SMN1 marker took the last row of step 35's SMN table that held exon 7, and the Paraphase comparison took the first. A table with two such rows made the marker message and the comparison read different psCN values. +- **Fix:** one rule in `og cn` (`rust/crates/og-cn/src/smn.rs`): the first row whose region or homologous regions hold exon 7 gives the psCN. The parity case `smn-smn-present-smn-two-rows` holds the fixed output. + ## Downloads (2026-11) ### One bad transfer of a large file failed the whole setup diff --git a/docs/rust.md b/docs/rust.md index 09409b85e..a218640e9 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -23,6 +23,7 @@ rust/ crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table + crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read crates/og-ancestry/ step 26b, fine-grained ancestry: og ancestry-fine crates/og-reports/ the opt-in step reports of steps 45, 47, 48 and 50 to 53 og/ the binary: og [args] @@ -30,7 +31,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report` and `segments` (were `family_report.py` and `family_segments.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader step 41's report imports) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report` and `segments` (were `family_report.py` and `family_segments.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py`, `ibd_report.py`, `pedigree_svg.py`, `annotsv_postfilter.py` and `cnvpytor_to_vcf.py` import. diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index 05b3b46d1..640057f79 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -3,7 +3,7 @@ BLOODTYPE — Blood group antigens (red cell, platelet, neutrophil) predicted from the genome ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('bloodtype' in --tools). Three processes, as in scripts/41-bloodtype.sh: - BLOODTYPE_REGIONS bin/copy_number.py bed --locus RHD, once: RHD, its + BLOODTYPE_REGIONS og cn bed --locus RHD, once: RHD, its exons and flanks (the RHD row of bin/cn_loci.tsv) BLOODTYPE_DEPTH samtools bedcov -Q 20 -j over the sites of assets/bloodtype_markers.tsv and those regions, for each @@ -29,7 +29,7 @@ process BLOODTYPE_REGIONS { script: """ - copy_number.py bed --locus RHD > rhd_regions.bed + og cn bed --locus RHD > rhd_regions.bed cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/lpa/main.nf b/modules/local/lpa/main.nf index 97ff4ea1f..c4e061cc8 100644 --- a/modules/local/lpa/main.nf +++ b/modules/local/lpa/main.nf @@ -3,18 +3,17 @@ LPA — the LPA KIV-2 repeat count estimated from read depth (Lp(a)) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('lpa' in --tools). Three processes, as in scripts/46-lpa.sh: - LPA_REGIONS bin/copy_number.py bed --locus LPA_KIV2, once: the KIV-2 - and single-copy LPA exons (the LPA_KIV2 row of bin/cn_loci.tsv), + LPA_REGIONS og cn bed --locus LPA_KIV2, once: the KIV-2 and + single-copy LPA exons (the LPA_KIV2 row of bin/cn_loci.tsv), and the seven GRCh38 copies of the KIV-2 4925G>A site - (bin/traits_report.py --marker-regions LPA_KIV2_4925) + (og cn marker-regions LPA_KIV2_4925) LPA_DEPTH samtools bedcov -j over the exons, all MAPQ and MAPQ >= 1, and samtools mpileup at the 4925G>A sites, for each sample with a BAM; empty tables when the BAM has no chr6 - LPA bin/copy_number.py call --locus LPA_KIV2: the count, its - 95% interval and the message (laid out by bin/lpa_kiv2.py), - then bin/traits_report.py --marker LPA_KIV2_4925: the 4925G>A - reading beside it, _lpa_kiv2_4925.tsv ('not assessed' - when that check fails; the count stays) + LPA og cn call --locus LPA_KIV2 --marker LPA_KIV2_4925: the + count, its 95% interval and the message, and in the same + run the 4925G>A reading beside it, _lpa_kiv2_4925.tsv + ('not assessed' when that check fails; the count stays) Equivalent to: scripts/46-lpa.sh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -32,8 +31,8 @@ process LPA_REGIONS { script: """ - copy_number.py bed --locus LPA_KIV2 > lpa_regions.bed - traits_report.py --marker-regions LPA_KIV2_4925 > lpa_sites.txt + og cn bed --locus LPA_KIV2 > lpa_regions.bed + og cn marker-regions LPA_KIV2_4925 > lpa_sites.txt cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -123,11 +122,14 @@ process LPA { script: """ - copy_number.py call --locus LPA_KIV2 --bedcov ${depth} --bedcov-mapq1 ${depth_q1} --table ${meta.id}_lpa_kiv2.tsv - # KIV-2 4925G>A beside the count (as scripts/46-lpa.sh [4/4]): a failure - # never costs the count; the table then says not assessed. - if ! traits_report.py --marker LPA_KIV2_4925 --pileup ${pileup} --copy-table ${meta.id}_lpa_kiv2.tsv \\ - --marker-out ${meta.id}_lpa_kiv2_4925.tsv; then + # The count and KIV-2 4925G>A beside it in one run (as scripts/46-lpa.sh + # [3/3]): a failure of the marker never costs the count; the table then + # says not assessed. No count table: the task fails. + rc=0 + og cn call --locus LPA_KIV2 --bedcov ${depth} --bedcov-mapq1 ${depth_q1} --table ${meta.id}_lpa_kiv2.tsv \\ + --marker LPA_KIV2_4925 --marker-pileup ${pileup} --marker-out ${meta.id}_lpa_kiv2_4925.tsv || rc=\$? + if [ "\$rc" -ne 0 ]; then + [ -s ${meta.id}_lpa_kiv2.tsv ] || exit "\$rc" echo "WARNING: the KIV-2 4925G>A check failed; ${meta.id}_lpa_kiv2_4925.tsv says not assessed." >&2 printf 'metric\\tvalue\\nmarker\\tLPA KIV-2 4925G>A\\nrsid\\trs1484919544 (first KIV-2 copy)\\nstatus\\tnot assessed\\nalt_reads\\t.\\ndepth\\t.\\nfraction\\t.\\ncopies\\t.\\nkiv2_copies\\t.\\nmessage\\t%s\\n' \\ 'LPA KIV-2 4925G>A: not assessed, the check failed; see the log of step 46.' > ${meta.id}_lpa_kiv2_4925.tsv diff --git a/modules/local/parascopy/main.nf b/modules/local/parascopy/main.nf index 16da880fc..862bbf569 100644 --- a/modules/local/parascopy/main.nf +++ b/modules/local/parascopy/main.nf @@ -26,7 +26,7 @@ HBA1 and HBA2 have no Parascopy model: samtools bedcov (in the same image) sums the depth over both gene bodies and two single-copy flanks, - over all reads and over reads of MAPQ 1 or more, and bin/copy_number.py + over all reads and over reads of MAPQ 1 or more, and og cn call (the HBA row of bin/cn_loci.tsv) turns it into a count of alpha-globin genes (not assessed when no read over the genes has MAPQ 0: the aligner then left out the reads that fit both genes). @@ -46,7 +46,7 @@ Beside the SMN1 psCN, SMN1 g.27134T>G (rs143838139, Luo et al. 2014), a marker of SMN1 duplication alleles: samtools mpileup at its SMN1 and SMN2 - copies, read as a fraction of copies by bin/traits_report.py --marker + copies, read as a fraction of copies by og cn marker-read SMN1_27134 with the SMN table (_smn1_marker.tsv). It never proves a silent 2+0 carrier, and its message says so. @@ -109,10 +109,10 @@ process PARASCOPY { # the locus profile (aggregate and paralog-specific copy number, each with # filter and quality). # A BAM with ALT or HLA contigs is refused, as scripts/35-paralogs.sh does - # (bin/copy_number.py check-bam): its depth splits between a gene and its + # (og cn check-bam): its depth splits between a gene and its # ALT copy, and a depth-based count reads that as a deletion. samtools view -H ${bam} > bam_header.txt - copy_number.py check-bam --header bam_header.txt + og cn check-bam --header bam_header.txt models=${parascopy_data}/models_GRCh38_1KGP/${params.parascopy_population} [ -d "\$models" ] || { echo "ERROR: no Parascopy models \$models (populations: AFR AMR EAS EUR SAS)" >&2; exit 1; } parascopy depth -i ${bam}::${meta.id} -f ${reference} ${background} -o depth -@ ${task.cpus} @@ -158,11 +158,11 @@ process PARASCOPY { # SMN1 g.27134T>G beside the SMN1 psCN (as scripts/35-paralogs.sh): mpileup at its SMN1 and SMN2 # copies, then the reading with the SMN table; not assessed when it fails - if traits_report.py --marker-regions SMN1_27134 > smn1_marker_sites.txt \ + if og cn marker-regions SMN1_27134 > smn1_marker_sites.txt \ && ( while read -r r; do samtools mpileup -A -B -q 0 -Q 13 -d 100000 -r "\$r" ${bam} || exit 1; done \ < smn1_marker_sites.txt ) > smn1_marker_pileup.tsv \ - && traits_report.py --marker SMN1_27134 --pileup smn1_marker_pileup.tsv \ - --copy-table ${meta.id}_smn_copy_number.tsv --marker-out ${meta.id}_smn1_marker.tsv.tmp; then + && og cn marker-read SMN1_27134 --pileup smn1_marker_pileup.tsv \ + --copy-table ${meta.id}_smn_copy_number.tsv --out ${meta.id}_smn1_marker.tsv.tmp; then mv ${meta.id}_smn1_marker.tsv.tmp ${meta.id}_smn1_marker.tsv else rm -f ${meta.id}_smn1_marker.tsv.tmp @@ -173,10 +173,10 @@ process PARASCOPY { rm -f smn1_marker_sites.txt smn1_marker_pileup.tsv # HBA1/HBA2 from depth (as scripts/35-paralogs.sh): all MAPQ and MAPQ 1 or more, then the verdict - if copy_number.py bed --locus HBA > hba_regions.bed \\ + if og cn bed --locus HBA > hba_regions.bed \\ && samtools bedcov hba_regions.bed ${bam} > hba_bedcov.tsv \\ && samtools bedcov -Q 1 hba_regions.bed ${bam} > hba_bedcov_q1.tsv \\ - && copy_number.py call --locus HBA --bedcov hba_bedcov.tsv --bedcov-mapq1 hba_bedcov_q1.tsv \\ + && og cn call --locus HBA --bedcov hba_bedcov.tsv --bedcov-mapq1 hba_bedcov_q1.tsv \\ --table ${meta.id}_hba_copy_number.tsv.tmp; then mv ${meta.id}_hba_copy_number.tsv.tmp ${meta.id}_hba_copy_number.tsv else @@ -192,10 +192,10 @@ process PARASCOPY { dosage_loci="HP LCE3BC CCL3L1" printf 'locus\\tstatus\\tcopies\\testimate\\treason\\treading\\n' > cn_dosage.part for loc in \$dosage_loci; do - if copy_number.py bed --locus "\$loc" > dosage_regions.bed \\ + if og cn bed --locus "\$loc" > dosage_regions.bed \\ && samtools bedcov dosage_regions.bed ${bam} > dosage_bedcov.tsv \\ && samtools bedcov -Q 1 dosage_regions.bed ${bam} > dosage_bedcov_q1.tsv \\ - && copy_number.py call --locus "\$loc" --bedcov dosage_bedcov.tsv --bedcov-mapq1 dosage_bedcov_q1.tsv \\ + && og cn call --locus "\$loc" --bedcov dosage_bedcov.tsv --bedcov-mapq1 dosage_bedcov_q1.tsv \\ --table dosage_table.tsv; then awk -F'\\t' -v OFS='\\t' -v l="\$loc" 'NR > 1 {v[\$1] = \$2} END {print l, v["status"], v["copies"], v["estimate"], v["reason"], v["reading"]}' dosage_table.tsv >> cn_dosage.part @@ -209,14 +209,14 @@ process PARASCOPY { # CYP21A2 from depth and PSVs (as scripts/35-paralogs.sh): bedcov over the row (the RCCX module # total), mpileup without a reference at its PSV sites, then the verdict; not assessed when it fails - if copy_number.py bed --locus CYP21A2 > cyp21a2_regions.bed \\ - && copy_number.py bed --psv --locus CYP21A2 > cyp21a2_psv.bed \\ + if og cn bed --locus CYP21A2 > cyp21a2_regions.bed \\ + && og cn bed --psv --locus CYP21A2 > cyp21a2_psv.bed \\ && samtools bedcov cyp21a2_regions.bed ${bam} > cyp21a2_bedcov.tsv \\ && samtools bedcov -Q 1 cyp21a2_regions.bed ${bam} > cyp21a2_bedcov_q1.tsv \\ && samtools mpileup -A -B -q 0 -Q 13 -d 100000 \\ -r "\$(awk 'NR == 1 {c = \$1; s = \$2 + 1} {e = \$3} END {print c ":" s "-" e}' cyp21a2_psv.bed)" \\ -l cyp21a2_psv.bed ${bam} > cyp21a2_pileup.tsv \\ - && copy_number.py call --locus CYP21A2 --bedcov cyp21a2_bedcov.tsv --bedcov-mapq1 cyp21a2_bedcov_q1.tsv \\ + && og cn call --locus CYP21A2 --bedcov cyp21a2_bedcov.tsv --bedcov-mapq1 cyp21a2_bedcov_q1.tsv \\ --pileup cyp21a2_pileup.tsv \\ --table ${meta.id}_cyp21a2_psv.tsv.tmp; then mv ${meta.id}_cyp21a2_psv.tsv.tmp ${meta.id}_cyp21a2_psv.tsv @@ -231,11 +231,11 @@ process PARASCOPY { # Interval rows of bin/cn_loci.tsv (as scripts/35-paralogs.sh): one bedcov over every row's segments, # then the verdict; a failed pass writes every row as not assessed (check_failed) - if ! { copy_number.py bed --kind interval > interval_regions.bed \ + if ! { og cn bed --kind interval > interval_regions.bed \ && samtools bedcov interval_regions.bed ${bam} > interval_bedcov.tsv \ - && copy_number.py intervals --bedcov interval_bedcov.tsv --sex ${sex} --out ${meta.id}_cn_intervals.tsv; }; then + && og cn intervals --bedcov interval_bedcov.tsv --sex ${sex} --out ${meta.id}_cn_intervals.tsv; }; then echo "WARNING: the interval depth check failed; ${meta.id}_cn_intervals.tsv lists every row as not assessed." >&2 - copy_number.py intervals --failed --out ${meta.id}_cn_intervals.tsv + og cn intervals --failed --out ${meta.id}_cn_intervals.tsv fi rm -f interval_regions.bed interval_bedcov.tsv diff --git a/modules/local/pypgx/main.nf b/modules/local/pypgx/main.nf index da2e96706..db5f67571 100644 --- a/modules/local/pypgx/main.nf +++ b/modules/local/pypgx/main.nf @@ -68,11 +68,11 @@ process PYPGX { def gq_floor = task.ext.gq_floor != null ? task.ext.gq_floor : 20 """ # A BAM with ALT or HLA contigs is refused, as scripts/32-pypgx.sh does - # (bin/copy_number.py check-bam): its depth splits between a gene and its + # (og cn check-bam): its depth splits between a gene and its # ALT copy, and a depth-based count reads that as a deletion. The contig # names come from pysam, which the image has. python3 -c "import pysam, sys; print(chr(10).join(pysam.AlignmentFile(sys.argv[1]).references))" ${bam} > bam_contigs.txt - copy_number.py check-bam --header bam_contigs.txt + og cn check-bam --header bam_contigs.txt OUTBASE="${meta.id}_pypgx_results" mkdir -p "\$OUTBASE" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index defb16dcc..869fd46c3 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -575,6 +575,7 @@ dependencies = [ "og-bam", "og-chip", "og-clinvar", + "og-cn", "og-dev", "og-filters", "og-intervals", @@ -630,6 +631,16 @@ dependencies = [ "og-io", ] +[[package]] +name = "og-cn" +version = "0.1.0" +dependencies = [ + "indexmap", + "og-filters", + "og-io", + "pycompat", +] + [[package]] name = "og-dev" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 39caabf4e..390e5d33b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "og"] [workspace.package] version = "0.1.0" @@ -27,6 +27,7 @@ og-setup = { path = "crates/og-setup" } og-steps = { path = "crates/og-steps" } og-ancestry = { path = "crates/og-ancestry" } og-chip = { path = "crates/og-chip" } +og-cn = { path = "crates/og-cn" } og-pedigree = { path = "crates/og-pedigree" } og-reports = { path = "crates/og-reports" } indexmap = "2" diff --git a/rust/README.md b/rust/README.md index a2833fdff..24fb6a2d2 100644 --- a/rust/README.md +++ b/rust/README.md @@ -11,6 +11,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | +| `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-reports` | the opt-in step reports: `og y str-bed`, `og y str-report`, `og viral report`, `og exomiser prepare`, `og exomiser report`, `og mei report`, `og mei annotsv-input`, `og mocha report`, `og vntyper report`, `og locityper report` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `crates/og-setup` | what `scripts/setup.sh` runs once: `og str catalog` (the STRchive catalog of steps 09 and 9b), `og setup unzip` (Exomiser's zips), `og setup genetic-map` (the genetic map of steps 26b and 51) | diff --git a/rust/crates/og-cn/Cargo.toml b/rust/crates/og-cn/Cargo.toml new file mode 100644 index 000000000..c82cdc159 --- /dev/null +++ b/rust/crates/og-cn/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "og-cn" +description = "Copy number from read depth: og cn bed, call, intervals, check-bam, marker-regions, marker-read" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +indexmap.workspace = true +og-filters.workspace = true +og-io.workspace = true +pycompat.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-cn/src/catalog.rs b/rust/crates/og-cn/src/catalog.rs new file mode 100644 index 000000000..a9564a447 --- /dev/null +++ b/rust/crates/og-cn/src/catalog.rs @@ -0,0 +1,743 @@ +//! `bin/cn_loci.tsv`: one row per locus, read and checked once per process. +//! +//! The checks and their messages are those of `copy_number.py` +//! (`parse_row`, `load_catalog`), plus one it lacked: a `more=` reading +//! needs a numbered reading to count past (the Python failed on `max()` of +//! an empty sequence when a call reached it, naming no row). + +use crate::{Error, py_float, py_int}; +use indexmap::IndexMap; +use og_filters::py_repr; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +/// The catalog's columns, in order. +pub const COLUMNS: [&str; 22] = [ + "locus", + "kind", + "chrom", + "targets", + "controls", + "details", + "target_depth", + "control_depth", + "scale", + "offset", + "bedcov", + "mapq0_min", + "mapq0_max", + "min_control_depth", + "max_control_ratio", + "call", + "interval", + "readings", + "input", + "owner", + "cross_check", + "pmids", +]; + +/// The findings a `call psv` row must have a reading for. +pub const PSV_FINDINGS: [&str; 4] = ["none", "deletion", "conversion", "pseudogene_deletion"]; + +/// What a row counts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Gene, + Interval, + Organelle, +} + +/// Where a row's depths come from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Input { + Bedcov, + CnvpytorLevel, +} + +/// How the target segments combine into T. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetMode { + /// Added up: each segment is one reference copy. + Sum, + /// Length-weighted mean. + Pooled, +} + +/// How the control segments combine into C. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ControlMode { + /// Plain mean of the segments. + Flanks, + /// Length-weighted mean. + Pooled, +} + +/// The `call` column. +#[derive(Debug, Clone, PartialEq)] +pub enum CallRule { + /// The nearest whole number, when the estimate lies within `max_off` of it. + Round { max_off: f64 }, + /// The number of cut points the estimate reaches. + Cut(Vec), + /// low, typical or high by the 95% interval against two bounds. + Class { low: f64, high: f64 }, + /// The module total as `Round`, then the PSV sites. + Psv { max_off: f64 }, +} + +/// One `NAME:start-end[:label]` segment, 0-based half-open. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Segment { + pub name: String, + pub start: i64, + pub end: i64, + pub label: String, +} + +/// One paralogous sequence variant site of a `call psv` row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PsvSite { + pub name: String, + /// 0-based position on the gene. + pub gene_pos: i64, + /// 0-based position on the pseudogene. + pub pseudo_pos: i64, + /// The gene's bases there (`b"AC"` for In2G). + pub gene: Vec, + /// The pseudogene's bases there. + pub pseudo: Vec, + pub text: String, +} + +/// The depth-ratio settings of a `bedcov` row. +#[derive(Debug, Clone, PartialEq)] +pub struct Depth { + pub target_mode: TargetMode, + pub control_mode: ControlMode, + pub scale: f64, + pub offset: f64, + /// The `bedcov` passes, as written (`.` for the plain pass). + pub passes: Vec, + pub mapq0_min: Option, + pub mapq0_max: Option, + pub min_control: f64, + pub max_ratio: Option, + /// A 95% interval from the per-segment spread (`interval cv`). + pub ci_cv: bool, +} + +/// One catalog row. +#[derive(Debug, Clone, PartialEq)] +pub struct Locus { + pub id: String, + pub kind: Kind, + pub chrom: String, + pub input: Input, + pub targets: Vec, + pub controls: Vec, + pub details: Vec, + pub call: CallRule, + /// `KEY=text` readings, in the row's order. + pub readings: IndexMap, + pub psv: Vec, + /// `target_depth` as written (a `call psv` row needs `sum` whatever its input). + pub target_depth: String, + /// Present for a `bedcov` row. + pub depth: Option, +} + +impl Locus { + /// The bedcov settings; every row `og cn` counts has them. + pub fn depth(&self) -> &Depth { + self.depth + .as_ref() + .expect("a bedcov row has its depth settings") + } + + /// `[(chrom, start, end, name)]` of the row's segments, sorted by + /// position (controls, targets, details; stable). + pub fn regions(&self) -> Vec<(&str, i64, i64, &str)> { + let mut out: Vec<_> = self + .controls + .iter() + .chain(&self.targets) + .chain(&self.details) + .map(|s| (self.chrom.as_str(), s.start, s.end, s.name.as_str())) + .collect(); + out.sort_by_key(|r| (r.1, r.2)); + out + } + + /// The row's reading for a count: its own line, else `more=` past the + /// highest numbered line, else empty. + pub fn reading_count(&self, n: i64) -> String { + if let Some(t) = self.readings.get(&n.to_string()) { + return t.clone(); + } + match (self.readings.get("more"), self.max_numbered()) { + (Some(more), Some(max)) if n > max => more.replace("{n}", &n.to_string()), + _ => String::new(), + } + } + + /// The row's reading for a word key (a class or a finding), or empty. + pub fn reading_word(&self, key: &str) -> String { + self.readings.get(key).cloned().unwrap_or_default() + } + + fn max_numbered(&self) -> Option { + self.readings + .keys() + .filter(|k| !k.is_empty() && k.bytes().all(|c| c.is_ascii_digit())) + .filter_map(|k| k.parse().ok()) + .max() + } + + /// An interval row `og cn intervals` calls: kind interval, bedcov, one + /// plain pass (LCE3BC, with a MAPQ >= 1 pass, stays a gene-dosage row). + pub fn is_interval_row(&self) -> bool { + self.kind == Kind::Interval && self.input == Input::Bedcov && self.depth().passes == ["."] + } +} + +/// The catalog, rows in file order. +#[derive(Debug, Clone)] +pub struct Catalog { + pub path: String, + pub loci: IndexMap, +} + +impl Catalog { + /// The row named `id`, when it is a bedcov row. + pub fn bedcov(&self, id: &str) -> Option<&Locus> { + self.loci.get(id).filter(|l| l.input == Input::Bedcov) + } + + /// Reads and checks the catalog at `path`. + pub fn load(path: &str) -> Result { + let lines = crate::read_lines_strict(path)?; + Catalog::parse(path, lines.iter().map(String::as_str)) + } + + /// Checks the catalog's lines (each without its `\n`); `path` names it + /// in messages. + pub fn parse<'a>(path: &str, lines: impl Iterator) -> Result { + let mut rows = lines.filter(|l| !crate::strip(l).is_empty() && !l.starts_with('#')); + let header_ok = rows + .next() + .is_some_and(|h| h.split('\t').eq(COLUMNS.iter().copied())); + let mut loci = IndexMap::new(); + let mut seen: HashSet = HashSet::new(); + let mut any = false; + for line in rows { + if !header_ok { + break; + } + any = true; + let cells: Vec<&str> = line.split('\t').collect(); + if cells.len() != COLUMNS.len() { + return Err(Error(format!( + "{path}: row {} has the wrong number of columns", + py_repr(cells[0]) + ))); + } + if loci.contains_key(cells[0]) { + return Err(Error(format!("{path}: locus {} is listed twice", cells[0]))); + } + let row = parse_row(&cells)?; + for s in row.targets.iter().chain(&row.controls).chain(&row.details) { + if !seen.insert(s.name.clone()) { + return Err(Error(format!( + "{path}: segment name {} is used by two loci", + s.name + ))); + } + } + loci.insert(row.id.clone(), row); + } + if !header_ok || !any { + return Err(Error(format!( + "{path}: the header must be {}", + COLUMNS.join(" ") + ))); + } + Ok(Catalog { + path: path.to_owned(), + loci, + }) + } +} + +/// The catalog beside the running binary: the first folder above it that +/// holds `cn_loci.tsv` or `bin/cn_loci.tsv` (`bin/.og/og-linux-*` in a +/// checkout, `/pgp-bin/.og/...` in a container, `rust/target/*/og`). +pub fn default_path() -> Option { + let exe = std::env::current_exe().ok()?; + exe.ancestors().skip(1).find_map(|d| { + [d.join("cn_loci.tsv"), d.join("bin").join("cn_loci.tsv")] + .into_iter() + .find(|p| Path::is_file(p)) + }) +} + +fn cell<'a>(cells: &[&'a str], col: &str) -> &'a str { + cells[COLUMNS.iter().position(|c| *c == col).expect("a column")] +} + +/// A number column: `.` gives `default`; anything not a finite number is +/// refused. +fn num(cells: &[&str], col: &str, default: Option) -> Result, Error> { + let v = cell(cells, col); + if v == "." { + return Ok(default); + } + match pycompat::num::parse_float(v) { + Ok(x) if x.is_finite() => Ok(Some(x)), + _ => Err(Error(format!( + "{}: {col} is {}, not a number or '.'", + cells[0], + py_repr(v) + ))), + } +} + +fn segments(cells: &[&str], col: &str) -> Result, Error> { + let v = cell(cells, col); + if v == "." { + return Ok(Vec::new()); + } + let loc = cells[0]; + let mut out = Vec::new(); + for part in v.split(';') { + let mut bits = part.splitn(3, ':'); + let name = bits.next().unwrap_or(""); + let span = bits.next(); + let label = bits.next().unwrap_or(""); + let bad = || { + Error(format!( + "{loc}: {col} segment {} is not NAME:start-end[:label]", + py_repr(part) + )) + }; + let span: Vec<&str> = span.ok_or_else(bad)?.split('-').collect(); + let [s, e] = span[..] else { return Err(bad()) }; + let (start, end) = match (py_int(s), py_int(e)) { + (Ok(s), Ok(e)) => (s, e), + _ => return Err(bad()), + }; + if name.is_empty() || end <= start || start < 0 { + return Err(Error(format!( + "{loc}: {col} segment {} has no name or an empty span", + py_repr(part) + ))); + } + out.push(Segment { + name: name.to_owned(), + start, + end, + label: label.to_owned(), + }); + } + Ok(out) +} + +fn readings(cells: &[&str]) -> Result, Error> { + let v = cell(cells, "readings"); + let mut out = IndexMap::new(); + if v == "." { + return Ok(out); + } + for part in v.split('|') { + match part.split_once('=') { + Some((k, t)) if !k.is_empty() && !t.is_empty() => { + out.insert(k.to_owned(), t.to_owned()); + } + _ => { + return Err(Error(format!( + "{}: reading {} is not KEY=text", + cells[0], + py_repr(part) + ))); + } + } + } + Ok(out) +} + +fn call(cells: &[&str]) -> Result { + let raw = cell(cells, "call"); + let bad = || { + Error(format!( + "{}: call {} is not round:D, cut:A,B..., class:LOW,HIGH or psv:D", + cells[0], + py_repr(raw) + )) + }; + let (mode, args) = raw.split_once(':').ok_or_else(bad)?; + let vals: Vec = args + .split(',') + .map(|x| py_float(x).ok().filter(|v| v.is_finite())) + .collect::>() + .unwrap_or_default(); + if vals.is_empty() || vals.windows(2).any(|w| w[0] > w[1]) { + return Err(bad()); + } + match (mode, &vals[..]) { + ("round", [d]) => Ok(CallRule::Round { max_off: *d }), + ("psv", [d]) => Ok(CallRule::Psv { max_off: *d }), + ("class", [lo, hi]) => Ok(CallRule::Class { + low: *lo, + high: *hi, + }), + ("cut", _) => Ok(CallRule::Cut(vals)), + _ => Err(bad()), + } +} + +fn psv_sites(loc: &str, details: &[Segment]) -> Result, Error> { + let mut out = Vec::new(); + for seg in details { + let label = &seg.label; + let bits: Vec<&str> = label.splitn(3, ' ').collect(); + let parsed = py_int(bits[0]) + .ok() + .and_then(|pos| bits.get(1).map(|b| (pos, b))); + let Some((pos, alleles)) = parsed else { + return Err(Error(format!( + "{loc}: PSV {} label {} is not 'PSEUDO_POS GENE>PSEUDO text'", + seg.name, + py_repr(label) + ))); + }; + let (gene, pseudo) = alleles.split_once('>').unwrap_or((alleles, "")); + let acgt = |s: &str| s.bytes().all(|c| b"ACGT".contains(&c)); + let shared = gene.bytes().any(|c| pseudo.as_bytes().contains(&c)); + if seg.end - seg.start != 1 + || pos < 0 + || gene.is_empty() + || pseudo.is_empty() + || !acgt(gene) + || !acgt(pseudo) + || shared + || bits.len() < 3 + { + return Err(Error(format!( + "{loc}: PSV {} must be one base with distinct gene and pseudogene bases and a text (got {}-{}, {})", + seg.name, + seg.start, + seg.end, + py_repr(label) + ))); + } + out.push(PsvSite { + name: seg.name.clone(), + gene_pos: seg.start, + pseudo_pos: pos, + gene: unique(gene), + pseudo: unique(pseudo), + text: bits[2].to_owned(), + }); + } + Ok(out) +} + +/// The distinct bases of `s`, in order (the Python kept them as a set). +fn unique(s: &str) -> Vec { + let mut out: Vec = Vec::new(); + for c in s.bytes() { + if !out.contains(&c) { + out.push(c); + } + } + out +} + +/// Checks one row (its 22 cells) as `copy_number.parse_row` does, in the +/// same order, so the first problem is the one the Python named. +pub fn parse_row(cells: &[&str]) -> Result { + let loc = cells[0]; + let kind = match cell(cells, "kind") { + "gene" => Kind::Gene, + "interval" => Kind::Interval, + "organelle" => Kind::Organelle, + k => { + return Err(Error(format!( + "{loc}: kind {} (one of gene, interval, organelle)", + py_repr(k) + ))); + } + }; + let input = match cell(cells, "input") { + "bedcov" => Input::Bedcov, + "cnvpytor-level" => Input::CnvpytorLevel, + i => { + return Err(Error(format!( + "{loc}: input {} (one of bedcov, cnvpytor-level)", + py_repr(i) + ))); + } + }; + let chrom = cell(cells, "chrom"); + if !is_chrom(chrom) { + return Err(Error(format!( + "{loc}: chrom {} (GRCh38 names, chr1 to chrM)", + py_repr(chrom) + ))); + } + let targets = segments(cells, "targets")?; + let controls = segments(cells, "controls")?; + let details = segments(cells, "details")?; + let call = call(cells)?; + let readings = readings(cells)?; + let is_psv = matches!(call, CallRule::Psv { .. }); + let psv = if is_psv { + psv_sites(loc, &details)? + } else { + Vec::new() + }; + let target_depth = cell(cells, "target_depth"); + if is_psv + && (psv.is_empty() + || target_depth != "sum" + || !PSV_FINDINGS.iter().all(|f| readings.contains_key(*f))) + { + return Err(Error(format!( + "{loc}: call psv needs PSV sites in details, target_depth sum (gene and pseudogene) and a reading for each of {}", + PSV_FINDINGS.join(", ") + ))); + } + if targets.is_empty() { + return Err(Error(format!("{loc}: no target segment"))); + } + let depth = if input == Input::Bedcov { + Some(depth(cells, &controls, &call)?) + } else { + None + }; + let mut names = HashSet::new(); + if !targets + .iter() + .chain(&controls) + .chain(&details) + .all(|s| names.insert(s.name.as_str())) + { + return Err(Error(format!("{loc}: a segment name is used twice"))); + } + let row = Locus { + id: loc.to_owned(), + kind, + chrom: chrom.to_owned(), + input, + targets, + controls, + details, + call, + readings, + psv, + target_depth: target_depth.to_owned(), + depth, + }; + if row.readings.contains_key("more") && row.max_numbered().is_none() { + return Err(Error(format!( + "{loc}: reading more= needs a numbered reading (0=, 1=, ...) to count past" + ))); + } + Ok(row) +} + +fn depth(cells: &[&str], controls: &[Segment], call: &CallRule) -> Result { + let loc = cells[0]; + if controls.is_empty() { + return Err(Error(format!("{loc}: a bedcov row needs control segments"))); + } + let target_mode = match cell(cells, "target_depth") { + "sum" => Some(TargetMode::Sum), + "pooled" => Some(TargetMode::Pooled), + _ => None, + }; + let control_mode = match cell(cells, "control_depth") { + "flanks" => Some(ControlMode::Flanks), + "pooled" => Some(ControlMode::Pooled), + _ => None, + }; + let (Some(target_mode), Some(control_mode)) = (target_mode, control_mode) else { + return Err(Error(format!( + "{loc}: target_depth sum|pooled and control_depth flanks|pooled" + ))); + }; + let scale = num(cells, "scale", None)?; + let offset = num(cells, "offset", Some(0.0))?.unwrap_or(0.0); + let scale = match scale { + Some(s) if s > 0.0 => s, + _ => return Err(Error(format!("{loc}: scale must be a positive number"))), + }; + let min_control = num(cells, "min_control_depth", Some(0.0))?.unwrap_or(0.0); + let max_ratio = num(cells, "max_control_ratio", None)?; + let mapq0_min = num(cells, "mapq0_min", None)?; + let mapq0_max = num(cells, "mapq0_max", None)?; + let passes: Vec = cell(cells, "bedcov") + .split(',') + .map(str::to_owned) + .collect(); + if (mapq0_min.is_some() || mapq0_max.is_some()) && passes.len() != 2 { + return Err(Error(format!( + "{loc}: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)" + ))); + } + let interval = cell(cells, "interval"); + if matches!(call, CallRule::Class { .. }) && interval != "cv" { + return Err(Error(format!("{loc}: call class needs interval cv"))); + } + if interval != "cv" && interval != "." { + return Err(Error(format!( + "{loc}: interval {} (cv or '.')", + py_repr(interval) + ))); + } + Ok(Depth { + target_mode, + control_mode, + scale, + offset, + passes, + mapq0_min, + mapq0_max, + min_control, + max_ratio, + ci_cv: interval == "cv", + }) +} + +/// A GRCh38 primary chromosome name: chr1 to chr22, chrX, chrY, chrM. +fn is_chrom(c: &str) -> bool { + match c.strip_prefix("chr") { + Some("X" | "Y" | "M") => true, + Some(n) => n + .parse::() + .is_ok_and(|v| (1..=22).contains(&v) && n == v.to_string()), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(over: &[(&str, &str)]) -> Vec { + let mut base: Vec<(&str, String)> = COLUMNS + .iter() + .map(|c| { + let v = match *c { + "locus" => "G", + "kind" => "gene", + "chrom" => "chr1", + "targets" => "G_A:1000-2000", + "controls" => "G_L:0-500;G_R:3000-3500", + "target_depth" => "sum", + "control_depth" => "flanks", + "scale" => "2", + "offset" => "0", + "bedcov" => ".", + "min_control_depth" => "8", + "call" => "round:0.3", + "input" => "bedcov", + _ => ".", + }; + (*c, v.to_owned()) + }) + .collect(); + for (k, v) in over { + base.iter_mut().find(|(c, _)| c == k).unwrap().1 = (*v).to_owned(); + } + base.into_iter().map(|(_, v)| v).collect() + } + + fn parse(over: &[(&str, &str)]) -> Result { + let r = row(over); + let cells: Vec<&str> = r.iter().map(String::as_str).collect(); + parse_row(&cells) + } + + #[test] + fn a_plain_row_loads() { + let l = parse(&[]).unwrap(); + assert_eq!(l.depth().scale, 2.0); + assert_eq!(l.regions()[0], ("chr1", 0, 500, "G_L")); + } + + #[test] + fn broken_rows_are_refused_with_the_python_text() { + let cases: &[(&[(&str, &str)], &str)] = &[ + ( + &[("kind", "cnv")], + "G: kind 'cnv' (one of gene, interval, organelle)", + ), + ( + &[("chrom", "chrx")], + "G: chrom 'chrx' (GRCh38 names, chr1 to chrM)", + ), + ( + &[("targets", "G_A:5-5")], + "G: targets segment 'G_A:5-5' has no name or an empty span", + ), + ( + &[("targets", "G_A:5")], + "G: targets segment 'G_A:5' is not NAME:start-end[:label]", + ), + ( + &[("call", "class:30,45")], + "G: call class needs interval cv", + ), + ( + &[("call", "cut:1.5,0.5")], + "G: call 'cut:1.5,0.5' is not round:D, cut:A,B..., class:LOW,HIGH or psv:D", + ), + ( + &[("controls", "G_A:0-500")], + "G: a segment name is used twice", + ), + (&[("scale", "0")], "G: scale must be a positive number"), + ( + &[("scale", "nan")], + "G: scale is 'nan', not a number or '.'", + ), + ( + &[("mapq0_min", "0.02")], + "G: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)", + ), + (&[("readings", "x")], "G: reading 'x' is not KEY=text"), + ]; + for (over, want) in cases { + assert_eq!(parse(over).unwrap_err().0, *want, "{over:?}"); + } + } + + #[test] + fn more_without_a_numbered_reading_is_a_catalog_error() { + let e = parse(&[("readings", "more={n} copies")]).unwrap_err(); + assert!(e.0.contains("more= needs a numbered reading"), "{e}"); + let l = parse(&[("readings", "2=two|more={n} copies")]).unwrap(); + assert_eq!(l.reading_count(5), "5 copies"); + assert_eq!(l.reading_count(2), "two"); + assert_eq!(l.reading_count(1), ""); + } + + #[test] + fn chrom_names() { + for c in ["chr1", "chr22", "chrX", "chrY", "chrM"] { + assert!(is_chrom(c), "{c}"); + } + for c in ["chr0", "chr23", "chr01", "chrx", "1", "chrUn"] { + assert!(!is_chrom(c), "{c}"); + } + } + + #[test] + fn the_repository_catalog_loads() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin/cn_loci.tsv"); + let cat = Catalog::load(path).unwrap(); + assert!(cat.bedcov("HBA").is_some() && cat.bedcov("LPA_KIV2").is_some()); + assert_eq!(cat.loci["CYP21A2"].psv.len(), 6); + } +} diff --git a/rust/crates/og-cn/src/cli.rs b/rust/crates/og-cn/src/cli.rs new file mode 100644 index 000000000..488f6d23d --- /dev/null +++ b/rust/crates/og-cn/src/cli.rs @@ -0,0 +1,746 @@ +//! `og cn `: +//! +//! ```text +//! og cn bed [--catalog TSV] [--locus L ...] [--kind interval] [--psv] +//! og cn call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] [--pileup P] +//! [--sex male|female|unknown] [--table OUT.tsv] [--bam-header H] +//! [--marker LPA_KIV2_4925 --marker-pileup P --marker-out OUT.tsv] +//! og cn intervals [--catalog TSV] --bedcov T --sex male|female|unknown --out OUT.tsv +//! og cn intervals [--catalog TSV] --failed --out OUT.tsv +//! og cn check-bam --header H +//! og cn marker-regions NAME +//! og cn marker-read NAME --pileup P [--copy-table T] --out OUT.tsv [--catalog TSV] +//! ``` +//! +//! The commands of `copy_number.py` with the same options (`--json`, which +//! no step passed, is gone), `traits_report.py --marker-regions` and +//! `--marker`, and two additions: `call --sex`, so a gene row on chrX or +//! chrY is read against the sample's count there (the Python never passed a +//! sex, so such a row could only read `sex_unknown`), and `call --marker`, +//! which reads the KIV-2 4925G>A marker beside the count in the same run and +//! judges the aligner on the count's own MAPQ 0 share. `marker-read +//! LPA_KIV2_4925 --copy-table` reads a table written earlier, where the +//! share is printed at 3 decimals, and judges that printed share. +//! +//! Exit codes: `call` 0 whatever the status, 2 on unreadable input, 3 on a +//! broken control, 4 on a BAM with ALT contigs; `check-bam` 0 or 4 (2 when +//! the header cannot be read); `bed`, `marker-regions` and `intervals` 0 +//! or 2 (2 also when stdout or the table cannot be written); `marker-read` +//! 0 whatever the reading (an unreadable pileup reads 'not assessed'), 2 +//! when the table cannot be written. Usage errors exit 2. + +use crate::catalog::{self, Catalog, Input, Locus}; +use crate::engine::{self, AssessError, Block, Means, Reason, Sex, Status}; +use crate::layout::{self, INTERVAL_COLUMNS}; +use crate::markers::{self, LPA_KIV2_4925, Marker, SMN1_27134}; +use crate::{Error, input, smn}; +use og_filters::py_repr; +use std::collections::HashMap; +use std::io::Write; + +const USAGE: &str = "usage: og cn bed [--catalog TSV] [--locus L ...] [--kind interval] [--psv] + og cn call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] [--pileup P] + [--sex male|female|unknown] [--table OUT] [--bam-header H] + [--marker LPA_KIV2_4925 --marker-pileup P --marker-out OUT] + og cn intervals [--catalog TSV] (--bedcov T --sex male|female|unknown | --failed) --out OUT + og cn check-bam --header H + og cn marker-regions NAME + og cn marker-read NAME --pileup P [--copy-table T] --out OUT [--catalog TSV] +"; + +/// A parsed command line: options with a value (`--locus` may repeat), +/// flags, positionals. +#[derive(Debug, Default)] +struct Args { + opts: HashMap<&'static str, Vec>, + flags: Vec<&'static str>, + pos: Vec, +} + +impl Args { + fn get(&self, name: &str) -> Option<&str> { + self.opts + .get(name) + .and_then(|v| v.last()) + .map(String::as_str) + } + + fn all(&self, name: &str) -> &[String] { + self.opts.get(name).map_or(&[], Vec::as_slice) + } + + fn flag(&self, name: &str) -> bool { + self.flags.contains(&name) + } +} + +/// Parses `args` by the options a verb takes, as argparse does: `--name +/// VALUE` or `--name=VALUE`, flags without a value, positionals in order. +fn parse( + args: &[String], + options: &[&'static str], + flags: &[&'static str], + positionals: usize, + required: &[&str], +) -> Result { + let mut out = Args::default(); + let mut extra = Vec::new(); + let mut i = 0; + while i < args.len() { + let a = &args[i]; + i += 1; + if a == "-h" || a == "--help" { + return Err(String::new()); + } + if a == "-" || !a.starts_with('-') { + if out.pos.len() < positionals { + out.pos.push(a.clone()); + } else { + extra.push(a.clone()); + } + continue; + } + let (name, inline) = match a.split_once('=') { + Some((n, v)) => (n, Some(v.to_owned())), + None => (a.as_str(), None), + }; + if let Some(&f) = flags.iter().find(|f| **f == name) { + if inline.is_some() { + return Err(format!( + "argument {f}: ignored explicit argument {}", + py_repr(inline.as_deref().unwrap_or("")) + )); + } + out.flags.push(f); + continue; + } + let Some(&known) = options.iter().find(|o| **o == name) else { + extra.push(a.clone()); + continue; + }; + let value = match inline { + Some(v) => v, + None => match args.get(i) { + Some(v) if v == "-" || !v.starts_with('-') => { + i += 1; + v.clone() + } + _ => return Err(format!("argument {known}: expected one argument")), + }, + }; + out.opts.entry(known).or_default().push(value); + } + if out.pos.len() < positionals { + return Err("the following arguments are required: NAME".into()); + } + let missing: Vec<&str> = required + .iter() + .copied() + .filter(|r| !out.opts.contains_key(r)) + .collect(); + if !missing.is_empty() { + return Err(format!( + "the following arguments are required: {}", + missing.join(", ") + )); + } + if !extra.is_empty() { + return Err(format!("unrecognized arguments: {}", extra.join(" "))); + } + Ok(out) +} + +fn usage(verb: &str, msg: &str, err: &mut dyn Write) -> u8 { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og cn {verb}: error: {msg}"); + 2 +} + +/// `og cn [args]`: the exit code. +pub fn main(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + return usage("", "the following arguments are required: verb", err); + }; + let rest = &args[1..]; + let spec: (&[&'static str], &[&'static str], usize, &[&str]) = match verb.as_str() { + "bed" => (&["--catalog", "--locus", "--kind"], &["--psv"], 0, &[]), + "call" => ( + &[ + "--catalog", + "--locus", + "--bedcov", + "--bedcov-mapq1", + "--table", + "--bam-header", + "--pileup", + "--sex", + "--marker", + "--marker-pileup", + "--marker-out", + ], + &[], + 0, + &["--locus", "--bedcov"], + ), + "intervals" => ( + &["--catalog", "--bedcov", "--sex", "--out"], + &["--failed"], + 0, + &["--out"], + ), + "check-bam" => (&["--header"], &[], 0, &["--header"]), + "marker-regions" => (&[], &[], 1, &[]), + "marker-read" => ( + &["--pileup", "--copy-table", "--out", "--catalog"], + &[], + 1, + &["--pileup", "--out"], + ), + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => return usage("", &format!("unknown cn verb {}", py_repr(other)), err), + }; + let a = match parse(rest, spec.0, spec.1, spec.2, spec.3) { + Ok(a) => a, + Err(m) if m.is_empty() => { + let _ = write!(out, "{USAGE}"); + return 0; + } + Err(m) => return usage(verb, &m, err), + }; + if let Some(m) = choice_error(&a) { + return usage(verb, &m, err); + } + let code = match verb.as_str() { + "bed" => bed(&a, out), + "call" => call(&a, out, err), + "intervals" => intervals(&a, out, err), + "check-bam" => check_bam(&a, err), + "marker-regions" => { + let m = markers::marker(&a.pos[0]).expect("checked"); + markers::regions(m) + .iter() + .try_for_each(|r| writeln!(out, "{r}")) + .and_then(|()| out.flush()) + .map_err(unwritten) + .map(|()| 0) + } + _ => marker_read(&a, out, err), + }; + let _ = out.flush(); + match code { + Ok(c) => c, + Err((c, e)) => { + let _ = writeln!(err, "ERROR: {e}"); + c + } + } +} + +/// The argparse `choices` checks. +fn choice_error(a: &Args) -> Option { + let bad = |name: &str, v: &str, choices: &[&str]| { + Some(format!( + "argument {name}: invalid choice: {} (choose from {})", + py_repr(v), + choices + .iter() + .map(|c| py_repr(c)) + .collect::>() + .join(", ") + )) + }; + if let Some(k) = a.get("--kind").filter(|k| *k != "interval") { + return bad("--kind", k, &["interval"]); + } + if let Some(s) = a.get("--sex").filter(|s| Sex::parse(s).is_none()) { + return bad("--sex", s, &["male", "female", "unknown"]); + } + if let Some(m) = a.get("--marker").filter(|m| *m != LPA_KIV2_4925.key) { + return bad("--marker", m, &[LPA_KIV2_4925.key]); + } + if let Some(n) = a.pos.first().filter(|n| markers::marker(n).is_none()) { + return bad("NAME", n, &[LPA_KIV2_4925.key, SMN1_27134.key]); + } + None +} + +type Fail = (u8, Error); + +fn fail2(e: Error) -> Fail { + (2, e) +} + +/// A region list that could not be written in full: exit 2, so a step +/// never reads a truncated list. +fn unwritten(e: std::io::Error) -> Fail { + fail2(Error(format!("cannot write the regions: {e}"))) +} + +fn load(a: &Args) -> Result { + match a.get("--catalog") { + Some(p) => Catalog::load(p), + None => { + let p = catalog::default_path() + .ok_or_else(|| Error("no bin/cn_loci.tsv beside og; give --catalog".to_owned()))?; + Catalog::load(&p.to_string_lossy()) + } + } +} + +fn bed(a: &Args, out: &mut dyn Write) -> Result { + let cat = load(a).map_err(fail2)?; + let loci = a.all("--locus"); + if a.flag("--psv") { + if loci.is_empty() || a.get("--kind").is_some() { + return Err(fail2(Error( + "bed --psv needs --locus (and no --kind)".into(), + ))); + } + for loc in loci { + let Some(r) = cat.loci.get(loc).filter(|r| !r.psv.is_empty()) else { + return Err(fail2(Error(format!( + "no psv locus {} in the catalog", + py_repr(loc) + )))); + }; + let mut pos: Vec<(i64, String)> = r + .psv + .iter() + .map(|p| (p.gene_pos, p.name.clone())) + .chain( + r.psv + .iter() + .map(|p| (p.pseudo_pos, format!("{}_pseudo", p.name))), + ) + .collect(); + pos.sort(); + for (s, n) in pos { + writeln!(out, "{}\t{s}\t{}\t{n}", r.chrom, s + 1).map_err(unwritten)?; + } + } + out.flush().map_err(unwritten)?; + return Ok(0); + } + let names: Vec = if a.get("--kind").is_some() { + if !loci.is_empty() { + return Err(fail2(Error( + "bed: --kind interval or --locus, not both".into(), + ))); + } + cat.loci + .values() + .filter(|r| r.input == Input::Bedcov && r.is_interval_row()) + .map(|r| r.id.clone()) + .collect() + } else if loci.is_empty() { + cat.loci + .values() + .filter(|r| r.input == Input::Bedcov) + .map(|r| r.id.clone()) + .collect() + } else { + loci.to_vec() + }; + for loc in &names { + let Some(r) = cat.bedcov(loc) else { + return Err(fail2(Error(format!( + "no bedcov locus {} in the catalog", + py_repr(loc) + )))); + }; + for (c, s, e, n) in r.regions() { + writeln!(out, "{c}\t{s}\t{e}\t{n}").map_err(unwritten)?; + } + } + out.flush().map_err(unwritten)?; + Ok(0) +} + +fn check_bam(a: &Args, err: &mut dyn Write) -> Result { + match input::check_bam(a.get("--header").expect("required")).map_err(fail2)? { + Some(refusal) => { + let _ = writeln!(err, "{refusal}"); + Ok(4) + } + None => Ok(0), + } +} + +/// The count of one locus, and with `--marker`, the KIV-2 4925G>A reading. +fn call(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + if let Some(h) = a.get("--bam-header") + && let Some(refusal) = input::check_bam(h).map_err(fail2)? + { + let _ = writeln!(err, "{refusal}"); + return Ok(4); + } + let marker = match ( + a.get("--marker"), + a.get("--marker-pileup"), + a.get("--marker-out"), + ) { + (None, None, None) => None, + (Some(_), Some(p), Some(o)) if a.get("--locus") == Some("LPA_KIV2") => Some((p, o)), + _ => { + return Err(fail2(Error( + "--marker LPA_KIV2_4925 goes with --locus LPA_KIV2, --marker-pileup and --marker-out".into(), + ))); + } + }; + let sex = a.get("--sex").and_then(Sex::parse).unwrap_or_default(); + let cat = load(a).map_err(fail2)?; + let locus = a.get("--locus").expect("required"); + let row = cat.bedcov(locus).ok_or_else(|| { + fail2(Error(format!( + "no bedcov locus {} in the catalog", + py_repr(locus) + ))) + })?; + let block = count(a, row, sex).map_err(|e| match e { + Count::Broken(s) => (3, Error(s)), + Count::Input(e) => (2, e), + })?; + let (rows, line) = layout::table(row, &block); + if let Some(t) = a.get("--table") { + write_table(t, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + } + let _ = writeln!(out, "{line}"); + if let Some((pileup, path)) = marker { + let kiv2 = rows + .iter() + .find(|(k, _)| k == "kiv2_copies") + .map_or(".", |(_, v)| v.as_str()); + let why = match (block.mapq0_share, row.depth().mapq0_min) { + (Some(s), Some(min)) => markers::mapq0_why_not(s, min), + _ => None, + }; + let counts = marker_counts(pileup, &LPA_KIV2_4925, "KIV-2 4925G>A is not assessed", err); + let rows = markers::null_allele(counts, kiv2, why.as_deref()); + write_table(path, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let _ = writeln!(out, "LPA {}", rows[8].1); + } + Ok(0) +} + +enum Count { + Broken(String), + Input(Error), +} + +impl From for Count { + fn from(e: Error) -> Count { + Count::Input(e) + } +} + +fn count(a: &Args, row: &Locus, sex: Sex) -> Result { + let bedcov = a.get("--bedcov").expect("required"); + let m = input::read_bedcov(bedcov, row)?; + let dp = row.depth(); + let q1_path = a.get("--bedcov-mapq1"); + if (dp.mapq0_min.is_some() || dp.mapq0_max.is_some()) && q1_path.is_none() { + return Err(Error(format!( + "{} has a MAPQ 0 guard: --bedcov-mapq1 is needed", + row.id + )) + .into()); + } + let q1 = q1_path.map(|p| input::read_bedcov(p, row)).transpose()?; + if let (Some(q), Some(p)) = (&q1, q1_path) + && m.is_empty() != q.is_empty() + { + let which = if m.is_empty() { bedcov } else { p }; + return Err(Error(format!( + "{which} is empty but the other table is not: both must cover the regions, or both be empty (no reads on that contig)" + )) + .into()); + } + let block = engine::assess(row, &m, q1.as_ref(), sex).map_err(|e| match e { + AssessError::BrokenControl(s) => Count::Broken(s), + AssessError::NeedsMapq1(s) => Count::Input(Error(s)), + })?; + if row.psv.is_empty() { + return Ok(block); + } + let Some(pileup) = a.get("--pileup") else { + return Err(Error(format!( + "{} is a psv row: --pileup (samtools mpileup over `og cn bed --psv`) is needed", + row.id + )) + .into()); + }; + let counts = if m.is_empty() { + HashMap::new() + } else { + input::read_psv_pileup(pileup, row)? + }; + Ok(engine::psv_assess(row, block, &counts)) +} + +fn intervals(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + let cat = load(a).map_err(fail2)?; + let rows: Vec<&Locus> = cat.loci.values().filter(|r| r.is_interval_row()).collect(); + let lines: Vec> = if a.flag("--failed") { + rows.iter() + .map(|r| { + let mut b = Block::empty(); + b.reason = Some(Reason::CheckFailed); + layout::interval_line(r, &b) + }) + .collect() + } else { + let (Some(bedcov), Some(sex)) = (a.get("--bedcov"), a.get("--sex").and_then(Sex::parse)) + else { + return Err(fail2(Error( + "intervals needs --bedcov and --sex (or --failed)".into(), + ))); + }; + call_intervals( + &rows, + &input::read_bedcov_all(bedcov).map_err(fail2)?, + sex, + err, + ) + .map_err(fail2)? + }; + let path = a.get("--out").expect("required"); + let mut body = INTERVAL_COLUMNS.join("\t"); + body.push('\n'); + for l in &lines { + body.push_str(&l.join("\t")); + body.push('\n'); + } + write_atomic(path, body.as_bytes()).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let mut n: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for l in &lines { + *n.entry(l[5].as_str()).or_default() += 1; + } + let parts: Vec = n.iter().map(|(k, v)| format!("{v} {k}")).collect(); + let _ = writeln!(out, "interval rows: {} ({})", lines.len(), parts.join(", ")); + Ok(0) +} + +/// Every interval row from one bedcov table; a broken control makes that +/// row `not assessed` (broken_control), never the whole table. +fn call_intervals( + rows: &[&Locus], + table: &[input::BedcovLine], + sex: Sex, + err: &mut dyn Write, +) -> Result>, Error> { + let mut by_name: HashMap<&str, &input::BedcovLine> = HashMap::new(); + for l in table { + if by_name.insert(&l.name, l).is_some() { + return Err(Error(format!( + "segment {} appears twice in the bedcov table", + l.name + ))); + } + } + let mut out = Vec::new(); + for row in rows { + let want = row.regions(); + if want.iter().any(|w| !by_name.contains_key(w.3)) { + return Err(Error(format!( + "the bedcov table lacks some of {}'s segments: its regions differ from `og cn bed --kind interval`", + row.id + ))); + } + let mut m = Means::new(); + for (c, s, e, n) in want { + let l = by_name[n]; + if (l.chrom.as_str(), l.start, l.end) != (c, s, e) { + return Err(Error(format!( + "segment {n} of {} has other coordinates in the bedcov table", + row.id + ))); + } + m.insert(n.to_owned(), l.sum / (e - s) as f64); + } + let block = match engine::assess(row, &m, None, sex) { + Ok(b) => b, + Err(e) => { + let _ = writeln!(err, "WARNING: {e}"); + let mut b = engine::assess(row, &Means::new(), None, sex) + .map_err(|e| Error(e.to_string()))?; + if b.status == Status::NotAssessed { + b.reason = Some(Reason::BrokenControl); + } + b + } + }; + out.push(layout::interval_line(row, &block)); + } + Ok(out) +} + +/// Reads a marker's pileup; an unreadable one is a warning and no counts. +fn marker_counts( + pileup: &str, + m: &Marker, + what: &str, + err: &mut dyn Write, +) -> Option { + match markers::read_pileup(pileup, m) { + Ok(c) => Some(c), + Err(e) => { + let _ = writeln!(err, "WARNING: {e}; {what}"); + None + } + } +} + +fn marker_read(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + let m = markers::marker(&a.pos[0]).expect("checked"); + let pileup = a.get("--pileup").expect("required"); + let path = a.get("--out").expect("required"); + let copy_table = a.get("--copy-table"); + let (rows, line) = if LPA_KIV2_4925.key == m.key { + // Step 46's table written earlier: the count, and the MAPQ 0 share + // as it was printed (3 decimals). + let t: HashMap = crate::read_lines(copy_table.unwrap_or("")) + .map(|ls| { + ls.iter() + .filter_map(|l| l.split_once('\t')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect() + }) + .unwrap_or_default(); + let kiv2 = t.get("kiv2_copies").map_or(".", String::as_str); + let why = match t.get("mapq0_share").map(|s| pycompat::num::parse_float(s)) { + Some(Ok(share)) => { + let cat = load(a).map_err(fail2)?; + let floor = cat + .bedcov("LPA_KIV2") + .and_then(|r| r.depth().mapq0_min) + .ok_or_else(|| { + fail2(Error( + "no LPA_KIV2 row with a MAPQ 0 floor in the catalog".into(), + )) + })?; + markers::mapq0_why_not(share, floor) + } + _ => None, + }; + let counts = marker_counts(pileup, m, "KIV-2 4925G>A is not assessed", err); + let rows = markers::null_allele(counts, kiv2, why.as_deref()); + let line = format!("LPA {}", rows[8].1); + (rows, line) + } else { + let counts = marker_counts(pileup, m, "the marker is not assessed", err); + let read = match smn::read(copy_table, m.chrom, m.copies) { + Ok(r) => r, + Err(e) => { + let _ = writeln!(err, "WARNING: {e}; the SMN copy numbers are not read"); + smn::SmnRead { + pscn: None, + text: "not read (the SMN table could not be read)".into(), + agcn: None, + } + } + }; + let rows = markers::smn1_marker(counts, &read); + let line = rows[9].1.clone(); + (rows, line) + }; + write_table(path, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let _ = writeln!(out, "{line}"); + Ok(0) +} + +/// Writes a `metricvalue` table whole, through a temporary file. +fn write_table(path: &str, rows: &[(String, String)]) -> Result<(), String> { + let mut body = String::from("metric\tvalue\n"); + for (k, v) in rows { + body.push_str(k); + body.push('\t'); + body.push_str(v); + body.push('\n'); + } + write_atomic(path, body.as_bytes()) +} + +fn write_atomic(path: &str, body: &[u8]) -> Result<(), String> { + let run = || -> std::io::Result<()> { + let mut f = og_io::AtomicFile::create(path)?; + f.write_all(body)?; + f.commit() + }; + run().map_err(|e| og_filters::os_error(&e, path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + let (mut o, mut e) = (Vec::new(), Vec::new()); + let c = main(&args, &mut o, &mut e); + ( + c, + String::from_utf8(o).unwrap(), + String::from_utf8(e).unwrap(), + ) + } + + const CAT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin/cn_loci.tsv"); + + #[test] + fn marker_regions_lists_every_copy() { + let (c, o, _) = run(&["marker-regions", "SMN1_27134"]); + assert_eq!( + (c, o.as_str()), + (0, "chr5:70952074-70952074\nchr5:70076654-70076654\n") + ); + assert_eq!(run(&["marker-regions", "NOPE"]).0, 2); + } + + #[test] + fn bed_psv_and_usage() { + let (c, o, _) = run(&["bed", "--catalog", CAT, "--psv", "--locus", "CYP21A2"]); + assert_eq!(c, 0); + assert_eq!(o.lines().count(), 12); + let (c, _, e) = run(&["bed", "--catalog", CAT, "--psv", "--locus", "HBA"]); + assert_eq!( + (c, e.as_str()), + (2, "ERROR: no psv locus 'HBA' in the catalog\n") + ); + assert_eq!(run(&["call", "--locus", "HBA"]).0, 2); + } + + /// A stdout that refuses every write, as a full disk does. + struct Full; + impl Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("no space left on device")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn a_region_list_that_cannot_be_written_fails() { + // A truncated region list would let a step read the marker at one + // copy, or count over part of a locus; each writer must exit 2. + for args in [ + &["marker-regions", "SMN1_27134"][..], + &["bed", "--catalog", CAT, "--locus", "HBA"], + &["bed", "--catalog", CAT, "--psv", "--locus", "CYP21A2"], + ] { + let args: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + let mut e = Vec::new(); + let c = main(&args, &mut Full, &mut e); + let e = String::from_utf8(e).unwrap(); + assert_eq!(c, 2, "{args:?}: {e}"); + assert!( + e.contains("cannot write the regions: no space left on device"), + "{e}" + ); + } + } +} diff --git a/rust/crates/og-cn/src/engine.rs b/rust/crates/og-cn/src/engine.rs new file mode 100644 index 000000000..4e49373d1 --- /dev/null +++ b/rust/crates/og-cn/src/engine.rs @@ -0,0 +1,658 @@ +//! The depth-ratio engine: the guards in their order, the estimate, its 95% +//! interval and the call (`copy_number.assess`), and the PSV sites of a +//! `call psv` row (`copy_number.psv_assess`). The arithmetic is the +//! Python's step for step, with CPython 3.12's compensated `sum()` for +//! every interpreter (Parascopy's Python 3.11 summed plainly; the two agree +//! at every printed precision). + +use crate::catalog::{CallRule, ControlMode, Locus, Segment, TargetMode}; +use crate::round_i; +use pycompat::num::sum as py_sum; +use std::collections::HashMap; +use std::fmt; + +/// Mean depth per segment name. +pub type Means = HashMap; + +/// z of a two-sided 95% interval. +pub const Z95: f64 = 1.96; +/// Reads a PSV site needs (gene and pseudogene bases pooled) before its +/// copies are read. +pub const PSV_MIN_READS: u64 = 30; +/// A site reads as converted when its copies estimate lies this far under +/// the others' count (or under the usual count) or more. At 30x, about 70 +/// reads at a site, sampling alone gives an estimate of 2 copies a standard +/// deviation of about 0.25; HG002's six sites read 1.62 to 2.82 (35x), so +/// 0.6 is a screen setting, not a measured error rate. +pub const PSV_MARGIN: f64 = 0.6; + +/// The sample's sex, which sets the copies of chrX and chrY. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Sex { + Male, + Female, + #[default] + Unknown, +} + +impl Sex { + /// `male`, `female` or `unknown`. + pub fn parse(s: &str) -> Option { + match s { + "male" => Some(Sex::Male), + "female" => Some(Sex::Female), + "unknown" => Some(Sex::Unknown), + _ => None, + } + } +} + +/// What a block says. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Called, + NotAssessed, + /// A chrY row on a female (reason `no_y`). + NotApplicable, +} + +impl fmt::Display for Status { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Status::Called => "called", + Status::NotAssessed => "not assessed", + Status::NotApplicable => "not applicable", + }) + } +} + +/// Why a block is not called: the guard that stopped it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reason { + NoData, + LowControlDepth, + ControlsDisagree, + LowMapq0Share, + HighMapq0Share, + BelowFloor, + BetweenCounts, + SexUnknown, + NoY, + BrokenControl, + CheckFailed, + PsvLowDepth, +} + +impl fmt::Display for Reason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Reason::NoData => "no_data", + Reason::LowControlDepth => "low_control_depth", + Reason::ControlsDisagree => "controls_disagree", + Reason::LowMapq0Share => "low_mapq0_share", + Reason::HighMapq0Share => "high_mapq0_share", + Reason::BelowFloor => "below_floor", + Reason::BetweenCounts => "between_counts", + Reason::SexUnknown => "sex_unknown", + Reason::NoY => "no_y", + Reason::BrokenControl => "broken_control", + Reason::CheckFailed => "check_failed", + Reason::PsvLowDepth => "psv_low_depth", + }) + } +} + +/// A `call class` result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Class { + Low, + Typical, + High, +} + +impl Class { + pub fn as_str(self) -> &'static str { + match self { + Class::Low => "low", + Class::Typical => "typical", + Class::High => "high", + } + } +} + +/// What the PSV sites of a `call psv` row show. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Finding { + None, + Deletion, + Conversion, + PseudogeneDeletion, +} + +impl Finding { + pub fn as_str(self) -> &'static str { + match self { + Finding::None => "none", + Finding::Deletion => "deletion", + Finding::Conversion => "conversion", + Finding::PseudogeneDeletion => "pseudogene_deletion", + } + } +} + +/// One PSV site as read. +#[derive(Debug, Clone, PartialEq)] +pub struct SiteRead { + pub name: String, + pub gene_reads: u64, + pub pseudo_reads: u64, + pub other_reads: u64, + pub copies: Option, + pub estimate: Option, + pub ratio: Option, +} + +/// A locus's result. +#[derive(Debug, Clone, PartialEq)] +pub struct Block { + pub status: Status, + pub reason: Option, + pub estimate: Option, + pub total: Option, + pub copies: Option, + pub ci95: Option<(f64, f64)>, + pub class: Option, + pub target_depth: Option, + pub control_depth: Option, + /// Mean depth over every segment read (every read). + pub depths: Means, + /// The share of the target depth from reads of MAPQ 0; `None` without a + /// MAPQ >= 1 pass, or when a guard stopped before the depths. + pub mapq0_share: Option, + pub reading: String, + /// The copies with no change, when the sample's count of the chromosome + /// is known. + pub expected: Option, + /// A `call psv` row: the gene and pseudogene copies together. + pub module_total: Option, + /// A `call psv` row: its sites, in catalog order (after `psv_assess`). + pub sites: Vec, + pub finding: Option, + pub conversion_sites: Vec, +} + +impl Block { + /// A block that says nothing yet (not assessed, no reason). + pub fn empty() -> Block { + Block { + status: Status::NotAssessed, + reason: None, + estimate: None, + total: None, + copies: None, + ci95: None, + class: None, + target_depth: None, + control_depth: None, + depths: Means::new(), + mapq0_share: None, + reading: String::new(), + expected: None, + module_total: None, + sites: Vec::new(), + finding: None, + conversion_sites: Vec::new(), + } + } + + fn stop(mut self, reason: Reason) -> Block { + self.reason = Some(reason); + self + } +} + +/// Why `assess` gave no block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AssessError { + /// A control segment has no read while the other controls are covered: + /// the row's coordinates are wrong, or the sample lacks that stretch. + BrokenControl(String), + /// A row with a MAPQ 0 guard was given no MAPQ >= 1 depths. + NeedsMapq1(String), +} + +impl fmt::Display for AssessError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + AssessError::BrokenControl(s) | AssessError::NeedsMapq1(s) => f.write_str(s), + } + } +} + +fn combine_targets(mode: TargetMode, segs: &[Segment], m: &dyn Fn(&str) -> f64) -> f64 { + match mode { + TargetMode::Sum => py_sum(segs.iter().map(|s| m(&s.name))), + TargetMode::Pooled => pooled(segs, m), + } +} + +fn combine_controls(mode: ControlMode, segs: &[&Segment], m: &dyn Fn(&str) -> f64) -> f64 { + match mode { + ControlMode::Flanks => py_sum(segs.iter().map(|s| m(&s.name))) / segs.len() as f64, + ControlMode::Pooled => { + let owned: Vec = segs.iter().map(|s| (*s).clone()).collect(); + pooled(&owned, m) + } + } +} + +/// Length-weighted mean: `sum(m * len) / sum(len)`. +fn pooled(segs: &[Segment], m: &dyn Fn(&str) -> f64) -> f64 { + let num = py_sum(segs.iter().map(|s| m(&s.name) * (s.end - s.start) as f64)); + let den: i64 = segs.iter().map(|s| s.end - s.start).sum(); + num / den as f64 +} + +/// Coefficient of variation: sample standard deviation over the mean; 0 +/// with fewer than two values or a mean of 0 or less. +fn cv(values: &[f64]) -> f64 { + let mu = py_sum(values.iter().copied()) / values.len() as f64; + if mu <= 0.0 || values.len() < 2 { + return 0.0; + } + (py_sum(values.iter().map(|v| (v - mu).powi(2))) / (values.len() - 1) as f64).sqrt() / mu +} + +/// The copies of the row's chromosome a sample has: 2, or 1 on a male's +/// chrX or chrY; `None` where it cannot be said (a chrY row on a female or +/// on a sample of no known sex, a chrX row without a sex). +pub fn ploidy(row: &Locus, sex: Sex) -> Option { + let x = row.chrom == "chrX"; + if !x && row.chrom != "chrY" { + return Some(2); + } + match sex { + Sex::Male => Some(1), + Sex::Female if x => Some(2), + _ => None, + } +} + +/// The locus's block for mean depths `m` (every read) and `q1` (reads of +/// MAPQ >= 1, or `None` for a row without a MAPQ 0 guard). +pub fn assess(row: &Locus, m: &Means, q1: Option<&Means>, sex: Sex) -> Result { + let dp = row.depth(); + if q1.is_none() && (dp.mapq0_min.is_some() || dp.mapq0_max.is_some()) { + return Err(AssessError::NeedsMapq1(format!( + "{} has a MAPQ 0 guard: the MAPQ >= 1 depths are needed", + row.id + ))); + } + let mut b = Block::empty(); + b.depths = m.clone(); + let units = if row.target_depth == "sum" { + row.targets.len() as i64 + } else { + 1 + }; + let pl = ploidy(row, sex); + b.expected = pl.map(|p| p * units); + if row.chrom == "chrY" && sex == Sex::Female { + b.status = Status::NotApplicable; + return Ok(b.stop(Reason::NoY)); + } + let Some(pl) = pl else { + return Ok(b.stop(Reason::SexUnknown)); + }; + let (tg, ct) = (&row.targets, &row.controls); + if m.is_empty() || tg.iter().chain(ct).any(|s| !m.contains_key(&s.name)) { + return Ok(b.stop(Reason::NoData)); + } + let get = |n: &str| m[n]; + for s in ct { + let rest: Vec<&Segment> = ct.iter().filter(|c| c.name != s.name).collect(); + if m[&s.name] == 0.0 && !rest.is_empty() { + let other = combine_controls(dp.control_mode, &rest, &get); + if other >= dp.min_control && dp.min_control > 0.0 { + return Err(AssessError::BrokenControl(format!( + "BROKEN_CONTROL: control segment {} of {} has no read while the other controls have {}x; check its coordinates in bin/cn_loci.tsv (or the sample lacks that stretch)", + s.name, + row.id, + crate::fixed(other, 1) + ))); + } + } + } + let all_ct: Vec<&Segment> = ct.iter().collect(); + let t = combine_targets(dp.target_mode, tg, &get); + let c = combine_controls(dp.control_mode, &all_ct, &get); + let share = q1.map(|q| { + let tq = combine_targets(dp.target_mode, tg, &|n: &str| { + q.get(n).copied().unwrap_or(0.0) + }); + if t > 0.0 { 1.0 - tq / t } else { 0.0 } + }); + b.target_depth = Some(t); + b.control_depth = Some(c); + b.mapq0_share = share; + let low = match dp.control_mode { + ControlMode::Flanks => ct.iter().map(|s| m[&s.name]).fold(f64::INFINITY, f64::min), + ControlMode::Pooled => c, + }; + if low < dp.min_control || c <= 0.0 { + return Ok(b.stop(Reason::LowControlDepth)); + } + if dp.control_mode == ControlMode::Flanks + && ct.len() > 1 + && let Some(max_ratio) = dp.max_ratio + { + let vals: Vec = ct.iter().map(|s| m[&s.name]).collect(); + let lo = vals.iter().copied().fold(f64::INFINITY, f64::min); + let hi = vals.iter().copied().fold(f64::NEG_INFINITY, f64::max); + if lo <= 0.0 || hi / lo > max_ratio { + return Ok(b.stop(Reason::ControlsDisagree)); + } + } + let total = dp.scale * pl as f64 / 2.0 * t / c; + let est = total - dp.offset; + b.total = Some(total); + b.estimate = Some(est); + // With no copy left there is no read to judge the aligner by (a row with + // an offset has a floor of its own below, after this guard). + let no_copy = dp.offset == 0.0 && round_i(est) == 0; + if let (Some(min), Some(s)) = (dp.mapq0_min, share) + && !no_copy + && s < min + { + return Ok(b.stop(Reason::LowMapq0Share)); + } + if let (Some(max), Some(s)) = (dp.mapq0_max, share) + && s > max + { + return Ok(b.stop(Reason::HighMapq0Share)); + } + if dp.offset > 0.0 && est <= 0.0 { + return Ok(b.stop(Reason::BelowFloor)); + } + if dp.ci_cv { + let tv: Vec = tg.iter().map(|s| m[&s.name]).collect(); + let cv_c: Vec = ct.iter().map(|s| m[&s.name]).collect(); + let rel = (cv(&tv).powi(2) / tg.len() as f64 + cv(&cv_c).powi(2) / ct.len() as f64).sqrt(); + b.ci95 = Some(( + (total * (-Z95 * rel).exp() - dp.offset).max(0.0), + total * (Z95 * rel).exp() - dp.offset, + )); + } + match &row.call { + CallRule::Psv { max_off } | CallRule::Round { max_off } => { + let n = round_i(est); + if (est - n as f64).abs() > *max_off { + return Ok(b.stop(Reason::BetweenCounts)); + } + if matches!(row.call, CallRule::Psv { .. }) { + b.module_total = Some(n); + } else { + b.copies = Some(n); + b.reading = row.reading_count(n); + } + } + CallRule::Cut(cuts) => { + let n = cut_copies(est, cuts); + b.copies = Some(n); + b.reading = row.reading_count(n); + } + CallRule::Class { low, high } => { + let (lo, hi) = b.ci95.expect("a class row has interval cv"); + let class = if hi < *low { + Class::Low + } else if lo > *high { + Class::High + } else { + Class::Typical + }; + b.class = Some(class); + b.reading = row.reading_word(class.as_str()); + } + } + b.status = Status::Called; + Ok(b) +} + +/// The number of cut points a value reaches (`cut:0.5,1.5` gives 0, 1 or 2). +pub fn cut_copies(value: f64, cuts: &[f64]) -> i64 { + cuts.iter().filter(|c| value >= **c).count() as i64 +} + +/// Gene-base, pseudogene-base and other reads at one PSV site. +pub type SiteCounts = (u64, u64, u64); + +/// A `call psv` row's block (from [`assess`]) with its sites read. With the +/// module total called, each site of [`PSV_MIN_READS`] reads or more gets +/// the copies carrying the gene's base there (total x gene reads / gene and +/// pseudogene reads) and its ratio to the median of the sites. A site is +/// low when its estimate is more than [`PSV_MARGIN`] under the larger of +/// the median's count and the usual. The CYP21A2 copies are the least count +/// over the low sites, or the median's count when none is low. Finding: +/// `deletion` when a site is low and the module total under the usual; +/// `conversion` at the low sites when the module total is kept; +/// `pseudogene_deletion` when the module total is under the usual with no +/// low site; else `none`. +pub fn psv_assess(row: &Locus, mut b: Block, counts: &HashMap) -> Block { + b.sites = row + .psv + .iter() + .map(|p| { + let (g, ps, o) = counts.get(&p.name).copied().unwrap_or((0, 0, 0)); + SiteRead { + name: p.name.clone(), + gene_reads: g, + pseudo_reads: ps, + other_reads: o, + copies: None, + estimate: None, + ratio: None, + } + }) + .collect(); + b.finding = None; + b.conversion_sites = Vec::new(); + let (Status::Called, Some(module_total)) = (b.status, b.module_total) else { + b.status = Status::NotAssessed; + return b; + }; + let total = b.total.expect("a called block has a total"); + let expected = b.expected.expect("a called block has an expected count"); + let usual = expected.div_euclid(row.targets.len() as i64); + let mut read = Vec::new(); + for (i, s) in b.sites.iter_mut().enumerate() { + let n = s.gene_reads + s.pseudo_reads; + if n >= PSV_MIN_READS { + let e = total * s.gene_reads as f64 / n as f64; + s.estimate = Some(e); + s.copies = Some(round_i(e)); + read.push(i); + } + } + if read.is_empty() { + b.status = Status::NotAssessed; + b.reason = Some(Reason::PsvLowDepth); + b.copies = None; + return b; + } + let mut est: Vec = read.iter().filter_map(|&i| b.sites[i].estimate).collect(); + est.sort_by(f64::total_cmp); + let mid = est.len() / 2; + let median = if est.len() % 2 == 1 { + est[mid] + } else { + (est[mid - 1] + est[mid]) / 2.0 + }; + for &i in &read { + let e = b.sites[i].estimate.expect("read"); + b.sites[i].ratio = (median > 0.0).then(|| e / median); + } + let floor = round_i(median).max(usual) as f64; + let conv: Vec<&SiteRead> = b + .sites + .iter() + .filter(|s| s.estimate.is_some_and(|e| e < floor - PSV_MARGIN)) + .collect(); + // Only a site past the margin lowers the count, so one noisy site never + // gives fewer copies than the finding says. + let copies = conv + .iter() + .filter_map(|s| s.copies) + .min() + .unwrap_or_else(|| round_i(median)); + let conv_names: Vec = conv.iter().map(|s| s.name.clone()).collect(); + let finding = if !conv_names.is_empty() && module_total < expected { + Finding::Deletion + } else if !conv_names.is_empty() { + b.conversion_sites = conv_names.clone(); + Finding::Conversion + } else if module_total < expected { + Finding::PseudogeneDeletion + } else { + Finding::None + }; + b.copies = Some(copies); + b.finding = Some(finding); + let sites = conv_names + .iter() + .map(|n| { + let text = row + .psv + .iter() + .find(|p| &p.name == n) + .map_or("", |p| p.text.as_str()); + format!("{n} ({text})") + }) + .collect::>() + .join(", "); + b.reading = row + .reading_word(finding.as_str()) + .replace("{sites}", &sites); + b +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::Catalog; + + fn catalog() -> Catalog { + Catalog::load(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../bin/cn_loci.tsv" + )) + .unwrap() + } + + /// Depths with every segment of `row` at `x`, `over` overriding some. + fn means(row: &Locus, x: f64, over: &[(&str, f64)]) -> Means { + let mut m: Means = row + .targets + .iter() + .chain(&row.controls) + .map(|s| (s.name.clone(), x)) + .collect(); + for (k, v) in over { + m.insert((*k).to_owned(), *v); + } + m + } + + fn scaled(m: &Means, f: f64) -> Means { + m.iter().map(|(k, v)| (k.clone(), v * f)).collect() + } + + #[test] + fn hba_counts_and_guards() { + let cat = catalog(); + let hba = &cat.loci["HBA"]; + for (h2, h1, want) in [ + (30.0, 30.0, Some(4)), + (15.0, 30.0, Some(3)), + (15.0, 15.0, Some(2)), + ] { + let m = means(hba, 30.0, &[("HBA2", h2), ("HBA1", h1)]); + let b = assess(hba, &m, Some(&scaled(&m, 0.75)), Sex::Unknown).unwrap(); + assert_eq!((b.status, b.copies), (Status::Called, want)); + } + let m = means(hba, 30.0, &[("HBA2", 22.5)]); + let b = assess(hba, &m, Some(&scaled(&m, 0.75)), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::BetweenCounts)); + let m = means(hba, 30.0, &[]); + let b = assess(hba, &m, Some(&scaled(&m, 0.99)), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::LowMapq0Share)); + let b = assess(hba, &Means::new(), Some(&Means::new()), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::NoData)); + } + + #[test] + fn a_control_with_no_read_is_a_broken_control() { + let cat = catalog(); + let hba = &cat.loci["HBA"]; + let m = means(hba, 30.0, &[("flank_left", 0.0)]); + let e = assess(hba, &m, Some(&m), Sex::Unknown).unwrap_err(); + assert!( + e.to_string() + .starts_with("BROKEN_CONTROL: control segment flank_left of HBA"), + "{e}" + ); + } + + #[test] + fn ties_round_to_even() { + // An HBA estimate of exactly 2.5 lies 0.5 from both counts: + // between_counts whatever the rounding; with round:0.5 it would call + // 2 (half to even), never 3. + assert_eq!(round_i(2.5), 2); + assert_eq!(round_i(3.5), 4); + assert_eq!(round_i(0.5), 0); + } + + #[test] + fn chrx_and_chry_follow_the_sex() { + let cat = catalog(); + let az = cat + .loci + .values() + .find(|l| l.chrom == "chrY" && l.is_interval_row()) + .unwrap(); + let m = means(az, 15.0, &[]); + let b = assess(az, &m, None, Sex::Female).unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotApplicable, Some(Reason::NoY)) + ); + let b = assess(az, &m, None, Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::SexUnknown)); + let b = assess(az, &m, None, Sex::Male).unwrap(); + let units = if az.target_depth == "sum" { + az.targets.len() as i64 + } else { + 1 + }; + assert_eq!(b.expected, Some(units)); + } + + #[test] + fn lpa_class_and_interval() { + let cat = catalog(); + let lpa = &cat.loci["LPA_KIV2"]; + // N + 2 = 14 x M / D: M = 30 / 14 x (N + 2) at D = 30. + let m = means(lpa, 30.0, &[]); + let kiv: Vec<(String, f64)> = lpa + .targets + .iter() + .map(|s| (s.name.clone(), 30.0 / 14.0 * 22.0)) + .collect(); + let mut m2 = m.clone(); + m2.extend(kiv); + let b = assess(lpa, &m2, Some(&scaled(&m2, 0.1)), Sex::Unknown).unwrap(); + assert_eq!(b.class, Some(Class::Low)); + let (lo, hi) = b.ci95.unwrap(); + assert!((b.estimate.unwrap() - 20.0).abs() < 1e-9 && lo <= 20.0 && hi >= 20.0); + } +} diff --git a/rust/crates/og-cn/src/input.rs b/rust/crates/og-cn/src/input.rs new file mode 100644 index 000000000..12d56a4a5 --- /dev/null +++ b/rust/crates/og-cn/src/input.rs @@ -0,0 +1,272 @@ +//! The tables `og cn` reads: `samtools bedcov` sums, `samtools mpileup` +//! bases columns and a BAM's header. + +use crate::catalog::Locus; +use crate::engine::{Means, SiteCounts}; +use crate::{Error, each_line, py_float, py_int, strip}; +use og_filters::py_repr; +use std::collections::{HashMap, HashSet}; + +/// One `samtools bedcov` line: chrom, start, end, name, depth sum. +#[derive(Debug, Clone, PartialEq)] +pub struct BedcovLine { + pub chrom: String, + pub start: i64, + pub end: i64, + pub name: String, + pub sum: f64, +} + +/// Every line of a `samtools bedcov` table (blank and `#` lines skipped). +pub fn read_bedcov_all(path: &str) -> Result, Error> { + let mut out = Vec::new(); + each_line(path, |line| { + if strip(line).is_empty() || line.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + return Err(Error(format!( + "{path}: expected chrom, start, end, name, depth sum; got {}", + py_repr(strip(line)) + ))); + } + out.push(BedcovLine { + chrom: c[0].to_owned(), + start: py_int(c[1])?, + end: py_int(c[2])?, + name: c[3].to_owned(), + sum: py_float(c[4])?, + }); + Ok(()) + })?; + Ok(out) +} + +/// `{name: mean depth}` from `samtools bedcov` over `row`'s segments, which +/// must all be there and nothing else; empty for an empty table (no reads +/// on that contig). +pub fn read_bedcov(path: &str, row: &Locus) -> Result { + let mut means = Means::new(); + let mut seen: HashSet<(String, i64, i64, String)> = HashSet::new(); + each_line(path, |line| { + if strip(line).is_empty() || line.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + return Err(Error(format!( + "{path}: expected chrom, start, end, name, depth sum; got {}", + py_repr(strip(line)) + ))); + } + let (chrom, start, end, name) = (c[0], py_int(c[1])?, py_int(c[2])?, c[3]); + if end <= start { + return Err(Error(format!( + "{path}: region {name} {chrom}:{start}-{end} is empty or reversed" + ))); + } + if !seen.insert((chrom.to_owned(), start, end, name.to_owned())) { + return Err(Error(format!( + "{path}: region {name} {chrom}:{start}-{end} appears twice" + ))); + } + means.insert(name.to_owned(), py_float(c[4])? / (end - start) as f64); + Ok(()) + })?; + let want: HashSet<(String, i64, i64, String)> = row + .regions() + .into_iter() + .map(|(c, s, e, n)| (c.to_owned(), s, e, n.to_owned())) + .collect(); + if !seen.is_empty() && seen != want { + return Err(Error(format!( + "{path}: its regions differ from the ones `og cn bed --locus {}` prints", + row.id + ))); + } + Ok(means) +} + +/// The A, C, G and T reads of one `samtools mpileup` bases column. Read +/// starts (`^` and its MAPQ character), read ends, indels (`+N` and `-N` +/// with their N bases), deletions and skips are not bases; `.` and `,` are +/// the reference base when there is one. +pub fn count_bases(bases: &str, reference: &str) -> [u64; 4] { + let mut n = [0u64; 4]; + let refb = match reference.chars().next() { + Some(c) if reference.chars().count() == 1 => c.to_ascii_uppercase(), + _ => 'N', + }; + let chars: Vec = bases.chars().collect(); + let mut i = 0; + while i < chars.len() { + let ch = chars[i]; + if ch == '^' { + i += 2; + continue; + } + if ch == '+' || ch == '-' { + let mut j = i + 1; + while j < chars.len() && chars[j].is_ascii_digit() { + j += 1; + } + let len: usize = chars[i + 1..j] + .iter() + .collect::() + .parse() + .unwrap_or(0); + i = j + len; + continue; + } + let b = if ch == '.' || ch == ',' { + refb + } else { + ch.to_ascii_uppercase() + }; + if let Some(k) = base_index(b) { + n[k] += 1; + } + i += 1; + } + n +} + +fn base_index(b: char) -> Option { + "ACGT".find(b) +} + +/// Streams a `samtools mpileup` table (chrom, pos, ref, depth, bases, ...) +/// and hands each line on `chrom` at a position in `want` to `each` with +/// its base counts. A line that is not an mpileup line is an error. +pub fn each_pileup_site( + path: &str, + chrom: &str, + want: &dyn Fn(i64) -> bool, + mut each: impl FnMut(i64, [u64; 4]), +) -> Result<(), Error> { + each_line(path, |line| { + if strip(line).is_empty() { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 || c[1].is_empty() || !c[1].bytes().all(|b| b.is_ascii_digit()) { + let shown: String = strip(line).chars().take(80).collect(); + return Err(Error(format!( + "{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); got {}", + py_repr(&shown) + ))); + } + let pos = py_int(c[1])?; + if c[0] == chrom && want(pos) { + each(pos, count_bases(c[4], c[2])); + } + Ok(()) + }) +} + +/// `{site: (gene-base reads, pseudogene-base reads, other reads)}` of a +/// `call psv` row from `samtools mpileup` over `og cn bed --psv`, each +/// site's two positions pooled: a read that carries the gene's base counts +/// for the gene wherever the aligner put it. A position missing from the +/// pileup has no read. +pub fn read_psv_pileup(path: &str, row: &Locus) -> Result, Error> { + let mut at: HashMap = HashMap::new(); + for (i, s) in row.psv.iter().enumerate() { + at.insert(s.gene_pos + 1, i); + at.insert(s.pseudo_pos + 1, i); + } + let mut counts = vec![[0u64; 4]; row.psv.len()]; + each_pileup_site(path, &row.chrom, &|p| at.contains_key(&p), |pos, n| { + let k = at[&pos]; + for (a, b) in counts[k].iter_mut().zip(n) { + *a += b; + } + })?; + Ok(row + .psv + .iter() + .zip(counts) + .map(|(s, n)| { + let pick = |bases: &[u8]| -> u64 { + bases + .iter() + .filter_map(|b| base_index(*b as char)) + .map(|k| n[k]) + .sum() + }; + let (g, p) = (pick(&s.gene), pick(&s.pseudo)); + (s.name.clone(), (g, p, n.iter().sum::() - g - p)) + }) + .collect()) +} + +/// The ALT or HLA contigs named by `samtools view -H` lines, or by contig +/// names one per line: a name ending in `_alt` or starting with `HLA-` +/// (the rule of `scripts/validate-setup.sh`). +pub fn alt_contig(line: &str) -> Option<&str> { + let name = if let Some(rest) = line.strip_prefix('@') { + if !rest.starts_with("SQ") { + return None; + } + line.split('\t') + .find_map(|f| f.strip_prefix("SN:")) + .unwrap_or("") + } else { + strip(line.split('\t').next().unwrap_or("")) + }; + (name.ends_with("_alt") || name.starts_with("HLA-")).then_some(name) +} + +/// The refusal line for a BAM header at `path` (`-` reads stdin) that names +/// an ALT or HLA contig, or `None`. +pub fn check_bam(path: &str) -> Result, Error> { + let mut found = 0usize; + let mut first = String::new(); + let mut each = |line: &str| { + if let Some(name) = alt_contig(line) { + if found == 0 { + first = name.to_owned(); + } + found += 1; + } + Ok(()) + }; + if path == "-" { + crate::each_line_of(std::io::stdin().lock(), "", &mut each)?; + } else { + each_line(path, &mut each)?; + } + Ok((found > 0).then(|| { + format!( + "REFUSED: the BAM was aligned to a reference with {found} ALT or HLA contigs (first: {first}). Reads that fit a primary locus and its ALT copy equally get MAPQ 0 or land on the ALT copy, so depth-based copy number reads deletions that are not there. Realign to the default no-ALT reference (docs/realignment.md)." + ) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bases_skip_starts_ends_and_indels() { + // ^] read start with its MAPQ; C . , c count; $ ends; +2AG and -1t + // are indels; * < > N are not bases. + assert_eq!(count_bases("^]C.,c$+2AGg-1tG*<>N", "C"), [0, 4, 2, 0]); + assert_eq!(count_bases(".,", "N"), [0, 0, 0, 0]); + } + + #[test] + fn alt_and_hla_contigs() { + assert_eq!( + alt_contig("@SQ\tSN:chr1_KI270762v1_alt\tLN:1"), + Some("chr1_KI270762v1_alt") + ); + assert_eq!( + alt_contig("HLA-A*01:01:01:01\t3503"), + Some("HLA-A*01:01:01:01") + ); + assert_eq!(alt_contig("@SQ\tSN:chr1\tLN:248956422"), None); + assert_eq!(alt_contig("@HD\tVN:1.6\tSN:x_alt"), None); + } +} diff --git a/rust/crates/og-cn/src/layout.rs b/rust/crates/og-cn/src/layout.rs new file mode 100644 index 000000000..cfc683669 --- /dev/null +++ b/rust/crates/og-cn/src/layout.rs @@ -0,0 +1,385 @@ +//! Each step's `metricvalue` table and stdout line for a block, and +//! the line of the interval table. The HBA and LPA_KIV2 layouts are those +//! `bin/hba_copy_number.py` and `bin/lpa_kiv2.py` wrote; a `call psv` row +//! gets the PSV layout and every other row the generic one. +//! +//! Each layout matches on the block's status and reason exhaustively. The +//! HBA layout wrote `status called` with no count for a reason it did not +//! name (a catalog edit could give `high_mapq0_share` or `below_floor`), and +//! the LPA layout failed on one; both now say `not assessed: `. + +use crate::catalog::Locus; +use crate::engine::{Block, Reason, Status}; +use crate::{d, f2, fixed}; + +/// The rows of a step table and its one stdout line. +pub type Table = (Vec<(String, String)>, String); + +/// How a row's table is laid out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Layout { + Generic, + Hba, + LpaKiv2, + Psv, +} + +impl Layout { + /// The layout of `row`. + pub fn of(row: &Locus) -> Layout { + if !row.psv.is_empty() { + Layout::Psv + } else { + match row.id.as_str() { + "HBA" => Layout::Hba, + "LPA_KIV2" => Layout::LpaKiv2, + _ => Layout::Generic, + } + } + } +} + +/// The step table of `row` for `b`. +pub fn table(row: &Locus, b: &Block) -> Table { + match Layout::of(row) { + Layout::Psv => psv(row, b), + Layout::Hba => hba(row, b), + Layout::LpaKiv2 => lpa(row, b), + Layout::Generic => generic(row, b), + } +} + +fn kv(rows: Vec<(&str, String)>) -> Vec<(String, String)> { + rows.into_iter().map(|(k, v)| (k.to_owned(), v)).collect() +} + +fn reason(b: &Block) -> String { + b.reason.map_or_else(|| ".".to_owned(), |r| r.to_string()) +} + +fn or_dot(s: &str) -> String { + if s.is_empty() { ".".into() } else { s.into() } +} + +fn generic(row: &Locus, b: &Block) -> Table { + let rows = kv(vec![ + ("status", b.status.to_string()), + ("copies", d(b.copies)), + ("estimate", f2(b.estimate)), + ("target_depth", f2(b.target_depth)), + ("control_depth", f2(b.control_depth)), + ("reason", reason(b)), + ("reading", or_dot(&b.reading)), + ]); + let why = b + .reason + .map_or_else(|| b.reading.clone(), |r| r.to_string()); + (rows, format!("{}: {} ({why})", row.id, b.status)) +} + +/// `""` for a reason a layout has no words for. +fn bare(b: &Block) -> String { + format!("not assessed: {}", reason(b)) +} + +fn hba(row: &Locus, b: &Block) -> Table { + const ORDER: [&str; 9] = [ + "status", + "alpha_globin_genes", + "estimate", + "hba2_depth", + "hba1_depth", + "flank_left_depth", + "flank_right_depth", + "mapq0_share", + "message", + ]; + if b.reason == Some(Reason::NoData) { + let msg = "not assessed: the BAM has no chr16 reads, so HBA1 and HBA2 have none"; + let rows = ORDER + .iter() + .map(|k| { + let v = match *k { + "status" => "not assessed", + "message" => msg, + _ => ".", + }; + ((*k).to_owned(), v.to_owned()) + }) + .collect(); + return (rows, format!("HBA1/HBA2: {msg}")); + } + let depth = |n: &str| b.depths.get(n).copied(); + let r2 = |x: Option| { + x.map_or_else( + || ".".to_owned(), + |v| pycompat::fmt::repr(pycompat::fmt::round(v, 2).unwrap_or(v)), + ) + }; + let (left, right) = (depth("flank_left"), depth("flank_right")); + let one = |x: Option| x.map_or_else(|| ".".to_owned(), |v| fixed(v, 1)); + let share = b.mapq0_share; + let est = b.estimate; + let (status, genes, message) = match (b.status, b.reason) { + (Status::Called, _) => ("called", d(b.copies), b.reading.clone()), + (_, Some(Reason::LowControlDepth)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: flank depth under {}x ({}x and {}x), too low to compare", + pycompat::fmt::general(row.depth().min_control, 6), + one(left), + one(right) + ), + ), + (_, Some(Reason::ControlsDisagree)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: the two flanks disagree ({}x and {}x); a deletion larger than --SEA, or uneven coverage, would skew the count", + one(left), + one(right) + ), + ), + (_, Some(Reason::LowMapq0Share)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: reads of mapping quality 0 make up {}% of the depth over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM", + one(share.map(|s| 100.0 * s)) + ), + ), + (_, Some(Reason::BetweenCounts)) => { + let e = est.expect("an estimate between counts"); + let n = e.trunc() as i64; + ( + "not assessed", + ".".into(), + format!( + "not assessed: the estimate {} lies between {n} and {} alpha-globin genes", + fixed(e, 2), + n + 1 + ), + ) + } + _ => ("not assessed", ".".into(), bare(b)), + }; + let values = [ + status.to_owned(), + genes, + f2(est), + r2(depth("HBA2")), + r2(depth("HBA1")), + r2(left), + r2(right), + share.map_or_else(|| ".".to_owned(), |s| fixed(s, 3)), + message, + ]; + let line = format!( + "HBA1/HBA2: {} (estimate {}; HBA2 {}x, HBA1 {}x, flanks {}x and {}x; MAPQ 0 share over the genes {})", + values[8], values[2], values[3], values[4], values[5], values[6], values[7] + ); + let rows = ORDER + .iter() + .zip(values) + .map(|(k, v)| ((*k).to_owned(), v)) + .collect(); + (rows, line) +} + +/// The advice line of step 46's table. +pub const LPA_ADVICE: &str = "A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 European Atherosclerosis Society consensus advises measuring it at least once in every adult."; + +fn lpa(row: &Locus, b: &Block) -> Table { + let mut status = "not assessed".to_owned(); + let (mut copies, mut lo, mut hi, mut class) = ( + ".".to_owned(), + ".".to_owned(), + ".".to_owned(), + ".".to_owned(), + ); + let (mut kd, mut fd, mut sh) = (".".to_owned(), ".".to_owned(), ".".to_owned()); + let message; + if b.reason == Some(Reason::NoData) { + message = "not assessed: the BAM has no chr6, so LPA has no reads".to_owned(); + } else { + let k = b.target_depth; + let f = b.control_depth; + let one = |x: Option| x.map_or_else(|| ".".to_owned(), |v| fixed(v, 1)); + kd = f2(k); + fd = f2(f); + sh = b + .mapq0_share + .map_or_else(|| ".".to_owned(), |s| fixed(s, 3)); + message = match (b.status, b.reason) { + (Status::Called, _) => match (b.estimate, b.ci95) { + (Some(est), Some((l, h))) => { + status = "estimated".into(); + copies = fixed(est, 1); + lo = fixed(l, 1); + hi = fixed(h, 1); + class = b.class.map_or(".", |c| c.as_str()).to_owned(); + format!( + "about {} KIV-2 repeats in total (95% interval {lo} to {hi}): {}", + fixed(est, 0), + b.reading + ) + } + _ => bare(b), + }, + (_, Some(Reason::LowControlDepth)) => { + status = "too little depth".into(); + format!( + "too little depth: {}x over the single-copy LPA exons, under the {}x the count needs", + one(f), + pycompat::fmt::general(row.depth().min_control, 6) + ) + } + (_, Some(Reason::LowMapq0Share)) => format!( + "not assessed: reads of mapping quality 0 make up {}% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM", + one(b.mapq0_share.map(|s| 100.0 * s)) + ), + (_, Some(Reason::BelowFloor)) => format!( + "not assessed: the KIV-2 exons have {}x against {}x over the single-copy exons, too little for any repeat count (a deletion or a mapping problem)", + one(k), + one(f) + ), + _ => bare(b), + }; + } + let line = format!( + "LPA KIV-2: {message} (KIV-2 exons {kd}x, single-copy exons {fd}x, MAPQ 0 share {sh})" + ); + let rows = kv(vec![ + ("status", status), + ("kiv2_copies", copies), + ("ci95_low", lo), + ("ci95_high", hi), + ("class", class), + ("kiv2_depth", kd), + ("flank_depth", fd), + ("mapq0_share", sh), + ("message", message), + ("advice", LPA_ADVICE.to_owned()), + ]); + (rows, line) +} + +/// `.` for a called PSV row, else `not assessed: ` and why; a low MAPQ 0 +/// share is spelled out, because the BAM, not the genome, stopped the call. +fn psv_message(row: &Locus, b: &Block) -> String { + if b.status == Status::Called { + return ".".into(); + } + if b.reason == Some(Reason::LowMapq0Share) { + let targets: Vec<&str> = row + .targets + .iter() + .map(|s| s.name.strip_suffix("_BODY").unwrap_or(&s.name)) + .collect(); + let share = b.mapq0_share.unwrap_or(f64::NAN); + let min = row.depth().mapq0_min.unwrap_or(f64::NAN); + return format!( + "not assessed: the BAM has few or no multi-mapping reads here (reads of mapping quality 0 make up {}% of the depth over {}, under {}%), a possible sign of an aligner that leaves out the reads that fit both copies (as novoalign does by default); the module total may then come out low and read as a deletion; realign with minimap2 (step 02) or BWA-MEM", + fixed(100.0 * share, 1), + targets.join(" and "), + pycompat::fmt::general(100.0 * min, 6) + ); + } + format!("not assessed: {}", reason(b)) +} + +fn psv(row: &Locus, b: &Block) -> Table { + let mut rows = kv(vec![ + ("status", b.status.to_string()), + ("cyp21a2_copies", d(b.copies)), + ("module_total", d(b.module_total)), + ("expected_total", d(b.expected)), + ("estimate", f2(b.estimate)), + ("finding", b.finding.map_or(".", |f| f.as_str()).to_owned()), + ("conversion_sites", or_dot(&b.conversion_sites.join(","))), + ]); + for s in &b.sites { + rows.push(( + format!("psv_{}", s.name), + format!( + "{} gene, {} pseudogene, {} other reads; copies {} ({}); ratio {}", + s.gene_reads, + s.pseudo_reads, + s.other_reads, + d(s.copies), + f2(s.estimate), + f2(s.ratio) + ), + )); + } + rows.extend(kv(vec![ + ("target_depth", f2(b.target_depth)), + ("control_depth", f2(b.control_depth)), + ("reason", reason(b)), + ("reading", or_dot(&b.reading)), + ( + "mapq0_share", + b.mapq0_share + .map_or_else(|| ".".to_owned(), |s| fixed(s, 3)), + ), + ("message", psv_message(row, b)), + ])); + let why = match (b.reason, b.finding) { + (Some(r), _) => r.to_string(), + (None, Some(f)) => f.as_str().to_owned(), + (None, None) => "None".to_owned(), + }; + let line = format!( + "{}: {} ({why}; {} gene copies, module total {} of {})", + row.id, + b.status, + d(b.copies), + d(b.module_total), + d(b.expected) + ); + (rows, line) +} + +/// The interval table's columns. +pub const INTERVAL_COLUMNS: [&str; 11] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", + "reason", +]; + +/// One line of the interval table. +pub fn interval_line(row: &Locus, b: &Block) -> Vec { + let tg = &row.targets; + let change = match (b.copies, b.expected) { + (Some(c), Some(e)) if c < e => "deletion", + (Some(c), Some(e)) if c > e => "duplication", + (Some(_), Some(_)) => "no change", + _ => ".", + }; + let name = if tg[0].label.is_empty() { + row.id.clone() + } else { + tg[0].label.clone() + }; + vec![ + row.id.clone(), + name, + row.chrom.clone(), + tg.iter() + .map(|s| s.start) + .min() + .expect("a target") + .to_string(), + tg.iter() + .map(|s| s.end) + .max() + .expect("a target") + .to_string(), + b.status.to_string(), + d(b.copies), + d(b.expected), + change.to_owned(), + f2(b.estimate), + reason(b), + ] +} diff --git a/rust/crates/og-cn/src/lib.rs b/rust/crates/og-cn/src/lib.rs new file mode 100644 index 000000000..8c122d4c3 --- /dev/null +++ b/rust/crates/og-cn/src/lib.rs @@ -0,0 +1,164 @@ +//! Copy number from short-read depth (replaces `bin/hba_copy_number.py`, +//! `bin/lpa_kiv2.py`, the commands of `bin/copy_number.py` and the +//! copy-aware markers of `bin/traits_report.py`). +//! +//! One method, one catalog row per locus (`bin/cn_loci.tsv`, +//! `docs/cn-loci.md`): the depth over the row's target segments against the +//! depth over single-copy control segments, as `samtools bedcov` sums it, +//! +//! ```text +//! estimate = scale x T / C - offset +//! ``` +//! +//! guarded by a minimum control depth, flank agreement and the share of the +//! target depth from reads of mapping quality 0, then called by the row's +//! `call` rule. +//! +//! - [`catalog`]: the catalog, read and checked once per process. +//! - [`engine`]: the guards and the call ([`engine::assess`]), and the PSV +//! sites of a `call psv` row ([`engine::psv_assess`]). +//! - [`input`]: `samtools bedcov` and `samtools mpileup` tables, BAM headers. +//! - [`layout`]: each step's `metricvalue` table and the interval table. +//! - [`markers`]: the copy-aware markers (LPA KIV-2 4925G>A, SMN1 +//! g.27134T>G) read from the pileup at every GRCh38 copy of the site. +//! - [`smn`]: the one rule that reads the SMN1 psCN at exon 7 from step 35's +//! SMN table. +//! - [`cli`]: `og cn bed|call|intervals|check-bam|marker-regions|marker-read`. +//! +//! Each command writes what the Python wrote, byte for byte (`tests/parity`), +//! except where it fixes it on purpose; the module that does says where. + +pub mod catalog; +pub mod cli; +pub mod engine; +pub mod input; +pub mod layout; +pub mod markers; +pub mod smn; + +use std::fmt; +use std::io; + +/// A problem the command reports as `ERROR: ` with its exit code. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub String); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Error {} + +impl Error { + /// An I/O error on `path`, worded as Python's `OSError`. + pub fn io(e: &io::Error, path: &str) -> Error { + if e.kind() == io::ErrorKind::InvalidData { + Error(format!("{path}: {e}")) + } else { + Error(og_filters::os_error(e, path)) + } + } +} + +/// `int(s)` with Python's message when it is not one. +pub(crate) fn py_int(s: &str) -> Result { + pycompat::num::parse_int(s) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| { + Error(format!( + "invalid literal for int() with base 10: {}", + og_filters::py_repr(s) + )) + }) +} + +/// `float(s)` with Python's message when it is not one. +pub(crate) fn py_float(s: &str) -> Result { + pycompat::num::parse_float(s).map_err(|_| { + Error(format!( + "could not convert string to float: {}", + og_filters::py_repr(s) + )) + }) +} + +/// `f"{x:.{prec}f}"`. +pub(crate) fn fixed(x: f64, prec: usize) -> String { + pycompat::fmt::fixed(x, prec) +} + +/// `"." if x is None else f"{x:.2f}"`. +pub(crate) fn f2(x: Option) -> String { + x.map_or_else(|| ".".to_owned(), |v| fixed(v, 2)) +} + +/// `"." if x is None else str(x)`. +pub(crate) fn d(x: Option) -> String { + x.map_or_else(|| ".".to_owned(), |v| v.to_string()) +} + +/// `round(x)` of a finite float, as an integer (half to even). +pub(crate) fn round_i(x: f64) -> i64 { + x.round_ties_even() as i64 +} + +/// The text lines of a file as Python's `open(path, errors="replace")` +/// reads them (universal newlines), each without its `\n`. +pub(crate) fn read_lines(path: &str) -> Result, Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + lines_of(io::BufReader::new(file), path) +} + +/// The lines of the file at `path`, as Python's `open(path)` reads them: +/// universal newlines, strict UTF-8. +pub(crate) fn read_lines_strict(path: &str) -> Result, Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + og_io::Lines::new(io::BufReader::new(file), og_io::Utf8::Strict) + .map(|l| l.map_err(|e| Error::io(&e, path))) + .collect() +} + +/// The lines of `r`, as [`read_lines`]. +pub(crate) fn lines_of(r: impl io::BufRead, path: &str) -> Result, Error> { + let mut out = Vec::new(); + each_line_of(r, path, |l| { + out.push(l.to_owned()); + Ok(()) + })?; + Ok(out) +} + +/// Streams the lines of `r` (as [`read_lines`] reads them) through `each`, +/// one at a time, so a long pileup or header is never held whole. +pub(crate) fn each_line_of( + r: impl io::BufRead, + path: &str, + mut each: impl FnMut(&str) -> Result<(), Error>, +) -> Result<(), Error> { + let mut lines = og_io::Lines::new(r, og_io::Utf8::Replace); + let mut line = String::new(); + while lines + .read_line(&mut line) + .map_err(|e| Error::io(&e, path))? + { + each(line.strip_suffix('\n').unwrap_or(&line))?; + } + Ok(()) +} + +/// [`each_line_of`] over the file at `path`. +pub(crate) fn each_line( + path: &str, + each: impl FnMut(&str) -> Result<(), Error>, +) -> Result<(), Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + each_line_of(io::BufReader::new(file), path, each) +} + +/// Python's `str.strip()`. +pub(crate) fn strip(s: &str) -> &str { + pycompat::text::strip(s, None) +} diff --git a/rust/crates/og-cn/src/markers.rs b/rust/crates/og-cn/src/markers.rs new file mode 100644 index 000000000..851a27c56 --- /dev/null +++ b/rust/crates/og-cn/src/markers.rs @@ -0,0 +1,451 @@ +//! Copy-aware markers: two well-sourced markers sit in sequence the genome +//! holds more than once, so a gVCF genotype at one position cannot read +//! them. Each is read from the BAM by the step that counts its copies: the +//! reads at every GRCh38 copy of the site are added up, and the share that +//! carries the marker's base is a fraction of copies, not a diploid +//! genotype. +//! +//! - `LPA_KIV2_4925`: the LPA KIV-2 4925G>A splice variant (Coassin et al. +//! 2017, PMID 28444229), read by step 46 beside the KIV-2 count. +//! - `SMN1_27134`: SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID +//! 23788250), read by step 35 beside the SMN1 psCN. +//! +//! The values are those `bin/traits_report.py` held (MARKERS), each copy +//! checked against the 21-base context and the dbSNP record by +//! `marker_errors` (a unit test here, and `tests/test_traits_report.py` +//! compares the records with what collect_summary reads). + +use crate::input::each_pileup_site; +use crate::{Error, fixed}; + +/// One copy-aware marker. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Marker { + pub key: &'static str, + pub name: &'static str, + pub chrom: &'static str, + /// The plus-strand base read at each copy, and the marker's base. + pub reference: u8, + pub alt: u8, + /// dbSNP b157's record of one copy: rsID, chrom, POS, REF, ALT. + pub dbsnp: (&'static str, &'static str, i64, &'static str, &'static str), + /// Every GRCh38 copy of the site, 1-based. + pub copies: &'static [i64], + /// The GRCh38 sequence of 10 bases on each side of every copy. + pub context: &'static str, + /// Fewer reads over all copies together: too little depth. + pub min_depth: u64, + /// Reads with the marker's base for 'present' ... + pub min_alt: u64, + /// ... and a share of the reads at least this. + pub min_fraction: f64, + /// This many reads with the marker's base or fewer: 'not seen'. + pub max_absent: u64, +} + +/// The LPA KIV-2 4925G>A splice variant: the last base of the second KIV-2 +/// exon, G>A on LPA's minus strand, so C>T on the plus strand; one copy at +/// the end of each of the six KIV-2 units of GRCh38 and one at the +/// near-identical last exon of KIV-1. dbSNP names the first KIV-2 copy. +pub const LPA_KIV2_4925: Marker = Marker { + key: "LPA_KIV2_4925", + name: "LPA KIV-2 4925G>A", + chrom: "chr6", + reference: b'C', + alt: b'T', + dbsnp: ("rs1484919544", "chr6", 160_640_667, "C", "T"), + copies: &[ + 160_646_214, + 160_640_667, + 160_635_123, + 160_629_574, + 160_624_030, + 160_618_484, + 160_612_932, + ], + context: "GACTCCTTACCTTGTTCGGAA", + min_depth: 100, + min_alt: 5, + min_fraction: 0.008, + max_absent: 2, +}; + +/// SMN1 g.27134T>G (NM_000344.4:c.*3+80T>G, intron 7, plus strand); SMN2 +/// holds the same 21 bases 875,420 bases upstream. +pub const SMN1_27134: Marker = Marker { + key: "SMN1_27134", + name: "SMN1 g.27134T>G", + chrom: "chr5", + reference: b'T', + alt: b'G', + dbsnp: ("rs143838139", "chr5", 70_952_074, "T", "G"), + copies: &[70_952_074, 70_076_654], + context: "TTTTTGAACATTTAAAAAGTT", + min_depth: 10, + min_alt: 3, + min_fraction: 0.05, + max_absent: 1, +}; + +/// Every marker, by key. +pub const MARKERS: [Marker; 2] = [LPA_KIV2_4925, SMN1_27134]; + +/// The marker named `key`. +pub fn marker(key: &str) -> Option<&'static Marker> { + MARKERS.iter().find(|m| m.key == key) +} + +/// A marker's reading. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Present, + NotSeen, + Uncertain, + TooLittleDepth, + NotAssessed, +} + +impl Status { + pub fn as_str(self) -> &'static str { + match self { + Status::Present => "present", + Status::NotSeen => "not seen", + Status::Uncertain => "uncertain", + Status::TooLittleDepth => "too little depth", + Status::NotAssessed => "not assessed", + } + } +} + +/// Reads with the reference base, with the marker's base, and others. +pub type Counts = (u64, u64, u64); + +/// A marker's reading and the numbers behind it, as the table prints them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reading { + pub status: Status, + pub alt_reads: String, + pub depth: String, + /// `alt / depth` at 4 decimals, or `.`. + pub fraction: String, +} + +/// The copy sites as samtools regions, `chrom:pos-pos`, in the marker's order. +pub fn regions(m: &Marker) -> Vec { + m.copies + .iter() + .map(|p| format!("{}:{p}-{p}", m.chrom)) + .collect() +} + +/// The reads over every copy of the marker from `samtools mpileup` lines +/// at its sites, without a reference: a read that carries the marker's base +/// counts wherever the aligner put it. +pub fn read_pileup(path: &str, m: &Marker) -> Result { + let mut n = [0u64; 4]; + each_pileup_site(path, m.chrom, &|p| m.copies.contains(&p), |_, c| { + for (a, b) in n.iter_mut().zip(c) { + *a += b; + } + })?; + let at = |b: u8| n["ACGT".find(b as char).expect("a base")]; + let (r, a) = (at(m.reference), at(m.alt)); + Ok((r, a, n.iter().sum::() - r - a)) +} + +/// The marker's reading from its counts (`None`: the pileup could not be +/// read). The depth counts the reads with the reference or the marker's base. +pub fn assess(m: &Marker, counts: Option) -> Reading { + let Some((r, a, _)) = counts else { + return Reading { + status: Status::NotAssessed, + alt_reads: ".".into(), + depth: ".".into(), + fraction: ".".into(), + }; + }; + let depth = r + a; + let frac = (depth > 0).then(|| a as f64 / depth as f64); + let status = if depth < m.min_depth { + Status::TooLittleDepth + } else if a >= m.min_alt && frac.is_some_and(|f| f >= m.min_fraction) { + Status::Present + } else if a <= m.max_absent { + Status::NotSeen + } else { + Status::Uncertain + }; + Reading { + status, + alt_reads: a.to_string(), + depth: depth.to_string(), + fraction: frac.map_or_else(|| ".".into(), |f| fixed(f, 4)), + } +} + +/// The printed fraction times `copies`, at one decimal (the reading of a +/// present or uncertain marker; `.` otherwise). +fn copies_of(a: &Reading, copies: Option) -> String { + match (a.status, copies, pycompat::num::parse_float(&a.fraction)) { + (Status::Present | Status::Uncertain, Some(c), Ok(f)) => fixed(f * c, 1), + _ => ".".into(), + } +} + +const NULL_LIMIT: &str = "A short-read fraction cannot say which chromosome or which repeat carries it, so it cannot say whether it sits on the small or the large isoform."; + +/// The KIV-2 4925G>A table: the reading beside step 46's count. +/// `kiv2_copies` is the count as its table prints it (`.` without one); +/// `why_not` is the reason the reads cannot be trusted (the aligner left +/// out the reads of MAPQ 0), which makes it not assessed whatever the counts. +pub fn null_allele( + counts: Option, + kiv2_copies: &str, + why_not: Option<&str>, +) -> Vec<(String, String)> { + let m = &LPA_KIV2_4925; + let a = assess(m, if why_not.is_some() { None } else { counts }); + let total = pycompat::num::parse_float(kiv2_copies) + .ok() + .map(|n| n + 2.0); + let copies = copies_of(&a, total); + let name = format!( + "{} (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017)", + m.name + ); + let st = a.status.as_str(); + let msg = if a.status == Status::NotAssessed { + format!( + "{name}: not assessed, {}", + why_not.unwrap_or("the reads at its seven GRCh38 copies could not be read.") + ) + } else { + let mut s = format!( + "{name}: {st}, {} of {} reads at its seven GRCh38 copies carry it", + a.alt_reads, a.depth + ); + if copies != "." { + let t = total.expect("copies need the count"); + s += &format!(" (about {copies} of the {} copies counted)", fixed(t, 0)); + } + if a.status == Status::TooLittleDepth { + s += &format!(", under the {} reads a reading needs", m.min_depth); + } + s.push('.'); + match a.status { + Status::Present => { + s += " Carriers had a lower Lp(a) than non-carriers with the same number of repeats, most of all with few repeats. "; + s += NULL_LIMIT; + } + Status::NotSeen => { + s += " Not seen in the reads is not proof of absence: one carrying repeat among many gives few reads, and a short-read fraction never says which chromosome or repeat holds a variant." + } + _ => { + s.push(' '); + s += NULL_LIMIT; + } + } + s + }; + rows(&[ + ("marker", m.name.to_owned()), + ("rsid", format!("{} (first KIV-2 copy)", m.dbsnp.0)), + ("status", st.to_owned()), + ("alt_reads", a.alt_reads), + ("depth", a.depth), + ("fraction", a.fraction), + ("copies", copies), + ("kiv2_copies", kiv2_copies.to_owned()), + ("message", msg), + ]) +} + +/// The reason a KIV-2 4925G>A reading cannot be trusted when reads of +/// MAPQ 0 make up less than `floor` of the KIV-2 depth: the reads that fit +/// several copies are the ones a carrying repeat gives. +pub fn mapq0_why_not(share: f64, floor: f64) -> Option { + (share < floor).then(|| { + format!( + "reads of mapping quality 0 make up {}% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives.", + fixed(100.0 * share, 1) + ) + }) +} + +/// SMN1's marker never proves 2+0, and its absence never rules it out. +pub const SMN1_NOT_PROOF: &str = + "This marker does not prove 2+0, and its absence does not rule 2+0 out."; + +/// Step 35's SMN1 marker table: the marker's reading beside the SMN1 psCN +/// of the SMN table (`smn`, read by [`crate::smn::read`]). +pub fn smn1_marker(counts: Option, smn: &crate::smn::SmnRead) -> Vec<(String, String)> { + let m = &SMN1_27134; + let a = assess(m, counts); + let copies = copies_of(&a, smn.agcn.map(|v| v as f64)); + let st = a.status.as_str(); + let head = if a.status == Status::NotAssessed { + format!( + "{} ({}): not assessed, the reads at the marker could not be read.", + m.name, m.dbsnp.0 + ) + } else { + let mut s = format!( + "{} ({}): {st}, {} of {} reads carry G at the SMN1 and SMN2 copies of the site together", + m.name, m.dbsnp.0, a.alt_reads, a.depth + ); + if copies != "." { + s += &format!( + " (about {copies} of the {} SMN1 and SMN2 copies)", + smn.agcn.expect("copies need the agCN") + ); + } + if a.status == Status::TooLittleDepth { + s += &format!(", under the {} reads a reading needs", m.min_depth); + } + s + ". The reads cannot tell whether a G sits on SMN1 or SMN2." + }; + let body = match smn.pscn { + Some(2) => match a.status { + Status::Present => " With two SMN1 copies, this marker goes with a higher chance that both sit on one chromosome (2+0, a silent carrier): in Luo 2014 it raised the residual carrier risk of a two-copy result (in African Americans from about 1 in 121 to 1 in 34); the figures depend on ancestry.".to_owned(), + Status::NotSeen => " With two SMN1 copies and no marker, the chance of 2+0 is lower than for a two-copy result alone (in Luo 2014, in African Americans, about 1 in 396 against 1 in 121), not zero; the figures depend on ancestry.".to_owned(), + _ => " With two SMN1 copies the marker would refine the chance of 2+0; here it gives no reading.".to_owned(), + }, + Some(ps) => format!( + " SMN1 has {ps} cop{} here; the marker refines the carrier risk of a two-copy result only.", + if ps == 1 { "y" } else { "ies" } + ), + None => format!( + " SMN1 copy number: {}; the marker refines the carrier risk of a two-copy result only.", + smn.text + ), + }; + rows(&[ + ("marker", m.name.to_owned()), + ("rsid", m.dbsnp.0.to_owned()), + ("status", st.to_owned()), + ("alt_reads", a.alt_reads), + ("depth", a.depth), + ("fraction", a.fraction), + ("copies", copies), + ("smn1_pscn", smn.text.clone()), + ("agcn", crate::d(smn.agcn)), + ("message", format!("{head}{body} {SMN1_NOT_PROOF}")), + ]) +} + +fn rows(r: &[(&str, String)]) -> Vec<(String, String)> { + r.iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() +} + +/// Problems of one marker (empty when it is consistent): the dbSNP record +/// is one of its copies with its ref and alt, on the plus strand and not +/// swapped; the context's middle base is the ref; no copy is listed twice. +pub fn marker_errors(m: &Marker) -> Vec { + let (rs, chrom, pos, r, a) = m.dbsnp; + let comp = |b: &str| match b { + "A" => "T", + "C" => "G", + "G" => "C", + "T" => "A", + _ => "", + }; + let (mr, ma) = ( + (m.reference as char).to_string(), + (m.alt as char).to_string(), + ); + let mut errs = Vec::new(); + if chrom != m.chrom || !m.copies.contains(&pos) { + errs.push(format!( + "{}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies", + m.key + )); + } + if (mr.as_str(), ma.as_str()) == (a, r) { + errs.push(format!( + "{}: ref and alt are swapped (marker {mr}>{ma}, dbSNP {r}>{a})", + m.key + )); + } else if (mr.as_str(), ma.as_str()) == (comp(r), comp(a)) { + errs.push(format!( + "{}: the alleles are the minus-strand bases (marker {mr}>{ma}, dbSNP {r}>{a}): write the plus-strand bases", + m.key + )); + } else if (mr.as_str(), ma.as_str()) != (r, a) { + errs.push(format!("{}: marker {mr}>{ma}, dbSNP {r}>{a}", m.key)); + } + let ctx = m.context.as_bytes(); + if ctx.len() % 2 != 1 || ctx[ctx.len() / 2] != m.reference { + errs.push(format!( + "{}: the middle base of the context is not the ref {mr}", + m.key + )); + } + let mut seen = std::collections::HashSet::new(); + if !m.copies.iter().all(|c| seen.insert(c)) { + errs.push(format!("{}: a copy is listed twice", m.key)); + } + errs +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn markers_are_consistent() { + for m in &MARKERS { + assert!(marker_errors(m).is_empty(), "{:?}", marker_errors(m)); + } + // A planted swap, a minus-strand pair and a copy listed twice are caught. + let mut bad = SMN1_27134; + bad.reference = b'G'; + bad.alt = b'T'; + assert!(marker_errors(&bad)[0].contains("swapped")); + bad.reference = b'A'; + bad.alt = b'C'; + assert!( + marker_errors(&bad) + .iter() + .any(|e| e.contains("minus-strand")) + ); + bad = LPA_KIV2_4925; + bad.copies = &[160_640_667, 160_640_667]; + assert!( + marker_errors(&bad) + .iter() + .any(|e| e.contains("listed twice")) + ); + } + + #[test] + fn statuses() { + let m = &LPA_KIV2_4925; + assert_eq!(assess(m, Some((95, 5, 0))).status, Status::Present); + assert_eq!(assess(m, Some((98, 2, 0))).status, Status::NotSeen); + assert_eq!(assess(m, Some((996, 4, 0))).status, Status::Uncertain); + assert_eq!(assess(m, Some((50, 5, 0))).status, Status::TooLittleDepth); + assert_eq!(assess(m, None).status, Status::NotAssessed); + assert_eq!(assess(m, Some((95, 5, 0))).fraction, "0.0500"); + } + + #[test] + fn null_allele_copies_and_why_not() { + let t = null_allele(Some((190, 10, 0)), "20.0", None); + assert_eq!(t[6], ("copies".into(), "1.1".into())); + assert!( + t[8].1.contains("(about 1.1 of the 22 copies counted)"), + "{}", + t[8].1 + ); + let why = mapq0_why_not(0.1996, 0.2).unwrap(); + assert!( + why.starts_with("reads of mapping quality 0 make up 20.0%"), + "{why}" + ); + let t = null_allele(Some((190, 10, 0)), "20.0", Some(&why)); + assert_eq!(t[2].1, "not assessed"); + assert!(mapq0_why_not(0.2, 0.2).is_none()); + } +} diff --git a/rust/crates/og-cn/src/smn.rs b/rust/crates/og-cn/src/smn.rs new file mode 100644 index 000000000..9cf353689 --- /dev/null +++ b/rust/crates/og-cn/src/smn.rs @@ -0,0 +1,214 @@ +//! The one rule that reads SMN1's copy number at exon 7 from step 35's SMN +//! table (Parascopy's `res.samples.bed`, as `chrom start end locus +//! agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual +//! homologous_regions`). +//! +//! Rows in file order; each row covers its own region and its homologous +//! regions. The first row with a region that holds exon 7 gives the psCN of +//! that region, and the first row with a region that holds a copy of the +//! SMN1 g.27134T>G site gives the agCN. A copy number counts only with +//! filter PASS, quality 20 or more and a whole number. This is the rule of +//! `paraphase_report.parascopy_smn1`; the SMN1 marker table took the last +//! matching row before, so a table with two rows over exon 7 made the +//! marker and the Paraphase comparison read different rows. + +use crate::Error; + +/// SMN1 exon 7 (c.840, chr5:70,951,946). +pub const SMN1_EXON7: (&str, i64) = ("chr5", 70_951_946); + +/// What the SMN table says at SMN1 exon 7 and at the marker site. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SmnRead { + /// SMN1 psCN at exon 7, when resolved. + pub pscn: Option, + /// How the psCN reads, for the table. + pub text: String, + /// The aggregate SMN1 + SMN2 copies at the marker site, when resolved. + pub agcn: Option, +} + +/// One value of a `psCN`/`agCN` column (comma separated, one per region). +fn value(row: &[(&str, &str)], kind: &str, idx: usize) -> (Option, String) { + let get = |k: &str| { + row.iter() + .rev() + .find(|(h, _)| *h == k) + .map(|(_, v)| *v) + .unwrap_or("") + }; + let vals: Vec<&str> = get(kind).split(',').collect(); + let quals: Vec<&str> = get(&format!("{kind}_qual")).split(',').collect(); + let filt = match get(&format!("{kind}_filter")) { + "" => ".", + f => f, + }; + let v = vals.get(idx).copied().unwrap_or("?"); + let q = quals.get(idx).copied().unwrap_or("*"); + let ok = filt == "PASS" + && pycompat::num::parse_float(q).is_ok_and(|q| q >= 20.0) + && !v.is_empty() + && v.bytes().all(|c| c.is_ascii_digit()); + let n = if ok { v.parse().ok() } else { None }; + let text = format!( + "{v} (quality {q}, filter {filt}){}", + if n.is_some() { "" } else { ", not resolved" } + ); + (n, text) +} + +/// The row's region and its homologous regions, 1-based closed. +fn regions(row: &[(&str, &str)]) -> Option> { + let get = |k: &str| row.iter().rev().find(|(h, _)| *h == k).map(|(_, v)| *v); + let start = pycompat::num::parse_int(get("start")?).ok()?; + let end = pycompat::num::parse_int(get("end")?).ok()?; + let mut out = vec![( + get("chrom")?.to_owned(), + i64::try_from(start).ok()? + 1, + i64::try_from(end).ok()?, + )]; + for h in get("homologous_regions").unwrap_or("").split(',') { + if let Some(r) = homologous(crate::strip(h)) { + out.push(r); + } + } + Some(out) +} + +/// `^([^:]+):(\d+)-(\d+)` at the start of `h`. +fn homologous(h: &str) -> Option<(String, i64, i64)> { + let (chrom, rest) = h.split_once(':')?; + if chrom.is_empty() { + return None; + } + let digits = |s: &str| s.bytes().take_while(u8::is_ascii_digit).count(); + let n = digits(rest); + let (s, rest) = rest.split_at(n); + let rest = rest.strip_prefix('-')?; + let e = &rest[..digits(rest)]; + if s.is_empty() || e.is_empty() { + return None; + } + Some((chrom.to_owned(), s.parse().ok()?, e.parse().ok()?)) +} + +/// Reads step 35's SMN table at `path` (`None`, or a path that does not +/// exist: no table). `marker_copies` are the marker's sites on `chrom`. +pub fn read(path: Option<&str>, chrom: &str, marker_copies: &[i64]) -> Result { + let mut out = SmnRead { + pscn: None, + text: "not read (no SMN table from step 35)".to_owned(), + agcn: None, + }; + let Some(path) = path.filter(|p| !p.is_empty() && std::path::Path::new(p).exists()) else { + return Ok(out); + }; + out.text = "not read (no row of the SMN table covers SMN1 exon 7)".to_owned(); + let (mut ps_done, mut ag_done) = (false, false); + let mut header: Option> = None; + crate::each_line(path, |line| { + let Some(h) = &header else { + header = Some(line.split('\t').map(str::to_owned).collect()); + return Ok(()); + }; + let row: Vec<(&str, &str)> = h.iter().map(String::as_str).zip(line.split('\t')).collect(); + let Some(regs) = regions(&row) else { + return Ok(()); + }; + if !ps_done + && let Some(i) = regs + .iter() + .position(|(c, s, e)| c == SMN1_EXON7.0 && *s <= SMN1_EXON7.1 && SMN1_EXON7.1 <= *e) + { + (out.pscn, out.text) = value(&row, "psCN", i); + ps_done = true; + } + if !ag_done + && regs + .iter() + .any(|(c, s, e)| c == chrom && marker_copies.iter().any(|p| s <= p && p <= e)) + { + out.agcn = value(&row, "agCN", 0).0; + ag_done = true; + } + Ok(()) + })?; + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HEAD: &str = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n"; + + /// A table file that is removed, with its folder once empty, when the + /// test ends. + struct Table(String); + + impl Drop for Table { + fn drop(&mut self) { + let p = std::path::Path::new(&self.0); + let _ = std::fs::remove_file(p); + if let Some(dir) = p.parent() { + let _ = std::fs::remove_dir(dir); + } + } + } + + fn table(rows: &[&str]) -> Table { + let dir = + std::env::temp_dir().join(format!("og-cn-smn-{}-{}", std::process::id(), rows.len())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join(format!("t{}.tsv", rows.join("").len())); + std::fs::write(&p, format!("{HEAD}{}", rows.join(""))).unwrap(); + Table(p.to_string_lossy().into_owned()) + } + + const SITES: [i64; 2] = [70_952_074, 70_076_654]; + + #[test] + fn the_first_row_over_exon_7_wins() { + // Two rows over exon 7: the first is unresolved, the second says 2. + // The first row's reading stands, as the Paraphase comparison reads it. + let p = table(&[ + "chr5\t70951000\t70953000\tSMN1\tPASS\t4\t30\tLowQual\t1,3\t12,40\tchr5:70075000-70078000\n", + "chr5\t70951900\t70952000\tSMN1\tPASS\t4\t30\tPASS\t2,2\t40,40\tchr5:70076000-70076100\n", + ]); + let r = read(Some(&p.0), "chr5", &SITES).unwrap(); + assert_eq!(r.pscn, None); + assert_eq!(r.text, "1 (quality 12, filter LowQual), not resolved"); + assert_eq!(r.agcn, Some(4)); + } + + #[test] + fn a_resolved_row_and_the_placeholders() { + let p = table(&[ + "chr5\t70925000\t70955000\tSMN1\tPASS\t4\t30\tPASS\t2\t40\tchr5:70049000-70080000\n", + ]); + let r = read(Some(&p.0), "chr5", &SITES).unwrap(); + assert_eq!( + (r.pscn, r.text.as_str()), + (Some(2), "2 (quality 40, filter PASS)") + ); + // The homologous copy holds exon 7 too, but psCN has one value. + let p = table(&[ + "chr5\t70049000\t70050000\tSMN2\tPASS\t4\t30\tPASS\t2\t40\tchr5:70951000-70952000\n", + ]); + let r = read(Some(&p.0), "chr5", &SITES).unwrap(); + assert_eq!(r.text, "? (quality *, filter PASS), not resolved"); + } + + #[test] + fn no_table_and_no_row() { + assert_eq!( + read(None, "chr5", &SITES).unwrap().text, + "not read (no SMN table from step 35)" + ); + let p = table(&["chr1\t1\t2\tX\tPASS\t2\t30\tPASS\t2\t40\t.\n"]); + assert_eq!( + read(Some(&p.0), "chr5", &SITES).unwrap().text, + "not read (no row of the SMN table covers SMN1 exon 7)" + ); + } +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml index 511900e6e..cd006de26 100644 --- a/rust/og/Cargo.toml +++ b/rust/og/Cargo.toml @@ -21,6 +21,7 @@ og-steps.workspace = true og-filters.workspace = true og-setup.workspace = true og-chip.workspace = true +og-cn.workspace = true og-reports.workspace = true og-runlog.workspace = true og-ancestry.workspace = true diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index fcbdd716b..d564f4e94 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -39,6 +39,7 @@ const USAGE: &str = "usage: og [args] og qc sex-loss --table TABLE [--regions REGIONS] [--vcf VCF] og prs format [args] og chip prepare|report|check-table [args] + og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] og family plan|prepare|check|report|segments [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] @@ -167,6 +168,11 @@ fn main() -> ExitCode { )), None => usage_error("the following arguments are required: chip tool"), }, + "cn" => ExitCode::from(og_cn::cli::main( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), "vntyper" | "locityper" | "mocha" | "mei" | "viral" | "y" | "exomiser" => { let verb = rest.first().map(String::as_str); let args = rest.get(1..).unwrap_or_default(); diff --git a/scripts/32-pypgx.sh b/scripts/32-pypgx.sh index 995e64df1..1584cc380 100755 --- a/scripts/32-pypgx.sh +++ b/scripts/32-pypgx.sh @@ -84,14 +84,14 @@ fi # A BAM aligned to a reference with ALT or HLA contigs is refused: pypgx # reads copy number from depth, and that depth splits between a gene and its -# ALT copy, which reads as a deletion (bin/copy_number.py check-bam, exit 4). +# ALT copy, which reads as a deletion (og cn check-bam, exit 4). # A header that cannot be read leaves the check undone, with a warning. It runs # after the bundle check, which starts no container. HDR="${OUTPUT_DIR}/bam_header.part" if run_in --cpus 1 --memory 1g "${SAMTOOLS_IMAGE}" samtools view -H "$(cpath "$BAM")" > "$HDR"; then rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PYTHON_IMAGE}" \ - python3 /pgp-bin/copy_number.py check-bam --header "$(cpath "$HDR")" || rc=$? + /pgp-bin/og cn check-bam --header "$(cpath "$HDR")" || rc=$? rm -f "$HDR" if [ "$rc" -eq 4 ]; then exit 1 diff --git a/scripts/35-paralogs.sh b/scripts/35-paralogs.sh index 64d609f9c..2e9c39088 100755 --- a/scripts/35-paralogs.sh +++ b/scripts/35-paralogs.sh @@ -21,8 +21,8 @@ # HBA1 and HBA2 (alpha-thalassaemia deletions) have no Parascopy model, so the # step counts them from read depth: the depth over both gene bodies against # single-copy sequence on each side of the cluster gives the number of -# alpha-globin genes (bin/copy_number.py on the HBA row of bin/cn_loci.tsv, -# laid out by bin/hba_copy_number.py). A failure there writes an HBA +# alpha-globin genes (og cn call on the HBA row of bin/cn_loci.tsv). A +# failure there writes an HBA # table that says not assessed, and does not stop the step. # # The gene-dosage rows of the same catalog (HP, LCE3BC, CCL3L1: loci no @@ -39,7 +39,7 @@ # Beside the SMN1 psCN the step reads SMN1 g.27134T>G (rs143838139; Luo et # al. 2014), a marker of SMN1 duplication alleles: samtools mpileup at its # SMN1 and SMN2 copies (the reads cannot tell them apart there), read as a -# fraction of copies by bin/traits_report.py --marker SMN1_27134. With two +# fraction of copies by og cn marker-read SMN1_27134. With two # SMN1 copies it raises or lowers the chance of a silent 2+0 carrier; it # never proves 2+0, and the table's message says so; # ${SAMPLE}_smn1_marker.tsv. @@ -106,14 +106,14 @@ for f in "$BAM" "${BAM}.bai" "$REF_FASTA" "${REF_FASTA}.fai" "$LOCI"; do done # A BAM aligned to a reference with ALT or HLA contigs is refused: Parascopy # and the HBA depth count read copy number from depth, and that depth splits -# between a gene and its ALT copy (bin/copy_number.py check-bam, exit 4). +# between a gene and its ALT copy (og cn check-bam, exit 4). # A header that cannot be read leaves the check undone, with a warning. mkdir -p "$OUTDIR" HDR="${OUTDIR}/bam_header.part" if run_in --cpus 1 --memory 1g "${PARASCOPY_IMAGE}" samtools view -H "$(cpath "$BAM")" > "$HDR"; then rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py check-bam --header "$(cpath "$HDR")" || rc=$? + /pgp-bin/og cn check-bam --header "$(cpath "$HDR")" || rc=$? rm -f "$HDR" if [ "$rc" -eq 4 ]; then exit 1 @@ -246,7 +246,7 @@ mv "$WORK" "${OUTDIR}/${SAMPLE}_parascopy" # SMN1 g.27134T>G beside the SMN1 psCN (as the PARASCOPY module): samtools # mpileup at its SMN1 and SMN2 copies, one -r per site, every MAPQ, # duplicates left out, bases of quality 13 or more; then -# bin/traits_report.py --marker SMN1_27134 with the SMN table (the SMN1 psCN +# og cn marker-read SMN1_27134 with the SMN table (the SMN1 psCN # at exon 7 and the agCN). A failure writes a table that says not assessed, # and the step goes on. SMN1M_OUT="${OUTDIR}/${SAMPLE}_smn1_marker.tsv" @@ -255,13 +255,13 @@ rm -rf "$SMN1M_OUT" "$SMN1M_WORK" mkdir -p "$SMN1M_WORK" # shellcheck disable=SC2016 # $1 to $3 belong to the inner bash if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/traits_report.py --marker-regions SMN1_27134 > "${SMN1M_WORK}/sites.txt" \ + /pgp-bin/og cn marker-regions SMN1_27134 > "${SMN1M_WORK}/sites.txt" \ && run_in --cpus 1 --memory 2g "${PARASCOPY_IMAGE}" bash -euo pipefail -c ' while read -r r; do samtools mpileup -A -B -q 0 -Q 13 -d 100000 -r "$r" "$1"; done < "$2" > "$3"' \ _ "$(cpath "$BAM")" "$(cpath "${SMN1M_WORK}/sites.txt")" "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/traits_report.py --marker SMN1_27134 --pileup "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ - --copy-table "$(cpath "${OUTDIR}/${SAMPLE}_smn_copy_number.tsv")" --marker-out "$(cpath "${SMN1M_OUT}.tmp")"; then + /pgp-bin/og cn marker-read SMN1_27134 --pileup "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ + --copy-table "$(cpath "${OUTDIR}/${SAMPLE}_smn_copy_number.tsv")" --out "$(cpath "${SMN1M_OUT}.tmp")"; then mv "${SMN1M_OUT}.tmp" "$SMN1M_OUT" else rm -f "${SMN1M_OUT}.tmp" @@ -274,19 +274,19 @@ rm -rf "$SMN1M_WORK" # HBA1/HBA2 from depth (as the PARASCOPY module): samtools bedcov over the # regions of the HBA row of bin/cn_loci.tsv, all MAPQ and MAPQ 1 or more (a # BAM with no MAPQ 0 read over the genes left out the reads that fit both), -# then the engine's verdict (bin/copy_number.py). +# then the engine's verdict (og cn call). HBA_OUT="${OUTDIR}/${SAMPLE}_hba_copy_number.tsv" HBA_WORK="${OUTDIR}/hba_depth.part" rm -rf "$HBA_OUT" "$HBA_WORK" mkdir -p "$HBA_WORK" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus HBA > "${HBA_WORK}/regions.bed" \ + /pgp-bin/og cn bed --locus HBA > "${HBA_WORK}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${HBA_WORK}/regions.bed")" "$(cpath "$BAM")" > "${HBA_WORK}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov -Q 1 "$(cpath "${HBA_WORK}/regions.bed")" "$(cpath "$BAM")" > "${HBA_WORK}/bedcov_q1.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus HBA --bedcov "$(cpath "${HBA_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus HBA --bedcov "$(cpath "${HBA_WORK}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${HBA_WORK}/bedcov_q1.tsv")" --table "$(cpath "${HBA_OUT}.tmp")"; then mv "${HBA_OUT}.tmp" "$HBA_OUT" else @@ -299,7 +299,7 @@ rm -rf "$HBA_WORK" # Gene-dosage rows of bin/cn_loci.tsv that no pinned caller models (as the # PARASCOPY module): per locus samtools bedcov, all MAPQ and MAPQ 1 or more, -# then copy_number.py call; one line per locus in _cn_dosage.tsv. A +# then og cn call; one line per locus in _cn_dosage.tsv. A # locus whose check fails is not assessed there, and the step goes on. DOSAGE_LOCI="HP LCE3BC CCL3L1" DOSAGE_OUT="${OUTDIR}/${SAMPLE}_cn_dosage.tsv" @@ -311,13 +311,13 @@ for loc in $DOSAGE_LOCI; do d="${DOSAGE_WORK}/${loc}" mkdir -p "$d" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus "$loc" > "${d}/regions.bed" \ + /pgp-bin/og cn bed --locus "$loc" > "${d}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${d}/regions.bed")" "$(cpath "$BAM")" > "${d}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov -Q 1 "$(cpath "${d}/regions.bed")" "$(cpath "$BAM")" > "${d}/bedcov_q1.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus "$loc" --bedcov "$(cpath "${d}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus "$loc" --bedcov "$(cpath "${d}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${d}/bedcov_q1.tsv")" --table "$(cpath "${d}/table.tsv")"; then awk -F'\t' -v OFS='\t' -v l="$loc" 'NR > 1 {v[$1] = $2} END {print l, v["status"], v["copies"], v["estimate"], v["reason"], v["reading"]}' "${d}/table.tsv" \ @@ -335,7 +335,7 @@ rm -rf "$DOSAGE_WORK" # the CYP21A2 row of bin/cn_loci.tsv (CYP21A2 and CYP21A1P added up against # single-copy flanks: the RCCX module total), samtools mpileup without a # reference at its paralogous sequence variant sites (each on CYP21A2 and on -# CYP21A1P, every read whatever its MAPQ), then copy_number.py call: the +# CYP21A1P, every read whatever its MAPQ), then og cn call: the # CYP21A2 copies at each site, and a deletion or a possible conversion. The # bedcov runs twice, all MAPQ and MAPQ 1 or more (as HBA): a BAM without the # MAPQ 0 reads that fit both copies reads not assessed, never a deletion. A @@ -345,9 +345,9 @@ CYP21_WORK="${OUTDIR}/CYP21A2.part" rm -rf "$CYP21_OUT" "$CYP21_WORK" mkdir -p "$CYP21_WORK" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus CYP21A2 > "${CYP21_WORK}/regions.bed" \ + /pgp-bin/og cn bed --locus CYP21A2 > "${CYP21_WORK}/regions.bed" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --psv --locus CYP21A2 > "${CYP21_WORK}/psv.bed" \ + /pgp-bin/og cn bed --psv --locus CYP21A2 > "${CYP21_WORK}/psv.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${CYP21_WORK}/regions.bed")" "$(cpath "$BAM")" > "${CYP21_WORK}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ @@ -357,7 +357,7 @@ if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ -r "$(awk 'NR == 1 {c = $1; s = $2 + 1} {e = $3} END {print c ":" s "-" e}' "${CYP21_WORK}/psv.bed")" \ -l "$(cpath "${CYP21_WORK}/psv.bed")" "$(cpath "$BAM")" > "${CYP21_WORK}/pileup.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus CYP21A2 --bedcov "$(cpath "${CYP21_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus CYP21A2 --bedcov "$(cpath "${CYP21_WORK}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${CYP21_WORK}/bedcov_q1.tsv")" --pileup "$(cpath "${CYP21_WORK}/pileup.tsv")" \ --table "$(cpath "${CYP21_OUT}.tmp")"; then mv "${CYP21_OUT}.tmp" "$CYP21_OUT" @@ -371,7 +371,7 @@ fi rm -rf "$CYP21_WORK" # Interval rows of bin/cn_loci.tsv (as the PARASCOPY module): one samtools -# bedcov over the segments of every row, then copy_number.py intervals; one +# bedcov over the segments of every row, then og cn intervals; one # line per row in _cn_intervals.tsv. A failed pass writes every row as # not assessed (check_failed), and the step goes on; when even that container # fails, the table is the header alone and the log says no row was assessed. @@ -380,15 +380,15 @@ INTERVAL_WORK="${OUTDIR}/cn_intervals.part" rm -rf "$INTERVAL_OUT" "$INTERVAL_WORK" mkdir -p "$INTERVAL_WORK" if ! { run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --kind interval > "${INTERVAL_WORK}/regions.bed" \ + /pgp-bin/og cn bed --kind interval > "${INTERVAL_WORK}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${INTERVAL_WORK}/regions.bed")" "$(cpath "$BAM")" > "${INTERVAL_WORK}/bedcov.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py intervals --bedcov "$(cpath "${INTERVAL_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn intervals --bedcov "$(cpath "${INTERVAL_WORK}/bedcov.tsv")" \ --sex "$SEX" --out "$(cpath "$INTERVAL_OUT")"; }; then echo "WARNING: the interval depth check failed; ${INTERVAL_OUT##*/} lists every row as not assessed." >&2 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py intervals --failed --out "$(cpath "$INTERVAL_OUT")" \ + /pgp-bin/og cn intervals --failed --out "$(cpath "$INTERVAL_OUT")" \ || printf 'locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\testimate\treason\n' > "$INTERVAL_OUT" fi rm -rf "$INTERVAL_WORK" diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index bc687f5e5..061500751 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -49,13 +49,13 @@ if [ -f "$BAM" ] && [ -f "${BAM}.bai" ]; then echo "[1/2] Read depth at the marker sites and over RHD (samtools bedcov, MAPQ >= 20)..." # The marker table is a BED with extra columns; bedcov reads it once the # comment and header lines are gone. The RHD regions come from - # bin/copy_number.py (BLOODTYPE_REGIONS in Nextflow) as a 4-column BED. + # og cn bed (BLOODTYPE_REGIONS in Nextflow) as a 4-column BED. # bedcov stops on a contig the BAM has no @SQ line for, so those rows are # left out: their sites get no depth row and are not determined. The # BLOODTYPE_DEPTH process runs the same commands. # shellcheck disable=SC2016 # $1 belongs to the inner shell run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - sh -c 'python3 /pgp-bin/copy_number.py bed --locus RHD > "$1"' _ "${O}/rhd.bed" + sh -c '/pgp-bin/og cn bed --locus RHD > "$1"' _ "${O}/rhd.bed" # shellcheck disable=SC2016 # $1 to $6 belong to the inner bash, the two programs to awk run_in -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$SAMTOOLS_IMAGE" bash -euo pipefail -c ' samtools view -H "$3" | awk -F "\t" "$5" > "$2/contigs.txt" diff --git a/scripts/46-lpa.sh b/scripts/46-lpa.sh index def07ded0..a682a9180 100755 --- a/scripts/46-lpa.sh +++ b/scripts/46-lpa.sh @@ -6,14 +6,12 @@ # of apo(a); a low total goes with a higher Lp(a) level, a mostly inherited # cardiovascular risk factor. The step counts the repeats from the depth over # the KIV-2 exons against the depth over single-copy LPA exons -# (bin/copy_number.py on the LPA_KIV2 row of bin/cn_loci.tsv, laid out by -# bin/lpa_kiv2.py; the same code as the LPA Nextflow processes), with a 95% -# interval. Beside the count it reads the KIV-2 4925G>A splice variant +# (og cn call on the LPA_KIV2 row of bin/cn_loci.tsv; the same code as the +# LPA Nextflow processes), with a 95% interval. Beside the count it reads the KIV-2 4925G>A splice variant # (Coassin et al. 2017), which lowers the Lp(a) made from the chromosome that # carries it: samtools mpileup at the site's seven GRCh38 copies (the six -# KIV-2 units and KIV-1), read as a fraction of copies by -# bin/traits_report.py --marker LPA_KIV2_4925 into its own table, laid out by -# bin/lpa_kiv2.py. +# KIV-2 units and KIV-1), read as a fraction of copies in the same og cn +# call, into its own table. # The two common risk variants, rs10455872 and rs3798220, are rows # of step 42's traits panel; the report's Lp(a) section reads both. A research # estimate, not a diagnostic test: the actionable test is a blood Lp(a) @@ -59,13 +57,13 @@ echo "BAM: ${BAM}" mkdir -p "$OUTDIR" rm -f "$TABLE" "$DEPTH" "$DEPTH_Q1" "$PILEUP" "$NULL" "$BED" "$SITES" -echo "[1/4] The regions (bin/copy_number.py bed --locus LPA_KIV2) and the 4925G>A sites..." +echo "[1/3] The regions (og cn bed --locus LPA_KIV2) and the 4925G>A sites..." # shellcheck disable=SC2016 # $1 and $2 belong to the inner shell run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - sh -c 'python3 /pgp-bin/copy_number.py bed --locus LPA_KIV2 > "$1" && - python3 /pgp-bin/traits_report.py --marker-regions LPA_KIV2_4925 > "$2"' _ "$(cpath "$BED")" "$(cpath "$SITES")" + sh -c '/pgp-bin/og cn bed --locus LPA_KIV2 > "$1" && + /pgp-bin/og cn marker-regions LPA_KIV2_4925 > "$2"' _ "$(cpath "$BED")" "$(cpath "$SITES")" -echo "[2/4] Read depth over the KIV-2 and single-copy LPA exons (samtools bedcov -j, all MAPQ and MAPQ >= 1)" +echo "[2/3] Read depth over the KIV-2 and single-copy LPA exons (samtools bedcov -j, all MAPQ and MAPQ >= 1)" echo " and the reads at the 4925G>A sites (samtools mpileup, every MAPQ)..." # bedcov stops on a contig the BAM has no @SQ line for: without chr6 the # tables stay empty and the count is not assessed. mpileup runs once per site @@ -86,18 +84,20 @@ run_in --cpus 1 --memory 2g "$SAMTOOLS_IMAGE" bash -euo pipefail -c ' "$(cpath "$PILEUP")" "$(cpath "$SITES")" rm -f "$BED" "$SITES" -echo "[3/4] The count and its interval (bin/copy_number.py call --locus LPA_KIV2)..." +echo "[3/3] The count, its interval and KIV-2 4925G>A beside it (og cn call --locus LPA_KIV2 --marker LPA_KIV2_4925)..." +# One run reads the count and the marker, so the marker is judged on the +# count's own MAPQ 0 share. A failure of the marker never costs the count (as +# step 35's SMN1 marker): when the count table is there, the 4925G>A table +# then says not assessed, and the step goes on. The LPA process writes the +# same table. +rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/copy_number.py call --locus LPA_KIV2 --bedcov "$(cpath "$DEPTH")" \ - --bedcov-mapq1 "$(cpath "$DEPTH_Q1")" --table "$(cpath "$TABLE")" - -echo "[4/4] KIV-2 4925G>A beside the count (bin/traits_report.py --marker LPA_KIV2_4925)..." -# A failure here never costs the count (as step 35's SMN1 marker): the -# 4925G>A table then says not assessed, and the step goes on. The LPA -# process writes the same table. -if ! run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/traits_report.py --marker LPA_KIV2_4925 --pileup "$(cpath "$PILEUP")" \ - --copy-table "$(cpath "$TABLE")" --marker-out "$(cpath "$NULL")"; then + /pgp-bin/og cn call --locus LPA_KIV2 --bedcov "$(cpath "$DEPTH")" \ + --bedcov-mapq1 "$(cpath "$DEPTH_Q1")" --table "$(cpath "$TABLE")" \ + --marker LPA_KIV2_4925 --marker-pileup "$(cpath "$PILEUP")" --marker-out "$(cpath "$NULL")" || rc=$? +if [ "$rc" -ne 0 ]; then + # No count table: the count itself failed (exit 3 for a broken control). + [ -s "$TABLE" ] || exit "$rc" echo "WARNING: the KIV-2 4925G>A check failed; ${NULL##*/} says not assessed." >&2 printf 'metric\tvalue\nmarker\tLPA KIV-2 4925G>A\nrsid\trs1484919544 (first KIV-2 copy)\nstatus\tnot assessed\nalt_reads\t.\ndepth\t.\nfraction\t.\ncopies\t.\nkiv2_copies\t.\nmessage\t%s\n' \ 'LPA KIV-2 4925G>A: not assessed, the check failed; see the log of step 46.' > "$NULL" diff --git a/scripts/ci/build-cn-intervals.py b/scripts/ci/build-cn-intervals.py index 6c186bf7d..5982debdf 100755 --- a/scripts/ci/build-cn-intervals.py +++ b/scripts/ci/build-cn-intervals.py @@ -2,7 +2,7 @@ """build-cn-intervals.py: write the interval rows of bin/cn_loci.tsv. The interval rows are the copy-number checks step 35 runs with one -`samtools bedcov` pass (bin/copy_number.py intervals): +`samtools bedcov` pass (og cn intervals): - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CLINGEN_RECURRENT_CNV_URL and _SHA256 in diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f89a737c4..8b0177502 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -60,9 +60,9 @@ bash-step-14-imputation scripts/14-imputation-prep.sh scripts/ci/fake-docker/lib bash-step-19-delly scripts/19-delly.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -bloodtype-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +bloodtype-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh -callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh +callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz @@ -73,90 +73,90 @@ clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/ clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/cdna_scan.py bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/cdna_scan.py bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -cn-engine-1-parity assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -cn-intervals-1-catalog bin/cn_loci.tsv scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh -cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-engine-1-parity assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh +cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/comphet_phase.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/ibd_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/ibd_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh -followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz +followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv bin/traits_report.py modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/genotype_rules_indels.py bin/traits_report.py -genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh -genotype-rules-2-smn1-marker bin/collect_summary.py bin/copy_number.py bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh +genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh +genotype-rules-2-smn1-marker bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/ibd_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh -nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/traits_report.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/family_report.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/clip_check.py bin/collect_summary.py bin/cpsr_sample_id bin/render_report.py bin/results_index.py bin/traits_report.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh -pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -platelet-antigens-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +platelet-antigens-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py bin/traits_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-1-sample-qc bin/collect_summary.py bin/og bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/family_report.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/clip_check.py bin/comphet_phase.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh -reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/traits_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh reports-and-pgx-correctness-revel scripts/30-vcfanno.sh scripts/ci/fake-docker/lib.sh -run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cpsr_sample_id bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cpsr_sample_id bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh @@ -169,9 +169,9 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 2f32a025a..e9da31699 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -27,13 +27,14 @@ - step 33's mosaic rows come from og qc sex-loss, step 26b's painting from bin/ancestry_fine_report.py, step 35b's table from bin/paraphase_report.py, step 39's from bin/couple_report.py and step 42's - from bin/traits_report.py, each fed made-up inputs; step 46's rows come - from bin/lpa_kiv2.py's assess, step 35's HBA1/HBA2 table from - bin/hba_copy_number.py's assess, step 49's depth rows from - bin/chip_report.py's detection_limit, with made-up counts; the tables of - steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written here, in the - columns their writers use (og for 45 to 53), with the values those - writers give for the made-up inputs named in each function. + from bin/traits_report.py, each fed made-up inputs; step 46's table and + step 35's HBA1/HBA2 table are what og cn call writes for made-up depths + (HBA_TABLE and LPA_TABLE; tests/test_hba_copy_number.py and + tests/test_lpa_kiv2.py check that og still writes them), step 49's depth + rows from bin/chip_report.py's detection_limit, with made-up counts; the + tables of steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written + here, in the columns their writers use (og for 45 to 53), with the values + those writers give for the made-up inputs named in each function. DEMO-001 carries every section, so every card is in the pictures: also the Paraphase card (a PacBio HiFi step, though the invented reads are short) and @@ -921,13 +922,8 @@ def region(total, rsi, **kw): "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\nchr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t2,2\t33,30\t" "chr5:70049000-70077000:-\n") - # Step 35's HBA1/HBA2 table: bin/hba_copy_number.py's own assess() on - # made-up depths, four alpha-globin genes. - import hba_copy_number as hba - depth = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} - res = hba.assess(depth, {k: v * (0.76 if k.startswith("HBA") else 0.98) for k, v in depth.items()}) - put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), - "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in hba.ORDER)) + # Step 35's HBA1/HBA2 table: four alpha-globin genes. + put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), HBA_TABLE) def couple(d, rng): @@ -1100,18 +1096,38 @@ def viral(d): head + "".join("\t".join(r[:7] + ("31.00",) + r[7:]) + "\n" for r in rows)) +# Step 35's HBA1/HBA2 depths (mean depth per region of the HBA row of +# bin/cn_loci.tsv, every read) and the share of each left at MAPQ 1 or more, +# and the table og cn call --locus HBA writes for them: four genes. +HBA_DEPTH = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} +HBA_Q1 = {"flank_left": 0.98, "HBA2": 0.76, "HBA1": 0.76, "flank_right": 0.98} +HBA_TABLE = ("metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t4\nestimate\t3.96\nhba2_depth\t30.9\n" + "hba1_depth\t30.1\nflank_left_depth\t31.2\nflank_right_depth\t30.4\nmapq0_share\t0.240\n" + "message\tfour alpha-globin genes (aa/aa): no deletion seen\n") + + +def lpa_depth(row): + """Step 46's made-up depths over the LPA_KIV2 row's segments (catalog + order, from bin/copy_number.py's load_catalog): every read, and MAPQ 1 or + more. About 37 KIV-2 repeats at a flank depth near 31, half the KIV-2 + depth at MAPQ 0.""" + m = {n: 86.5 * (1.02 if i % 2 == 0 else 0.98) for i, (n, *_) in enumerate(row["_targets"])} + m.update({n: 31.0 * (1.03 if i % 2 == 0 else 0.97) for i, (n, *_) in enumerate(row["_controls"])}) + return m, {n: v * (0.5 if n.startswith("kiv2_") else 0.98) for n, v in m.items()} + + +# The table og cn call --locus LPA_KIV2 writes for lpa_depth. +LPA_TABLE = ("metric\tvalue\nstatus\testimated\nkiv2_copies\t37.0\nci95_low\t36.3\nci95_high\t37.8\n" + "class\ttypical\nkiv2_depth\t86.61\nflank_depth\t31.09\nmapq0_share\t0.500\n" + "message\tabout 37 KIV-2 repeats in total (95% interval 36.3 to 37.8): within or overlapping the range " + "most people have (30 to 45 for about 70% of the 1000 Genomes samples)\n" + "advice\tA research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 " + "European Atherosclerosis Society consensus advises measuring it at least once in every adult.\n") + + def lpa(d, rng): - """Step 46: bin/lpa_kiv2.py's assess on made-up depths, about 37 KIV-2 - repeats at a flank depth near 31, half the KIV-2 depth at MAPQ 0.""" - import lpa_kiv2 - m, q1 = {}, {} - for *_, name in lpa_kiv2.REGIONS: - m[name] = rng.uniform(83, 90) if name.startswith("kiv2_") else rng.uniform(29.5, 32.5) - q1[name] = m[name] * (0.5 if name.startswith("kiv2_") else 0.98) - res = lpa_kiv2.assess(m, q1) - res["advice"] = lpa_kiv2.ADVICE - put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), - "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in lpa_kiv2.ORDER)) + """Step 46: the table og cn call writes for lpa_depth (LPA_TABLE).""" + put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), LPA_TABLE) def chip(d, rng): diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index 3d90a68c4..984840f2f 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -112,7 +112,7 @@ check_eq "Kidd: GIAB's c.838G>A 1/1" "$(cell "$H" Kidd phenotype)" "Jk(a-b+)" check_eq "ABO confidence (every site has reads)" "$(cell "$H" ABO confidence)" "high" check_eq "the evidence file has a row per site and per RhD region (the RHD row of bin/cn_loci.tsv)" \ "$(awk 'END {print NR - 1}' "${G}/${H}/bloodtype/${H}_bloodtype_evidence.tsv" 2>/dev/null)" \ - "$(( $(grep -v '^#' "$MARKERS" | awk 'END {print NR - 1}') + $(python3 "${REPO}/bin/copy_number.py" bed --locus RHD | wc -l) ))" + "$(( $(grep -v '^#' "$MARKERS" | awk 'END {print NR - 1}') + $("${REPO}/bin/og" cn bed --locus RHD | wc -l) ))" # --- planted ------------------------------------------------------------------------- P="${SAMPLE}btp" diff --git a/tests/e2e/cn-engine-1-parity.sh b/tests/e2e/cn-engine-1-parity.sh index d43124710..a4b04c253 100755 --- a/tests/e2e/cn-engine-1-parity.sh +++ b/tests/e2e/cn-engine-1-parity.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The copy-number engine (bin/copy_number.py, bin/cn_loci.tsv) against the +# The copy-number engine (og cn, bin/cn_loci.tsv) against the # per-gene code it replaced, on real reads: HG002's Illumina reads at the # alpha-globin cluster (chr16:95,001-235,000), the RH locus # (chr1:25.15-25.45 Mb) and LPA (chr6:160.53-160.67 Mb) from the 35x BWA-MEM @@ -10,8 +10,8 @@ # - step 41 of this tree and of the old tree write the same RhD row and the # same RhD evidence rows (other systems may change after the engine); # - step 35's HBA block (bedcov all MAPQ and MAPQ >= 1, then the verdict) -# writes the same table through copy_number.py as the old -# hba_copy_number.py does. +# writes the same table through og cn as the old hba_copy_number.py +# does. # The old tree is main at OLD_SHA, the last commit before the engine (the one # tests/test_copy_number.py froze its GOLD tables from), fetched by its sha # at run time, so the comparison keeps its meaning once the engine is on main. @@ -53,19 +53,20 @@ check_ge "HG002 reads at the three loci" "$(dk "$SAMTOOLS_IMAGE" samtools view - # dkb BIN IMAGE CMD...: dk with a tree's bin/ at its own path (this checkout is outside the work folder). dkb() { docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "$1:$1:ro" -w "$T" "$2" "${@:3}"; } -# hba_block TREE OUT BAM: step 35's HBA block with TREE's bin (main: hba_copy_number.py). +# hba_block TREE OUT BAM: step 35's HBA block with TREE's bin (this tree: og cn; +# the old tree: hba_copy_number.py). hba_block() { local bin="$1/bin" out=$2 bam=$3 d="${T}/hba.$$" rm -rf "$d"; mkdir -p "$d" - if [ -e "${bin}/copy_number.py" ]; then - dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/copy_number.py" bed --locus HBA > "${d}/r.bed" + if [ -e "${bin}/og" ]; then + dkb "$bin" "$PARASCOPY_IMAGE" "${bin}/og" cn bed --locus HBA > "${d}/r.bed" else dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/hba_copy_number.py" bed > "${d}/r.bed" fi dkb "$bin" "$PARASCOPY_IMAGE" samtools bedcov "${d}/r.bed" "$bam" > "${d}/c.tsv" dkb "$bin" "$PARASCOPY_IMAGE" samtools bedcov -Q 1 "${d}/r.bed" "$bam" > "${d}/q.tsv" - if [ -e "${bin}/copy_number.py" ]; then - dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/copy_number.py" call --locus HBA --bedcov "${d}/c.tsv" \ + if [ -e "${bin}/og" ]; then + dkb "$bin" "$PARASCOPY_IMAGE" "${bin}/og" cn call --locus HBA --bedcov "${d}/c.tsv" \ --bedcov-mapq1 "${d}/q.tsv" --table "$out" else dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/hba_copy_number.py" check --bedcov "${d}/c.tsv" \ diff --git a/tests/e2e/cn-guard-1-mapq0.sh b/tests/e2e/cn-guard-1-mapq0.sh index e1c5ca2a4..422773fdf 100755 --- a/tests/e2e/cn-guard-1-mapq0.sh +++ b/tests/e2e/cn-guard-1-mapq0.sh @@ -20,7 +20,7 @@ SRC="https://storage.googleapis.com/deepvariant/case-study-testdata/HG002.novase P=HG002cnguard W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" -python3 "${REPO}/bin/copy_number.py" bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" check_ge "segments of the CYP21A2 row to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/cn-intervals-2-hg002.sh b/tests/e2e/cn-intervals-2-hg002.sh index 371861a09..bb2cf3bb5 100755 --- a/tests/e2e/cn-intervals-2-hg002.sh +++ b/tests/e2e/cn-intervals-2-hg002.sh @@ -23,7 +23,7 @@ W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" ROWS=(AZFa AZFb AZFc_DAZ 22q11.2_DGS_proximal_A-B) args=(); for r in "${ROWS[@]}"; do args+=(--locus "$r"); done -python3 "${REPO}/bin/copy_number.py" bed "${args[@]}" | sort -k1,1 -k2,2n -u > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed "${args[@]}" | sort -k1,1 -k2,2n -u > "${W}/fetch.bed" check_ge "segments of the four rows to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/cyp21a2-short-reads-1-hg002.sh b/tests/e2e/cyp21a2-short-reads-1-hg002.sh index d7e2cf8b2..13a26e226 100755 --- a/tests/e2e/cyp21a2-short-reads-1-hg002.sh +++ b/tests/e2e/cyp21a2-short-reads-1-hg002.sh @@ -21,7 +21,7 @@ SRC="https://storage.googleapis.com/deepvariant/case-study-testdata/HG002.novase P=HG002cyp W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" -python3 "${REPO}/bin/copy_number.py" bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" check_ge "segments of the CYP21A2 row to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/genotype-rules-2-lpa-null.sh b/tests/e2e/genotype-rules-2-lpa-null.sh index c287305ce..f92c2bc11 100755 --- a/tests/e2e/genotype-rules-2-lpa-null.sh +++ b/tests/e2e/genotype-rules-2-lpa-null.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Step 46's KIV-2 4925G>A reading (bin/lpa_kiv2.py null-allele, MARKERS in -# bin/traits_report.py) on real reads: +# Step 46's KIV-2 4925G>A reading (og cn call --marker LPA_KIV2_4925, the +# marker of MARKERS in bin/traits_report.py) on real reads: # - dbSNP orientation, online: the record pinned for the marker is what # dbSNP's GRCh38 VCF of the build versions.env pins holds for rs1484919544 # (same position and REF, the pinned ALT among dbSNP's); control: the same @@ -19,10 +19,10 @@ # dropped: 'not seen' # The reading goes to lpa/_lpa_kiv2_4925.tsv; step 46's own table # keeps its rows. -# - the LPA module's pileup and 4925G>A lines, cut from its main.nf, write the -# same pileup and table in their images, byte for byte, on the planted BAM; -# with a 4925G>A reader that crashes, its block writes a table that says -# not assessed and leaves the count; +# - the LPA module's sites, pileup and count lines, cut from its main.nf, +# write the same pileup and tables in their images, byte for byte, on the +# planted BAM; with a 4925G>A reader that crashes after the count, its +# block writes a table that says not assessed and keeps the count; # - steps 24 and generate-report.sh on the planted and the reference samples: # both reports print the reading under the count with its caveat # ('present' with the reads and copies, 'not seen'). @@ -159,14 +159,15 @@ MOD="${REPO}/modules/local/lpa/main.nf" M="${T}/module" rm -rf "$M"; mkdir -p "$M" unesc() { sed -e 's/\\\\/\\/g' -e 's/\\\$/$/g' -e "s|\${bam}|${G}/${P}/aligned/${P}_sorted.bam|g" -e "s|\${meta.id}|${P}|g" \ - -e "s|\${pileup}|${P}_lpa_kiv2_4925_pileup.tsv|g"; } -grep -E '^ *traits_report\.py --marker-regions LPA_KIV2_4925 > lpa_sites\.txt$' "$MOD" | unesc > "${M}/sites.sh" + -e "s|\${pileup}|${P}_lpa_kiv2_4925_pileup.tsv|g" -e "s|\${depth}|${P}_lpa_kiv2_depth.tsv|g" \ + -e "s|\${depth_q1}|${P}_lpa_kiv2_depth_q1.tsv|g"; } +grep -E '^ *og cn marker-regions LPA_KIV2_4925 > lpa_sites\.txt$' "$MOD" | unesc > "${M}/sites.sh" sed -n '/^ *( while read -r r; do samtools mpileup /,/|| echo /p' "$MOD" | unesc > "${M}/pileup.sh" -sed -n '/^ *if ! traits_report\.py --marker LPA_KIV2_4925 --pileup/,/^ *fi$/p' "$MOD" | unesc > "${M}/null.sh" -check_eq "the module's sites, pileup and 4925G>A lines were cut from main.nf" \ - "$(cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" | wc -l | tr -d ' ')" 9 +sed -n '/^ *rc=0$/,/^ *fi$/p' "$MOD" | unesc > "${M}/null.sh" +check_eq "the module's sites, pileup and count lines were cut from main.nf" \ + "$(cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" | wc -l | tr -d ' ')" 12 cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" -cp "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "$M/" +cp "${G}/${P}/lpa/${P}_lpa_kiv2_depth.tsv" "${G}/${P}/lpa/${P}_lpa_kiv2_depth_q1.tsv" "$M/" mod() { docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "${REPO}/bin:/pgp-bin:ro" \ -w "$M" "$1" sh -c 'PATH="/pgp-bin:$PATH"; set -e; . "./$1"' _ "$2"; } mod "$PYTHON_IMAGE" sites.sh && mod "$SAMTOOLS_IMAGE" pileup.sh && mod "$PYTHON_IMAGE" null.sh 2>&1 | tail -n 3 @@ -174,16 +175,30 @@ check "the module writes step 46's pileup, byte for byte" \ cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2_4925_pileup.tsv" "${M}/${P}_lpa_kiv2_4925_pileup.tsv" check "the module writes step 46's 4925G>A table, byte for byte" \ cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2_4925.tsv" "${M}/${P}_lpa_kiv2_4925.tsv" -check "the module leaves step 46's table byte for byte" cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/${P}_lpa_kiv2.tsv" +check "the module writes step 46's count table, byte for byte" cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/${P}_lpa_kiv2.tsv" -# The module's fallback, in its image: a 4925G>A reader that crashes leaves the -# count and a table that says not assessed (scripts/46-lpa.sh writes the same -# table; tests/test_lpa_fallback.sh checks that). Control: the planted table -# above was 'present'. +# The module's fallback, in its image: an og whose 4925G>A reader crashes after +# it wrote the count (the planted og runs the real one without --marker, then +# exits 1) keeps the count and a table that says not assessed +# (scripts/46-lpa.sh writes the same table; tests/test_lpa_fallback.sh checks +# that). Control: the planted table above was 'present'. mkdir -p "${M}/broken" -printf '#!/bin/sh\necho "planted crash" >&2\nexit 1\n' > "${M}/broken/traits_report.py" -chmod +x "${M}/broken/traits_report.py" -cp "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/count_before.tsv" +cat > "${M}/broken/og" <<'OG' +#!/bin/sh +n=$# +while [ "$n" -gt 0 ]; do + case "$1" in + --marker|--marker-pileup|--marker-out) shift 2; n=$((n - 2)); continue ;; + esac + set -- "$@" "$1"; shift; n=$((n - 1)) +done +/pgp-bin/og "$@" || exit +echo "planted crash" >&2 +exit 1 +OG +chmod +x "${M}/broken/og" +mv "${M}/${P}_lpa_kiv2.tsv" "${M}/count_before.tsv" +rm -f "${M}/${P}_lpa_kiv2_4925.tsv" docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "${REPO}/bin:/pgp-bin:ro" \ -w "$M" "$PYTHON_IMAGE" sh -c "PATH=\"${M}/broken:/pgp-bin:\$PATH\"; set -e; . ./null.sh" 2>&1 | tail -n 2 check_eq "the module with a crashing 4925G>A reader: not assessed" \ diff --git a/tests/e2e/genotype-rules-2-smn1-marker.sh b/tests/e2e/genotype-rules-2-smn1-marker.sh index a5804dbea..38cc4b746 100755 --- a/tests/e2e/genotype-rules-2-smn1-marker.sh +++ b/tests/e2e/genotype-rules-2-smn1-marker.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Step 35's SMN1 g.27134T>G reading (bin/traits_report.py --marker -# SMN1_27134) on real reads: +# Step 35's SMN1 g.27134T>G reading (og cn marker-read SMN1_27134) on +# real reads: # - dbSNP orientation, online: the record pinned for the marker is what # dbSNP's GRCh38 VCF of the build versions.env pins holds for rs143838139; # control: the same compare with REF and ALT swapped fails; diff --git a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh index f9eee7a47..ef465eec9 100755 --- a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh +++ b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # The Nextflow HTML report and step 35's HBA1/HBA2 table -# (_hba_copy_number.tsv, bin/hba_copy_number.py), which the bash report +# (_hba_copy_number.tsv, og cn call --locus HBA), which the bash report # already reads beside the SMN table. # 1. HTML_REPORT, run for real in its image through a two-line pipeline, on # HG002's VCF with an SMN table and an HBA table that -# bin/hba_copy_number.py writes from made-up depths (four genes). It +# og cn writes from made-up depths (four genes). It # must place the HBA table in paralogs/ (a name it has no place for # stops the task), the report must carry the HBA line with the count, # and the not-assessed list must leave HBA1/HBA2 out. Negative control: @@ -28,12 +28,12 @@ mkdir -p "$T" printf 'chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n' \ > "${T}/${SAMPLE}_smn_copy_number.tsv" # Made-up depths: 30x everywhere, a quarter of the gene depth at MAPQ 0 -python3 "${REPO}/bin/hba_copy_number.py" bed \ +"${REPO}/bin/og" cn bed --locus HBA \ | awk -F'\t' -v OFS='\t' '{print $0, ($3 - $2) * 30}' > "${T}/bedcov.tsv" -python3 "${REPO}/bin/hba_copy_number.py" bed \ +"${REPO}/bin/og" cn bed --locus HBA \ | awk -F'\t' -v OFS='\t' '{print $0, ($3 - $2) * ($4 ~ /^HBA/ ? 22.5 : 30)}' > "${T}/bedcov_q1.tsv" -python3 "${REPO}/bin/hba_copy_number.py" check --bedcov "${T}/bedcov.tsv" --bedcov-mapq1 "${T}/bedcov_q1.tsv" \ - --out "${T}/${SAMPLE}_hba_copy_number.tsv" +"${REPO}/bin/og" cn call --locus HBA --bedcov "${T}/bedcov.tsv" --bedcov-mapq1 "${T}/bedcov_q1.tsv" \ + --table "${T}/${SAMPLE}_hba_copy_number.tsv" cat "${T}/${SAMPLE}_hba_copy_number.tsv" check_eq "the made-up HBA table counts four" \ "$(awk -F'\t' '$1 == "alpha_globin_genes" {print $2}' "${T}/${SAMPLE}_hba_copy_number.tsv")" 4 diff --git a/tests/e2e/lpa-1.sh b/tests/e2e/lpa-1.sh index 2a01370da..e7dead184 100755 --- a/tests/e2e/lpa-1.sh +++ b/tests/e2e/lpa-1.sh @@ -11,7 +11,7 @@ # HG002 is published that this case could compare with, so the value goes # to the notes and the checks are on its arithmetic: # - the same reads with half of those over the KIV-2 block: N + 2 halves -# (bin/lpa_kiv2.py: N = 14 x M / D - 2); +# (og cn call --locus LPA_KIV2: N = 14 x M / D - 2); # - the same reads at an eighth of the depth: "too little depth"; # - nextflow run main.nf --tools lpa,html_report on the real reads publishes # the same three files as the bash step, byte for byte, and the HTML diff --git a/tests/e2e/paralog-catalog-1-gba.sh b/tests/e2e/paralog-catalog-1-gba.sh index 25f112717..cd548aef5 100755 --- a/tests/e2e/paralog-catalog-1-gba.sh +++ b/tests/e2e/paralog-catalog-1-gba.sh @@ -127,7 +127,7 @@ M="${GENOME_DIR}/${P}/module" rm -rf "$M"; mkdir -p "$M" SPECS=$(awk -F'\t' '/^#/ || !NF {next} !h {for (i = 1; i <= NF; i++) c[$i] = i; h = 1; next} {printf "%s%s|%s|%s", sep, $c["model"], $c["region"], $c["table"]; sep = " "}' "$LOCI") -# bin/ is on PATH in Nextflow; here copy_number.py runs from it mounted, as step 35 runs it. +# bin/ is on PATH in Nextflow; here og runs from it mounted, as step 35 runs it. sed -n '/# Paralog loci (as scripts\/35-paralogs.sh)/,/# end of the paralog loci/p' "$MOD" \ | sed -e 's/\\\\/\\/g' -e 's/\\\$/$/g' \ -e "s|\${parascopy_data}|/genome/reference/parascopy-${PARASCOPY_DATA_VERSION}|g" \ @@ -138,7 +138,7 @@ sed -n '/# Paralog loci (as scripts\/35-paralogs.sh)/,/# end of the paralog loci -e "s|\${background}|-b /genome/${P}/parascopy_windows.bed --no-gc|g" \ -e "s|\${task.cpus}|2|g" \ -e "s/\${loci}/${SPECS}/g" \ - -e "s|^\( *\)copy_number.py |\1python3 /pgp-bin/copy_number.py |" > "${M}/block.sh" + -e "s|^\( *\)og cn |\1/pgp-bin/og cn |" > "${M}/block.sh" check_ge "the module's loop was cut from main.nf" "$(wc -l < "${M}/block.sh" | tr -d ' ')" 25 check "the cut loop has no Nextflow variable left" \ sh -c "! grep -E '[$][{](parascopy_data|params[.]|bam|meta[.]|reference|background|task[.]|loci)' '${M}/block.sh'" diff --git a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh index e6659f6c6..7e203c434 100755 --- a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh +++ b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Step 35's HBA1/HBA2 count from read depth (bin/hba_copy_number.py) on +# Step 35's HBA1/HBA2 count from read depth (og cn call --locus HBA) on # synthetic reads, with a planted one-gene deletion. The fixture has no reads # at the alpha-globin cluster, so the case writes its own: 150 bp reads cut # from the fixture reference every 5 bp over chr16:95,000-235,000 (30x), @@ -64,7 +64,7 @@ fa, header, sample, out = sys.argv[1:] seq = "".join(l.strip() for l in open(fa) if not l.startswith(">")).upper() start0 = 95000 # 0-based start of chr16:95001-235000 rg = next((l.split("\t")[1][3:] for l in open(header) if l.startswith("@RG")), None) -hba2 = (172875, 173710) # bin/hba_copy_number.py's HBA2, 0-based +hba2 = (172875, 173710) # the HBA row's HBA2 (og cn bed --locus HBA), 0-based L, step, n = 150, 5, 0 comp = str.maketrans("ACGTN", "TGCAN") seen = {} diff --git a/tests/fake-docker/cn-engine-calls.sh b/tests/fake-docker/cn-engine-calls.sh index 043cb99a5..f14492911 100755 --- a/tests/fake-docker/cn-engine-calls.sh +++ b/tests/fake-docker/cn-engine-calls.sh @@ -1,19 +1,26 @@ #!/usr/bin/env bash # Steps 35 (the HBA1/HBA2 block), 41 (RhD) and 46 (LPA KIV-2) count copies -# through one engine, bin/copy_number.py, with a hook that plays Parascopy and -# samtools bedcov (made-up depths: 30x everywhere, the KIV-2 block at 20 -# repeats) and runs the engine for real on the host: -# - each step calls copy_number.py bed once and step 35 and 46 call -# copy_number.py call once (step 41's report imports the engine), and no -# step calls hba_copy_number.py or lpa_kiv2.py any more; +# through one engine, og cn, with a hook that plays Parascopy and samtools +# bedcov (made-up depths: 30x everywhere, the KIV-2 block at 20 repeats) and +# runs og for real on the host (bin/og): +# - each step calls og cn bed once and step 35 and 46 call og cn call once +# (step 41's report imports bin/copy_number.py), step 46 with the 4925G>A +# marker in the same call, and no step calls the old Python any more; # - the tables say what the depths give: four alpha-globin genes, RhD D+ # with two copies, 20.0 KIV-2 repeats; # - a control segment with no read (FAKE_ZERO=flank_e19, the others at 30x) # fails step 46 with BROKEN_CONTROL, never a count. +# It needs og built (scripts/setup.sh --og). The fake-docker job builds no og, +# so there the case says it is skipped; Guard's rust job runs it with og and +# fails on that line. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" # shellcheck source=../../versions.env . "${REPO_ROOT}/versions.env" +if ! "${REPO_ROOT}/bin/og" version > /dev/null 2>&1; then + echo "skip: bin/og is not built here (Guard's rust job runs this case with og)" + exit 0 +fi export GENOME_DIR="${CASE_WORK}/genome" seed_reference "$GENOME_DIR" @@ -70,7 +77,7 @@ case "$*" in "parascopy cn-using"*) o=$(host_path "$(arg -o "$@")"); mkdir -p "$o" printf 'chr5\t70925087\t70953015\tSMN1\t.\tPASS\t4\t40\tPASS\t2,2\t30\t.\tchr5:70049669-70077595\n' | gzip -c > "${o}/res.samples.bed.gz" ;; - "python3 /pgp-bin/"*) + "python3 /pgp-bin/"* | "/pgp-bin/og "*) args=(); for a in "$@"; do args+=("$(map "$a")"); done exec "${args[@]}" ;; "sh -c "*) @@ -101,9 +108,9 @@ val() { awk -F'\t' -v k="$2" '$1 == k {print $2; exit}' "$1"; } # --- step 35 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 paralogs "${SCRIPTS}/35-paralogs.sh" sample1 -[ "$(calls 'copy_number.py bed --locus HBA')" = 1 ] || fail "step 35: copy_number.py bed --locus HBA was not called once" -[ "$(calls 'copy_number.py call --locus HBA')" = 1 ] || fail "step 35: copy_number.py call --locus HBA was not called once" -[ "$(calls 'hba_copy_number.py')" = 0 ] || fail "step 35 still calls hba_copy_number.py" +[ "$(calls 'og cn bed --locus HBA')" = 1 ] || fail "step 35: og cn bed --locus HBA was not called once" +[ "$(calls 'og cn call --locus HBA')" = 1 ] || fail "step 35: og cn call --locus HBA was not called once" +[ "$(calls '.py')" = 0 ] || fail "step 35 still calls Python: $(grep -F '.py' "$FAKE_DOCKER_LOG")" H="${S}/paralogs/sample1_hba_copy_number.tsv" [ "$(val "$H" alpha_globin_genes)" = 4 ] || fail "step 35: four alpha-globin genes expected: $(cat "$H")" [ "$(val "$H" status)" = called ] || fail "step 35: HBA not called: $(cat "$H")" @@ -111,7 +118,7 @@ H="${S}/paralogs/sample1_hba_copy_number.tsv" # --- step 41 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 bloodtype "${SCRIPTS}/41-bloodtype.sh" sample1 -[ "$(calls 'copy_number.py bed --locus RHD')" = 1 ] || fail "step 41: copy_number.py bed --locus RHD was not called once" +[ "$(calls 'og cn bed --locus RHD')" = 1 ] || fail "step 41: og cn bed --locus RHD was not called once" [ "$(awk -F'\t' '$4 == "RHD_GENE" {n++} END {print n + 0}' "${S}/bloodtype/sample1_bloodtype_depth.tsv")" = 1 ] \ || fail "step 41: the depth table has no RHD_GENE row" R=$(awk -F'\t' '$1 == "RhD"' "${S}/bloodtype/sample1_bloodtype.tsv") @@ -120,11 +127,14 @@ case "$R" in *"D+"*"RHD/RHD (RHD copies by read depth 2.00)"*) ;; *) fail "step # --- step 46 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 lpa "${SCRIPTS}/46-lpa.sh" sample1 -[ "$(calls 'copy_number.py bed --locus LPA_KIV2')" = 1 ] || fail "step 46: copy_number.py bed --locus LPA_KIV2 was not called once" -[ "$(calls 'copy_number.py call --locus LPA_KIV2')" = 1 ] || fail "step 46: copy_number.py call --locus LPA_KIV2 was not called once" -[ "$(calls 'lpa_kiv2.py')" = 0 ] || fail "step 46 still calls lpa_kiv2.py" +[ "$(calls 'og cn bed --locus LPA_KIV2')" = 1 ] || fail "step 46: og cn bed --locus LPA_KIV2 was not called once" +[ "$(calls 'og cn call --locus LPA_KIV2')" = 1 ] || fail "step 46: og cn call --locus LPA_KIV2 was not called once" +[ "$(calls '--marker LPA_KIV2_4925')" = 1 ] || fail "step 46: the 4925G>A marker was not read in the count's call" +[ "$(calls '.py')" = 0 ] || fail "step 46 still calls Python: $(grep -F '.py' "$FAKE_DOCKER_LOG")" L="${S}/lpa/sample1_lpa_kiv2.tsv" [ "$(val "$L" kiv2_copies)" = 20.0 ] || fail "step 46: 20.0 KIV-2 repeats expected: $(cat "$L")" +N="${S}/lpa/sample1_lpa_kiv2_4925.tsv" +[ -n "$(val "$N" status)" ] || fail "step 46: no 4925G>A table beside the count: $(cat "$N" 2>/dev/null)" # A control exon with no read while the others have 30x: a named error, no table FAKE_ZERO=flank_e19 run_expect 3 lpa-broken "${SCRIPTS}/46-lpa.sh" sample1 diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 1bf7668ea..0c215d477 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -563,3 +563,77 @@ exomiser report-family exomiser/family.tsv,exomiser/terms.tsv,exomiser/S1_exomis exomiser report-missing-column exomiser/no-acmg.tsv,exomiser/terms.tsv report --sample S1 --variants no-acmg.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE exomiser report-short-line exomiser/short.tsv,exomiser/terms.tsv report --sample S1 --variants short.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE exomiser report-empty-file exomiser/empty.tsv,exomiser/terms.tsv report --sample S1 --variants empty.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE +cn bed-all gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv - kv PYTHON_IMAGE +cn bed-loci gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --locus HBA --locus RHD - kv PYTHON_IMAGE +cn bed-interval gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --kind interval - kv PYTHON_IMAGE +cn bed-psv gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --psv --locus CYP21A2 - kv PYTHON_IMAGE +cn bed-psv-not-psv gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --psv --locus HBA - kv PYTHON_IMAGE +cn bed-unknown gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --locus HBA --locus NOPE - kv PYTHON_IMAGE +cn hba4 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba3 gen:cn-oracle.sh,cn/oracle.sh,cn/hba3.bedcov.tsv,cn/hba3.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba3.bedcov.tsv --bedcov-mapq1 hba3.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-between gen:cn-oracle.sh,cn/oracle.sh,cn/hba-between.bedcov.tsv,cn/hba-between.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-between.bedcov.tsv --bedcov-mapq1 hba-between.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-lowflank gen:cn-oracle.sh,cn/oracle.sh,cn/hba-lowflank.bedcov.tsv,cn/hba-lowflank.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-lowflank.bedcov.tsv --bedcov-mapq1 hba-lowflank.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-disagree gen:cn-oracle.sh,cn/oracle.sh,cn/hba-disagree.bedcov.tsv,cn/hba-disagree.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-disagree.bedcov.tsv --bedcov-mapq1 hba-disagree.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/hba-mapq0.bedcov.tsv,cn/hba-mapq0.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-mapq0.bedcov.tsv --bedcov-mapq1 hba-mapq0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba0 gen:cn-oracle.sh,cn/oracle.sh,cn/hba0.bedcov.tsv,cn/hba0.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba0.bedcov.tsv --bedcov-mapq1 hba0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba5 gen:cn-oracle.sh,cn/oracle.sh,cn/hba5.bedcov.tsv,cn/hba5.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba5.bedcov.tsv --bedcov-mapq1 hba5.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-broken gen:cn-oracle.sh,cn/oracle.sh,cn/hba-broken.bedcov.tsv,cn/hba-broken.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-broken.bedcov.tsv --bedcov-mapq1 hba-broken.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus HBA --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-one-empty gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/empty.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 empty.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-no-mapq1 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-wrong-regions gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-missing-file gen:cn-oracle.sh,cn/oracle.sh call --catalog cn_loci.tsv --locus HBA --bedcov nope.tsv --bedcov-mapq1 nope.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-alt-header gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv,cn/header-alt.sam call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --bam-header header-alt.sam --table t.tsv - kv PYTHON_IMAGE +cn lpa12 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa12.bedcov.tsv,cn/lpa12.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa12.bedcov.tsv --bedcov-mapq1 lpa12.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa29 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa29.bedcov.tsv,cn/lpa29.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa29.bedcov.tsv --bedcov-mapq1 lpa29.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa47 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa47.bedcov.tsv,cn/lpa47.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa47.bedcov.tsv --bedcov-mapq1 lpa47.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-lowflank gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-lowflank.bedcov.tsv,cn/lpa-lowflank.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-lowflank.bedcov.tsv --bedcov-mapq1 lpa-lowflank.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-mapq0.bedcov.tsv,cn/lpa-mapq0.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-mapq0.bedcov.tsv --bedcov-mapq1 lpa-mapq0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-floor gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-floor.bedcov.tsv,cn/lpa-floor.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-floor.bedcov.tsv --bedcov-mapq1 lpa-floor.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn marker-lpa29-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa29.bedcov.tsv,cn/lpa29.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa29.bedcov.tsv --bedcov-mapq1 lpa29.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa12-kiv-notseen gen:cn-oracle.sh,cn/oracle.sh,cn/lpa12.bedcov.tsv,cn/lpa12.q1.tsv,cn/kiv-notseen.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa12.bedcov.tsv --bedcov-mapq1 lpa12.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-notseen.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa47-bad gen:cn-oracle.sh,cn/oracle.sh,cn/lpa47.bedcov.tsv,cn/lpa47.q1.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa47.bedcov.tsv --bedcov-mapq1 lpa47.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup bad.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa-mapq0-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-mapq0.bedcov.tsv,cn/lpa-mapq0.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-mapq0.bedcov.tsv --bedcov-mapq1 lpa-mapq0.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa-share1996-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-share1996.bedcov.tsv,cn/lpa-share1996.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-share1996.bedcov.tsv --bedcov-mapq1 lpa-share1996.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-empty-chr6 gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup empty.tsv --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn dosage-hp3 gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HP --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-ccl1 gen:cn-oracle.sh,cn/oracle.sh,cn/ccl1.bedcov.tsv,cn/ccl1.q1.tsv call --catalog cn_loci.tsv --locus CCL3L1 --bedcov ccl1.bedcov.tsv --bedcov-mapq1 ccl1.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-lce0 gen:cn-oracle.sh,cn/oracle.sh,cn/lce0.bedcov.tsv,cn/lce0.q1.tsv call --catalog cn_loci.tsv --locus LCE3BC --bedcov lce0.bedcov.tsv --bedcov-mapq1 lce0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-rhd1 gen:cn-oracle.sh,cn/oracle.sh,cn/rhd1.bedcov.tsv call --catalog cn_loci.tsv --locus RHD --bedcov rhd1.bedcov.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-rhd0 gen:cn-oracle.sh,cn/oracle.sh,cn/rhd0.bedcov.tsv call --catalog cn_loci.tsv --locus RHD --bedcov rhd0.bedcov.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-normal gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv,cn/cyp-normal.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --pileup cyp-normal.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-deletion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-deletion.bedcov.tsv,cn/cyp-deletion.q1.tsv,cn/cyp-deletion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-deletion.bedcov.tsv --bedcov-mapq1 cyp-deletion.q1.tsv --pileup cyp-deletion.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-conversion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-conversion.bedcov.tsv,cn/cyp-conversion.q1.tsv,cn/cyp-conversion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-conversion.bedcov.tsv --bedcov-mapq1 cyp-conversion.q1.tsv --pileup cyp-conversion.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-mapq0.q1.tsv,cn/cyp-normal.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-mapq0.q1.tsv --pileup cyp-normal.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --pileup bad.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-no-pileup gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn cyp-bad-pileup gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --pileup bad.pileup --table t.tsv - kv PYTHON_IMAGE +cn intervals-male gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex male --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-female gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex female --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-unknown gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex unknown --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-failed gen:cn-oracle.sh,cn/oracle.sh intervals --catalog cn_loci.tsv --failed --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-no-sex gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --out iv.tsv - kv PYTHON_IMAGE +cn intervals-wrong-table gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov hba4.bedcov.tsv --sex male --out iv.tsv - kv PYTHON_IMAGE +cn check-bam-alt gen:cn-oracle.sh,cn/oracle.sh,cn/header-alt.sam check-bam --header header-alt.sam - kv PYTHON_IMAGE +cn check-bam-clean gen:cn-oracle.sh,cn/oracle.sh,cn/header-clean.sam check-bam --header header-clean.sam - kv PYTHON_IMAGE +cn check-bam-stdin gen:cn-oracle.sh,cn/oracle.sh,cn/stdin check-bam --header - - kv PYTHON_IMAGE +cn marker-regions-lpa gen:cn-oracle.sh,cn/oracle.sh marker-regions LPA_KIV2_4925 - kv PYTHON_IMAGE +cn marker-regions-smn gen:cn-oracle.sh,cn/oracle.sh marker-regions SMN1_27134 - kv PYTHON_IMAGE +cn smn-smn-present-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-notseen-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-notseen.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-notseen.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn1 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn1.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn1.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn-lowq gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn-lowq.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn-lowq.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn-two-rows gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn-two-rows.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn-two-rows.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-bad-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/bad.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup bad.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-no-table gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup marker-read SMN1_27134 --pileup smn-present.pileup --out m.tsv m.tsv kv PYTHON_IMAGE +cn lpa-share1996 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-share1996.bedcov.tsv,cn/lpa-share1996.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-share1996.bedcov.tsv --bedcov-mapq1 lpa-share1996.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-marker-read-table gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-table.tsv,cn/kiv-present.pileup marker-read LPA_KIV2_4925 --pileup kiv-present.pileup --copy-table lpa-table.tsv --out n.tsv --catalog cn_loci.tsv n.tsv kv PYTHON_IMAGE +cn lpa-marker-read-no-table gen:cn-oracle.sh,cn/oracle.sh,cn/kiv-present.pileup marker-read LPA_KIV2_4925 --pileup kiv-present.pileup --out n.tsv --catalog cn_loci.tsv n.tsv kv PYTHON_IMAGE +cn py311-hba4 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-hba-between gen:cn-oracle.sh,cn/oracle.sh,cn/hba-between.bedcov.tsv,cn/hba-between.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-between.bedcov.tsv --bedcov-mapq1 hba-between.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-dosage-hp3 gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HP --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-cyp-conversion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-conversion.bedcov.tsv,cn/cyp-conversion.q1.tsv,cn/cyp-conversion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-conversion.bedcov.tsv --bedcov-mapq1 cyp-conversion.q1.tsv --pileup cyp-conversion.pileup --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-intervals-male gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex male --out iv.tsv iv.tsv tsv PARASCOPY_IMAGE +cn py311-smn-present gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PARASCOPY_IMAGE +cn py311-check-bam-alt gen:cn-oracle.sh,cn/oracle.sh,cn/header-alt.sam check-bam --header header-alt.sam - kv PARASCOPY_IMAGE diff --git a/tests/parity/gen/cn-oracle.sh b/tests/parity/gen/cn-oracle.sh new file mode 100644 index 000000000..92a487dda --- /dev/null +++ b/tests/parity/gen/cn-oracle.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# The Python that og cn replaced, as it was at the commit tests/parity/oracle.ref +# names for cn, for the cn oracle (inputs/cn/oracle.sh): copy_number.py imports +# hba_copy_number.py and lpa_kiv2.py, the markers lived in traits_report.py, +# and every case reads that commit's bin/cn_loci.tsv. +set -euo pipefail +root=$(cd "$(dirname "$0")/../../.." && pwd) +sha=$(awk -F'\t' '$1 == "cn" {print $2}' "${root}/tests/parity/oracle.ref") +[ -n "$sha" ] || { echo "cn-oracle.sh: no cn row in oracle.ref" >&2; exit 1; } +git -C "$root" cat-file -e "${sha}^{commit}" 2> /dev/null || git -C "$root" fetch -q --depth=1 origin "$sha" +for f in hba_copy_number.py lpa_kiv2.py traits_report.py cn_loci.tsv; do + git -C "$root" show "${sha}:bin/${f}" > "$f" +done diff --git a/tests/parity/golden/cn/cyp-no-pileup/stderr b/tests/parity/golden/cn/cyp-no-pileup/stderr new file mode 100644 index 000000000..fb715e0ff --- /dev/null +++ b/tests/parity/golden/cn/cyp-no-pileup/stderr @@ -0,0 +1 @@ +ERROR: CYP21A2 is a psv row: --pileup (samtools mpileup over `og cn bed --psv`) is needed diff --git a/tests/parity/golden/cn/hba-wrong-regions/stderr b/tests/parity/golden/cn/hba-wrong-regions/stderr new file mode 100644 index 000000000..ce426b040 --- /dev/null +++ b/tests/parity/golden/cn/hba-wrong-regions/stderr @@ -0,0 +1 @@ +ERROR: hp3.bedcov.tsv: its regions differ from the ones `og cn bed --locus HBA` prints diff --git a/tests/parity/golden/cn/intervals-wrong-table/stderr b/tests/parity/golden/cn/intervals-wrong-table/stderr new file mode 100644 index 000000000..491caeebd --- /dev/null +++ b/tests/parity/golden/cn/intervals-wrong-table/stderr @@ -0,0 +1 @@ +ERROR: the bedcov table lacks some of 1q21.1_TAR_syndrome_proximal_BP2-BP3's segments: its regions differ from `og cn bed --kind interval` diff --git a/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv new file mode 100644 index 000000000..933da31f0 --- /dev/null +++ b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv @@ -0,0 +1,10 @@ +metric value +marker LPA KIV-2 4925G>A +rsid rs1484919544 (first KIV-2 copy) +status not assessed +alt_reads . +depth . +fraction . +copies . +kiv2_copies . +message LPA KIV-2 4925G>A (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017): not assessed, reads of mapping quality 0 make up 20.0% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives. diff --git a/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout new file mode 100644 index 000000000..354d95fca --- /dev/null +++ b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout @@ -0,0 +1,2 @@ +LPA KIV-2: not assessed: reads of mapping quality 0 make up 20.0% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM (KIV-2 exons 68.57x, single-copy exons 30.00x, MAPQ 0 share 0.200) +LPA LPA KIV-2 4925G>A (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017): not assessed, reads of mapping quality 0 make up 20.0% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives. diff --git a/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv new file mode 100644 index 000000000..9e7eb9363 --- /dev/null +++ b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv @@ -0,0 +1,11 @@ +metric value +marker SMN1 g.27134T>G +rsid rs143838139 +status present +alt_reads 5 +depth 60 +fraction 0.0833 +copies 0.3 +smn1_pscn 1 (quality 12, filter LowQual), not resolved +agcn 4 +message SMN1 g.27134T>G (rs143838139): present, 5 of 60 reads carry G at the SMN1 and SMN2 copies of the site together (about 0.3 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or SMN2. SMN1 copy number: 1 (quality 12, filter LowQual), not resolved; the marker refines the carrier risk of a two-copy result only. This marker does not prove 2+0, and its absence does not rule 2+0 out. diff --git a/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout new file mode 100644 index 000000000..35fcc0935 --- /dev/null +++ b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout @@ -0,0 +1 @@ +SMN1 g.27134T>G (rs143838139): present, 5 of 60 reads carry G at the SMN1 and SMN2 copies of the site together (about 0.3 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or SMN2. SMN1 copy number: 1 (quality 12, filter LowQual), not resolved; the marker refines the carrier risk of a two-copy result only. This marker does not prove 2+0, and its absence does not rule 2+0 out. diff --git a/tests/parity/inputs/cn/bad.pileup b/tests/parity/inputs/cn/bad.pileup new file mode 100644 index 000000000..1a14d3e27 --- /dev/null +++ b/tests/parity/inputs/cn/bad.pileup @@ -0,0 +1 @@ +# samtools mpileup failed: 4925G>A not assessed diff --git a/tests/parity/inputs/cn/ccl1.bedcov.tsv b/tests/parity/inputs/cn/ccl1.bedcov.tsv new file mode 100644 index 000000000..012420305 --- /dev/null +++ b/tests/parity/inputs/cn/ccl1.bedcov.tsv @@ -0,0 +1,3 @@ +chr17 36112000 36148000 CCL3L_FLANK_L 1080000 +chr17 36194868 36196758 CCL3L3 28350 +chr17 36460000 36500000 CCL3L_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/ccl1.q1.tsv b/tests/parity/inputs/cn/ccl1.q1.tsv new file mode 100644 index 000000000..9003d9ad9 --- /dev/null +++ b/tests/parity/inputs/cn/ccl1.q1.tsv @@ -0,0 +1,3 @@ +chr17 36112000 36148000 CCL3L_FLANK_L 540000 +chr17 36194868 36196758 CCL3L3 14175 +chr17 36460000 36500000 CCL3L_FLANK_R 600000 diff --git a/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv b/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-conversion.pileup b/tests/parity/inputs/cn/cyp-conversion.pileup new file mode 100644 index 000000000..bb2216f14 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 12 ^]Cc+2AGCgGgGgGgGg$ IIIIIIIIIIII +chr6 32006690 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007074 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007374 N 12 ^]Gg+2AGGgGgTtTtTt$ IIIIIIIIIIII +chr6 32007686 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32007800 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32039081 N 48 ^]Cc+2AGCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 48 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-conversion.q1.tsv b/tests/parity/inputs/cn/cyp-conversion.q1.tsv new file mode 100644 index 000000000..b0cb3dfc5 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 72603 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv b/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv new file mode 100644 index 000000000..0b4604b7b --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 40335 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-deletion.pileup b/tests/parity/inputs/cn/cyp-deletion.pileup new file mode 100644 index 000000000..ef5331e7a --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 9 ^]Cc+2AGCgGgGgG$ IIIIIIIII +chr6 32006690 N 9 ^]Tt+2AGTaAaAaA$ IIIIIIIII +chr6 32007074 N 9 ^]Tt+2AGTtTtAaA$ IIIIIIIII +chr6 32007374 N 9 ^]Gg+2AGGgGgTtT$ IIIIIIIII +chr6 32007686 N 9 ^]Cc+2AGCcCcTtT$ IIIIIIIII +chr6 32007800 N 9 ^]Cc+2AGCcCcTtT$ IIIIIIIII +chr6 32039081 N 36 ^]Cc+2AGCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 36 ^]Tt+2AGTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 36 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 36 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 36 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 36 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-deletion.q1.tsv b/tests/parity/inputs/cn/cyp-deletion.q1.tsv new file mode 100644 index 000000000..f2528c3ac --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 36302 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/cyp-mapq0.q1.tsv b/tests/parity/inputs/cn/cyp-mapq0.q1.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-mapq0.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-normal.bedcov.tsv b/tests/parity/inputs/cn/cyp-normal.bedcov.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-normal.pileup b/tests/parity/inputs/cn/cyp-normal.pileup new file mode 100644 index 000000000..72b23db42 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 12 ^]Cc+2AGCcCcGgGgGg$ IIIIIIIIIIII +chr6 32006690 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007074 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007374 N 12 ^]Gg+2AGGgGgTtTtTt$ IIIIIIIIIIII +chr6 32007686 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32007800 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32039081 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 48 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-normal.q1.tsv b/tests/parity/inputs/cn/cyp-normal.q1.tsv new file mode 100644 index 000000000..b0cb3dfc5 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 72603 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/empty.tsv b/tests/parity/inputs/cn/empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/cn/hba-between.bedcov.tsv b/tests/parity/inputs/cn/hba-between.bedcov.tsv new file mode 100644 index 000000000..988dc3fe5 --- /dev/null +++ b/tests/parity/inputs/cn/hba-between.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-between.q1.tsv b/tests/parity/inputs/cn/hba-between.q1.tsv new file mode 100644 index 000000000..445c896d2 --- /dev/null +++ b/tests/parity/inputs/cn/hba-between.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 14091 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba-broken.bedcov.tsv b/tests/parity/inputs/cn/hba-broken.bedcov.tsv new file mode 100644 index 000000000..16000c2ed --- /dev/null +++ b/tests/parity/inputs/cn/hba-broken.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 0 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-broken.q1.tsv b/tests/parity/inputs/cn/hba-broken.q1.tsv new file mode 100644 index 000000000..e8c4e8b7b --- /dev/null +++ b/tests/parity/inputs/cn/hba-broken.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 0 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba-disagree.bedcov.tsv b/tests/parity/inputs/cn/hba-disagree.bedcov.tsv new file mode 100644 index 000000000..20b8343f3 --- /dev/null +++ b/tests/parity/inputs/cn/hba-disagree.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 600000 diff --git a/tests/parity/inputs/cn/hba-disagree.q1.tsv b/tests/parity/inputs/cn/hba-disagree.q1.tsv new file mode 100644 index 000000000..7b20f3051 --- /dev/null +++ b/tests/parity/inputs/cn/hba-disagree.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 450000 diff --git a/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv b/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv new file mode 100644 index 000000000..f55852795 --- /dev/null +++ b/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 240000 +chr16 172875 173710 HBA2 4175 +chr16 176679 177522 HBA1 4215 +chr16 190000 230000 flank_right 240000 diff --git a/tests/parity/inputs/cn/hba-lowflank.q1.tsv b/tests/parity/inputs/cn/hba-lowflank.q1.tsv new file mode 100644 index 000000000..4e340cfde --- /dev/null +++ b/tests/parity/inputs/cn/hba-lowflank.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 180000 +chr16 172875 173710 HBA2 3131 +chr16 176679 177522 HBA1 3161 +chr16 190000 230000 flank_right 180000 diff --git a/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv b/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv new file mode 100644 index 000000000..c04168f01 --- /dev/null +++ b/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-mapq0.q1.tsv b/tests/parity/inputs/cn/hba-mapq0.q1.tsv new file mode 100644 index 000000000..b0950dddc --- /dev/null +++ b/tests/parity/inputs/cn/hba-mapq0.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1188000 +chr16 172875 173710 HBA2 24800 +chr16 176679 177522 HBA1 25037 +chr16 190000 230000 flank_right 1188000 diff --git a/tests/parity/inputs/cn/hba0.bedcov.tsv b/tests/parity/inputs/cn/hba0.bedcov.tsv new file mode 100644 index 000000000..e88ac074c --- /dev/null +++ b/tests/parity/inputs/cn/hba0.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 0 +chr16 176679 177522 HBA1 0 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba0.q1.tsv b/tests/parity/inputs/cn/hba0.q1.tsv new file mode 100644 index 000000000..e88ac074c --- /dev/null +++ b/tests/parity/inputs/cn/hba0.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 0 +chr16 176679 177522 HBA1 0 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba3.bedcov.tsv b/tests/parity/inputs/cn/hba3.bedcov.tsv new file mode 100644 index 000000000..3828c69a2 --- /dev/null +++ b/tests/parity/inputs/cn/hba3.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 12525 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba3.q1.tsv b/tests/parity/inputs/cn/hba3.q1.tsv new file mode 100644 index 000000000..4392cb0fd --- /dev/null +++ b/tests/parity/inputs/cn/hba3.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 9394 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba4.bedcov.tsv b/tests/parity/inputs/cn/hba4.bedcov.tsv new file mode 100644 index 000000000..c04168f01 --- /dev/null +++ b/tests/parity/inputs/cn/hba4.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba4.q1.tsv b/tests/parity/inputs/cn/hba4.q1.tsv new file mode 100644 index 000000000..34a3baeb5 --- /dev/null +++ b/tests/parity/inputs/cn/hba4.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba5.bedcov.tsv b/tests/parity/inputs/cn/hba5.bedcov.tsv new file mode 100644 index 000000000..860ea454b --- /dev/null +++ b/tests/parity/inputs/cn/hba5.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 37575 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba5.q1.tsv b/tests/parity/inputs/cn/hba5.q1.tsv new file mode 100644 index 000000000..f8980fade --- /dev/null +++ b/tests/parity/inputs/cn/hba5.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 28181 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/header-alt.sam b/tests/parity/inputs/cn/header-alt.sam new file mode 100644 index 000000000..fb5b18f5c --- /dev/null +++ b/tests/parity/inputs/cn/header-alt.sam @@ -0,0 +1,5 @@ +@HD VN:1.6 +@SQ SN:chr1 LN:248956422 +@SQ SN:chr1_KI270706v1_random LN:175055 +@SQ SN:chr6_GL000250v2_alt LN:4672374 +@SQ SN:HLA-A*01:01:01:01 LN:3503 diff --git a/tests/parity/inputs/cn/header-clean.sam b/tests/parity/inputs/cn/header-clean.sam new file mode 100644 index 000000000..2ee1fce7e --- /dev/null +++ b/tests/parity/inputs/cn/header-clean.sam @@ -0,0 +1,4 @@ +@HD VN:1.6 +@SQ SN:chr1 LN:248956422 +@SQ SN:chrM LN:16569 +@PG ID:x PN:HLA-x_alt diff --git a/tests/parity/inputs/cn/hp3.bedcov.tsv b/tests/parity/inputs/cn/hp3.bedcov.tsv new file mode 100644 index 000000000..4f4acb262 --- /dev/null +++ b/tests/parity/inputs/cn/hp3.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 72005000 72045000 HP_FLANK_L 1200000 +chr16 72056529 72057466 HP_UNIT1 21082 +chr16 72058253 72059190 HP_UNIT2 21082 +chr16 72085000 72125000 HP_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/hp3.q1.tsv b/tests/parity/inputs/cn/hp3.q1.tsv new file mode 100644 index 000000000..0f4fcbe43 --- /dev/null +++ b/tests/parity/inputs/cn/hp3.q1.tsv @@ -0,0 +1,4 @@ +chr16 72005000 72045000 HP_FLANK_L 600000 +chr16 72056529 72057466 HP_UNIT1 10541 +chr16 72058253 72059190 HP_UNIT2 10541 +chr16 72085000 72125000 HP_FLANK_R 600000 diff --git a/tests/parity/inputs/cn/intervals.bedcov.tsv b/tests/parity/inputs/cn/intervals.bedcov.tsv new file mode 100644 index 000000000..ada5c732f --- /dev/null +++ b/tests/parity/inputs/cn/intervals.bedcov.tsv @@ -0,0 +1,946 @@ +chr1 143034587 143134587 1q21.1_TAR_syndrome_proximal_BP2-BP3_L 3000000 +chr1 145686999 145809382 1q21.1_TAR_syndrome_proximal_BP2-BP3_t1 1835745 +chr1 145823828 145923649 1q21.1_TAR_syndrome_proximal_BP2-BP3_t2 1497315 +chr1 145927476 146048495 1q21.1_TAR_syndrome_proximal_BP2-BP3_t3 1815285 +chr1 149926380 150026380 1q21.1_TAR_syndrome_proximal_BP2-BP3_R 3000000 +chr1 143034587 143134587 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_L 3000000 +chr1 145687000 145809382 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t1 3671460 +chr1 145823828 145923649 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t2 2994630 +chr1 145927476 146048496 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t3 3630600 +chr1 147105903 147473859 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t4 11038680 +chr1 147473859 147841815 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t5 11038680 +chr1 147842996 147917509 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t6 2235390 +chr1 149926380 150026380 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_R 3000000 +chr1 143034587 143134587 1q21.1_distal_BP3-BP4_L 3000000 +chr1 147105904 147473860 1q21.1_distal_BP3-BP4_t1 11038680 +chr1 147473860 147841815 1q21.1_distal_BP3-BP4_t2 11038650 +chr1 147842996 147917509 1q21.1_distal_BP3-BP4_t3 2235390 +chr1 149926380 150026380 1q21.1_distal_BP3-BP4_R 3000000 +chr2 86554206 86654206 2p11.2_L 3000000 +chr2 88022824 88389269 2p11.2_t1 10993350 +chr2 88389269 88755713 2p11.2_t2 10993320 +chr2 90405714 90505714 2p11.2_R 3000000 +chr2 95249415 95349415 2q11.2_L 3000000 +chr2 96073264 96539478 2q11.2_t1 13986420 +chr2 96539478 97005692 2q11.2_t2 13986420 +chr2 97693145 97793145 2q11.2_R 3000000 +chr2 109561396 109661396 2q13_proximal_L 3000000 +chr2 110104531 110185037 2q13_proximal_t1 3622770 +chr2 110186837 110228181 2q13_proximal_t2 1860480 +chr2 112509875 112609875 2q13_proximal_R 3000000 +chr2 109561396 109661396 2q13_distal_L 3000000 +chr2 110634616 110635231 2q13_distal_t1 18450 +chr2 110635407 110695526 2q13_distal_t2 1803570 +chr2 110699475 110967774 2q13_distal_t3 8048970 +chr2 110968796 111223194 2q13_distal_t4 7631940 +chr2 111225336 111252925 2q13_distal_t5 827670 +chr2 111519068 111525222 2q13_distal_t6 184620 +chr2 111615296 111719716 2q13_distal_t7 3132600 +chr2 111882691 112278330 2q13_distal_t8 11869170 +chr2 112287893 112291400 2q13_distal_t9 105210 +chr2 112293808 112347165 2q13_distal_t10 1600710 +chr2 112509875 112609875 2q13_distal_R 3000000 +chr2 129811225 129911225 2q21.1_L 3000000 +chr2 130719936 131172120 2q21.1_t1 13565520 +chr2 132414062 132514062 2q21.1_R 3000000 +chr3 195370170 195470170 3q29_L 3000000 +chr3 196029183 196395167 3q29_t1 10979520 +chr3 196395167 196761150 3q29_t2 10979490 +chr3 196761870 196828315 3q29_t3 1993350 +chr3 196832644 196986594 3q29_t4 4618500 +chr3 196987637 197032456 3q29_t5 1344570 +chr3 197035076 197436093 3q29_t6 12030510 +chr3 197438625 197617791 3q29_t7 5374980 +chr3 197770554 197870554 3q29_R 3000000 +chr5 175752456 175852456 5q35_Sotos_syndrome_L 0 +chr5 176301976 176346064 5q35_Sotos_syndrome_t1 1322640 +chr5 176348356 176609606 5q35_Sotos_syndrome_t2 7837500 +chr5 176609606 176870856 5q35_Sotos_syndrome_t3 7837500 +chr5 176873194 177156949 5q35_Sotos_syndrome_t4 8512650 +chr5 177156949 177440704 5q35_Sotos_syndrome_t5 8512650 +chr5 177442622 177620792 5q35_Sotos_syndrome_t6 5345100 +chr5 178185883 178285883 5q35_Sotos_syndrome_R 3000000 +chr7 72415064 72515064 7q11.23_Williams-Beuren_syndrome_L 3000000 +chr7 73330452 73373003 7q11.23_Williams-Beuren_syndrome_t1 1276530 +chr7 73380068 73398750 7q11.23_Williams-Beuren_syndrome_t2 560460 +chr7 73417318 73440409 7q11.23_Williams-Beuren_syndrome_t3 692730 +chr7 73442040 73900880 7q11.23_Williams-Beuren_syndrome_t4 13765200 +chr7 73911711 73943245 7q11.23_Williams-Beuren_syndrome_t5 946020 +chr7 73973000 73982740 7q11.23_Williams-Beuren_syndrome_t6 292200 +chr7 73990695 73993203 7q11.23_Williams-Beuren_syndrome_t7 75240 +chr7 73995261 74194738 7q11.23_Williams-Beuren_syndrome_t8 5984310 +chr7 74197101 74536524 7q11.23_Williams-Beuren_syndrome_t9 10182690 +chr7 74565485 74567048 7q11.23_Williams-Beuren_syndrome_t10 46890 +chr7 74571222 74576545 7q11.23_Williams-Beuren_syndrome_t11 159690 +chr7 74599408 74650287 7q11.23_Williams-Beuren_syndrome_t12 1526370 +chr7 74650795 74728173 7q11.23_Williams-Beuren_syndrome_t13 2321340 +chr7 77185726 77285726 7q11.23_Williams-Beuren_syndrome_R 3000000 +chr7 72415790 72515790 7q11.23_L 3000000 +chr7 75528718 75922712 7q11.23_t1 11819820 +chr7 75924249 76071493 7q11.23_t2 4417320 +chr7 76073808 76087747 7q11.23_t3 418170 +chr7 76105300 76105897 7q11.23_t4 17910 +chr7 76177734 76433859 7q11.23_t5 7683750 +chr7 77229655 77329655 7q11.23_R 3000000 +chr8 6826263 6926263 8p23.1_L 3000000 +chr8 8242542 8631046 8p23.1_t1 11655120 +chr8 8631046 9019550 8p23.1_t2 11655120 +chr8 9019550 9408054 8p23.1_t3 11655120 +chr8 9408054 9796558 8p23.1_t4 11655120 +chr8 9799208 10151922 8p23.1_t5 10581420 +chr8 10151922 10504636 8p23.1_t6 10581420 +chr8 10506824 10918077 8p23.1_t7 12337590 +chr8 10918077 11329329 8p23.1_t8 12337560 +chr8 11332201 11624615 8p23.1_t9 8772420 +chr8 11657572 11758866 8p23.1_t10 3038820 +chr8 11760002 11790306 8p23.1_t11 909120 +chr8 11793265 11908820 8p23.1_t12 3466650 +chr8 12769873 12869873 8p23.1_R 3000000 +chr10 45521834 45621834 10q11.2_LCR_C-LCR_D_L 3000000 +chr10 48181660 48588063 10q11.2_LCR_C-LCR_D_t1 12192090 +chr10 48588063 48994466 10q11.2_LCR_C-LCR_D_t2 12192090 +chr10 48994466 49400869 10q11.2_LCR_C-LCR_D_t3 12192090 +chr10 49403230 49515152 10q11.2_LCR_C-LCR_D_t4 3357660 +chr10 49517557 49771065 10q11.2_LCR_C-LCR_D_t5 7605240 +chr10 49774433 49778835 10q11.2_LCR_C-LCR_D_t6 132060 +chr10 49781219 49845537 10q11.2_LCR_C-LCR_D_t7 1929540 +chr10 50834350 50934350 10q11.2_LCR_C-LCR_D_R 3000000 +chr10 45521834 45621834 10q11.2_LCR_C-LCR_F_L 3000000 +chr10 48181661 48588064 10q11.2_LCR_C-LCR_F_t1 12192090 +chr10 48588064 48994467 10q11.2_LCR_C-LCR_F_t2 12192090 +chr10 48994467 49400869 10q11.2_LCR_C-LCR_F_t3 12192060 +chr10 49403230 49515152 10q11.2_LCR_C-LCR_F_t4 3357660 +chr10 49517557 49771065 10q11.2_LCR_C-LCR_F_t5 7605240 +chr10 49774433 49778835 10q11.2_LCR_C-LCR_F_t6 132060 +chr10 49781219 49845538 10q11.2_LCR_C-LCR_F_t7 1929570 +chr10 50201000 50628504 10q11.2_LCR_C-LCR_F_t8 12825120 +chr10 50630070 50655358 10q11.2_LCR_C-LCR_F_t9 758640 +chr10 50834350 50934350 10q11.2_LCR_C-LCR_F_R 3000000 +chr10 79400803 79500803 10q22.3q23.2_L 3000000 +chr10 79923087 79982995 10q22.3q23.2_t1 1797240 +chr10 79987131 80009954 10q22.3q23.2_t2 684690 +chr10 80013908 80018239 10q22.3q23.2_t3 129930 +chr10 80045634 80081133 10q22.3q23.2_t4 1064970 +chr10 80103139 80131070 10q22.3q23.2_t5 837930 +chr10 80147027 80209920 10q22.3q23.2_t6 1886790 +chr10 80258685 80700907 10q22.3q23.2_t7 13266660 +chr10 80700907 81143128 10q22.3q23.2_t8 13266630 +chr10 81145195 81603712 10q22.3q23.2_t9 13755510 +chr10 81603712 82062229 10q22.3q23.2_t10 13755510 +chr10 82062229 82520746 10q22.3q23.2_t11 13755510 +chr10 82520746 82979263 10q22.3q23.2_t12 13755510 +chr10 82979263 83437780 10q22.3q23.2_t13 13755510 +chr10 83437780 83896292 10q22.3q23.2_t14 13755360 +chr10 83897369 84081413 10q22.3q23.2_t15 5521320 +chr10 84083161 84390905 10q22.3q23.2_t16 9232320 +chr10 84393707 84679573 10q22.3q23.2_t17 8575980 +chr10 84679573 84965438 10q22.3q23.2_t18 8575950 +chr10 84966462 84968275 10q22.3q23.2_t19 54390 +chr10 84969639 85315330 10q22.3q23.2_t20 10370730 +chr10 85315330 85661021 10q22.3q23.2_t21 10370730 +chr10 85661021 86006710 10q22.3q23.2_t22 10370670 +chr10 86008355 86465864 10q22.3q23.2_t23 13725270 +chr10 86465864 86923373 10q22.3q23.2_t24 13725270 +chr10 86925182 86979631 10q22.3q23.2_t25 1633470 +chr10 87550607 87650607 10q22.3q23.2_R 3000000 +chr11 67556564 67656564 11q13.2q13.4_L 3000000 +chr11 67996175 68021419 11q13.2q13.4_t1 757320 +chr11 68022783 68303534 11q13.2q13.4_t2 8422530 +chr11 68332191 68356675 11q13.2q13.4_t3 734520 +chr11 68392474 68403076 11q13.2q13.4_t4 318060 +chr11 68407506 68890756 11q13.2q13.4_t5 14497500 +chr11 68890756 69374006 11q13.2q13.4_t6 14497500 +chr11 69374006 69857256 11q13.2q13.4_t7 14497500 +chr11 69857256 70340504 11q13.2q13.4_t8 14497440 +chr11 70341528 70344601 11q13.2q13.4_t9 92190 +chr11 70345623 70643160 11q13.2q13.4_t10 8926110 +chr11 70643160 70940696 11q13.2q13.4_t11 8926080 +chr11 70955696 71055696 11q13.2q13.4_t12 3000000 +chr11 71070696 71116951 11q13.2q13.4_t13 1387650 +chr11 71165977 71525885 11q13.2q13.4_t14 10797240 +chr11 71963397 72063397 11q13.2q13.4_R 3000000 +chr13 22651590 22751590 13q12.12_L 3000000 +chr13 22970443 23134172 13q12.12_t1 4911870 +chr13 23135887 23244684 13q12.12_t2 3263910 +chr13 23250456 23422194 13q12.12_t3 5152140 +chr13 23424929 23713633 13q12.12_t4 8661120 +chr13 23715919 23716442 13q12.12_t5 15690 +chr13 23716630 23880085 13q12.12_t6 4903650 +chr13 23904739 23938589 13q12.12_t7 1015500 +chr13 23939257 23940551 13q12.12_t8 38820 +chr13 23946299 23953512 13q12.12_t9 216390 +chr13 23956049 24121032 13q12.12_t10 4949490 +chr13 24123815 24307222 13q12.12_t11 5502210 +chr13 24684489 24784489 13q12.12_R 3000000 +chr15 22782170 23040134 15q11.2_BP1-BP2_t1 7738920 +chr15 34647508 34747508 15q11.2_BP1-BP2_R 3000000 +chr15 22782170 23040135 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t1 7738950 +chr15 23465364 23685679 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t2 6609450 +chr15 23687303 24010500 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t3 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t4 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t5 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t6 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t7 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t8 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t9 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t10 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t11 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t12 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t13 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t14 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t15 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t16 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t17 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t18 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t19 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t20 3099480 +chr15 27999198 28134728 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t21 4065900 +chr15 34647508 34747508 15q11.2q13_PWS_AS_Class_I_BP1-BP3_R 3000000 +chr15 22782171 23040135 15q11.2q13_PWS_AS_BP1-BP4_t1 7738920 +chr15 23465364 23685679 15q11.2q13_PWS_AS_BP1-BP4_t2 6609450 +chr15 23687303 24010500 15q11.2q13_PWS_AS_BP1-BP4_t3 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_BP1-BP4_t4 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_BP1-BP4_t5 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_BP1-BP4_t6 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_BP1-BP4_t7 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_BP1-BP4_t8 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_BP1-BP4_t9 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_BP1-BP4_t10 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_BP1-BP4_t11 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_BP1-BP4_t12 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_BP1-BP4_t13 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_BP1-BP4_t14 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_BP1-BP4_t15 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_BP1-BP4_t16 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_BP1-BP4_t17 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_BP1-BP4_t18 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_BP1-BP4_t19 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_BP1-BP4_t20 3099480 +chr15 27999198 28134729 15q11.2q13_PWS_AS_BP1-BP4_t21 4065930 +chr15 28916885 29053771 15q11.2q13_PWS_AS_BP1-BP4_t22 4106580 +chr15 29055911 29065471 15q11.2q13_PWS_AS_BP1-BP4_t23 286800 +chr15 29069337 29228978 15q11.2q13_PWS_AS_BP1-BP4_t24 4789230 +chr15 29233760 29730610 15q11.2q13_PWS_AS_BP1-BP4_t25 14905500 +chr15 29731821 30005618 15q11.2q13_PWS_AS_BP1-BP4_t26 8213910 +chr15 30006801 30044722 15q11.2q13_PWS_AS_BP1-BP4_t27 1137630 +chr15 30046094 30076787 15q11.2q13_PWS_AS_BP1-BP4_t28 920790 +chr15 34647508 34747508 15q11.2q13_PWS_AS_BP1-BP4_R 3000000 +chr15 23465365 23685679 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t1 6609420 +chr15 23687303 24010500 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t2 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t3 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t4 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t5 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t6 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t7 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t8 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t9 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t10 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t11 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t12 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t13 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t14 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t15 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t16 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t17 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t18 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t19 3099480 +chr15 27999198 28134728 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t20 4065900 +chr15 34647508 34747508 15q11.2q13_PWS_AS_Class_II_BP2-BP3_R 3000000 +chr15 23465366 23685679 15q11.2q13_PWS_AS_BP2-BP4_t1 6609390 +chr15 23687303 24010500 15q11.2q13_PWS_AS_BP2-BP4_t2 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_BP2-BP4_t3 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_BP2-BP4_t4 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_BP2-BP4_t5 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_BP2-BP4_t6 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_BP2-BP4_t7 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_BP2-BP4_t8 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_BP2-BP4_t9 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_BP2-BP4_t10 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_BP2-BP4_t11 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_BP2-BP4_t12 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_BP2-BP4_t13 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_BP2-BP4_t14 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_BP2-BP4_t15 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_BP2-BP4_t16 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_BP2-BP4_t17 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_BP2-BP4_t18 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_BP2-BP4_t19 3099480 +chr15 27999198 28134729 15q11.2q13_PWS_AS_BP2-BP4_t20 4065930 +chr15 28916885 29053771 15q11.2q13_PWS_AS_BP2-BP4_t21 4106580 +chr15 29055911 29065471 15q11.2q13_PWS_AS_BP2-BP4_t22 286800 +chr15 29069337 29228978 15q11.2q13_PWS_AS_BP2-BP4_t23 4789230 +chr15 29233760 29730610 15q11.2q13_PWS_AS_BP2-BP4_t24 14905500 +chr15 29731821 30005618 15q11.2q13_PWS_AS_BP2-BP4_t25 8213910 +chr15 30006801 30044722 15q11.2q13_PWS_AS_BP2-BP4_t26 1137630 +chr15 30046094 30076787 15q11.2q13_PWS_AS_BP2-BP4_t27 920790 +chr15 34647508 34747508 15q11.2q13_PWS_AS_BP2-BP4_R 3000000 +chr15 28916886 29053771 15q13_BP3-BP4_t1 4106550 +chr15 29055911 29065471 15q13_BP3-BP4_t2 286800 +chr15 29069337 29228978 15q13_BP3-BP4_t3 4789230 +chr15 29233760 29730610 15q13_BP3-BP4_t4 14905500 +chr15 29731821 30005618 15q13_BP3-BP4_t5 8213910 +chr15 30006801 30044722 15q13_BP3-BP4_t6 1137630 +chr15 30046094 30076787 15q13_BP3-BP4_t7 920790 +chr15 34647508 34747508 15q13_BP3-BP4_R 3000000 +chr15 28916887 29053771 15q13.1q13.3_BP3-BP5_t1 4106520 +chr15 29055911 29065471 15q13.1q13.3_BP3-BP5_t2 286800 +chr15 29069337 29228978 15q13.1q13.3_BP3-BP5_t3 4789230 +chr15 29233760 29730610 15q13.1q13.3_BP3-BP5_t4 14905500 +chr15 29731821 30005618 15q13.1q13.3_BP3-BP5_t5 8213910 +chr15 30006801 30044722 15q13.1q13.3_BP3-BP5_t6 1137630 +chr15 30046094 30076788 15q13.1q13.3_BP3-BP5_t7 920820 +chr15 30900685 31259237 15q13.1q13.3_BP3-BP5_t8 10756560 +chr15 31259237 31617789 15q13.1q13.3_BP3-BP5_t9 10756560 +chr15 31735058 31764141 15q13.1q13.3_BP3-BP5_t10 872490 +chr15 31768318 32153204 15q13.1q13.3_BP3-BP5_t11 11546580 +chr15 34647508 34747508 15q13.1q13.3_BP3-BP5_R 3000000 +chr15 30900686 31259238 15q13.3_BP4-BP5_t1 10756560 +chr15 31259238 31617789 15q13.3_BP4-BP5_t2 10756530 +chr15 31735058 31764141 15q13.3_BP4-BP5_t3 872490 +chr15 31768318 32153204 15q13.3_BP4-BP5_t4 11546580 +chr15 34647508 34747508 15q13.3_BP4-BP5_R 3000000 +chr15 31735058 31764141 15q13.3_D-CHRNA7_to_BP5_t1 872490 +chr15 31768318 32153204 15q13.3_D-CHRNA7_to_BP5_t2 11546580 +chr15 34647508 34747508 15q13.3_D-CHRNA7_to_BP5_R 3000000 +chr15 32607509 33012322 15q13.3q14_BP5-BP6_t1 12144390 +chr15 33013398 33468532 15q13.3q14_BP5-BP6_t2 13654020 +chr15 33468532 33923666 15q13.3q14_BP5-BP6_t3 13654020 +chr15 33923666 34378800 15q13.3q14_BP5-BP6_t4 13654020 +chr15 34690717 34790717 15q13.3q14_BP5-BP6_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_C_L 3000000 +chr15 72671374 72682053 15q24_LCR_A-LCR_C_t1 320370 +chr15 72683710 72899078 15q24_LCR_A-LCR_C_t2 6461040 +chr15 72900361 73284156 15q24_LCR_A-LCR_C_t3 11513850 +chr15 73284156 73667951 15q24_LCR_A-LCR_C_t4 11513850 +chr15 73667951 74051745 15q24_LCR_A-LCR_C_t5 11513820 +chr15 74120301 74595608 15q24_LCR_A-LCR_C_t6 14259210 +chr15 74598046 74907009 15q24_LCR_A-LCR_C_t7 9268890 +chr15 74907009 75215971 15q24_LCR_A-LCR_C_t8 9268860 +chr15 78050834 78150834 15q24_LCR_A-LCR_C_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_D_L 3000000 +chr15 72671374 72682053 15q24_LCR_A-LCR_D_t1 320370 +chr15 72683710 72899078 15q24_LCR_A-LCR_D_t2 6461040 +chr15 72900361 73284156 15q24_LCR_A-LCR_D_t3 11513850 +chr15 73284156 73667951 15q24_LCR_A-LCR_D_t4 11513850 +chr15 73667951 74051745 15q24_LCR_A-LCR_D_t5 11513820 +chr15 74120301 74595608 15q24_LCR_A-LCR_D_t6 14259210 +chr15 74598046 74907009 15q24_LCR_A-LCR_D_t7 9268890 +chr15 74907009 75215972 15q24_LCR_A-LCR_D_t8 9268890 +chr15 75339445 75381921 15q24_LCR_A-LCR_D_t9 1274280 +chr15 75386986 75480313 15q24_LCR_A-LCR_D_t10 2799810 +chr15 75482930 75512491 15q24_LCR_A-LCR_D_t11 886830 +chr15 75514448 75680568 15q24_LCR_A-LCR_D_t12 4983600 +chr15 78050834 78150834 15q24_LCR_A-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_B_L 3000000 +chr15 72671375 72682053 15q24_LCR_A-LCR_B_t1 320340 +chr15 72683710 72899078 15q24_LCR_A-LCR_B_t2 6461040 +chr15 72900361 73284155 15q24_LCR_A-LCR_B_t3 11513820 +chr15 73284155 73667949 15q24_LCR_A-LCR_B_t4 11513820 +chr15 73667949 74051743 15q24_LCR_A-LCR_B_t5 11513820 +chr15 78050834 78150834 15q24_LCR_A-LCR_B_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_C_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_C_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_C_t2 9268890 +chr15 74907009 75215971 15q24_LCR_B-LCR_C_t3 9268860 +chr15 78050834 78150834 15q24_LCR_B-LCR_C_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_D_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_D_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_D_t2 9268890 +chr15 74907009 75215972 15q24_LCR_B-LCR_D_t3 9268890 +chr15 75339445 75381921 15q24_LCR_B-LCR_D_t4 1274280 +chr15 75386986 75480313 15q24_LCR_B-LCR_D_t5 2799810 +chr15 75482930 75512491 15q24_LCR_B-LCR_D_t6 886830 +chr15 75514448 75680568 15q24_LCR_B-LCR_D_t7 4983600 +chr15 78050834 78150834 15q24_LCR_B-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_E_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_E_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_E_t2 9268890 +chr15 74907009 75215972 15q24_LCR_B-LCR_E_t3 9268890 +chr15 75339445 75381921 15q24_LCR_B-LCR_E_t4 1274280 +chr15 75386986 75480313 15q24_LCR_B-LCR_E_t5 2799810 +chr15 75482930 75512491 15q24_LCR_B-LCR_E_t6 886830 +chr15 75514448 75680569 15q24_LCR_B-LCR_E_t7 4983630 +chr15 75822911 75846817 15q24_LCR_B-LCR_E_t8 717180 +chr15 75850869 75881298 15q24_LCR_B-LCR_E_t9 912870 +chr15 75882410 76257102 15q24_LCR_B-LCR_E_t10 11240760 +chr15 76259376 76619173 15q24_LCR_B-LCR_E_t11 10793910 +chr15 76619173 76978969 15q24_LCR_B-LCR_E_t12 10793880 +chr15 76980529 77278454 15q24_LCR_B-LCR_E_t13 8937750 +chr15 77279850 77314069 15q24_LCR_B-LCR_E_t14 1026570 +chr15 77315649 77559142 15q24_LCR_B-LCR_E_t15 7304790 +chr15 77562827 77858909 15q24_LCR_B-LCR_E_t16 8882460 +chr15 78050834 78150834 15q24_LCR_B-LCR_E_R 3000000 +chr15 72455531 72555531 15q24_LCR_C-LCR_D_L 3000000 +chr15 75339446 75381921 15q24_LCR_C-LCR_D_t1 1274250 +chr15 75386986 75480313 15q24_LCR_C-LCR_D_t2 2799810 +chr15 75482930 75512491 15q24_LCR_C-LCR_D_t3 886830 +chr15 75514448 75680568 15q24_LCR_C-LCR_D_t4 4983600 +chr15 78050834 78150834 15q24_LCR_C-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_D-LCR_E_L 3000000 +chr15 75822911 75846817 15q24_LCR_D-LCR_E_t1 717180 +chr15 75850869 75881298 15q24_LCR_D-LCR_E_t2 912870 +chr15 75882410 76257102 15q24_LCR_D-LCR_E_t3 11240760 +chr15 76259376 76619173 15q24_LCR_D-LCR_E_t4 10793910 +chr15 76619173 76978969 15q24_LCR_D-LCR_E_t5 10793880 +chr15 76980529 77278454 15q24_LCR_D-LCR_E_t6 8937750 +chr15 77279850 77314069 15q24_LCR_D-LCR_E_t7 1026570 +chr15 77315649 77559142 15q24_LCR_D-LCR_E_t8 7304790 +chr15 77562827 77858909 15q24_LCR_D-LCR_E_t9 8882460 +chr15 78050834 78150834 15q24_LCR_D-LCR_E_R 3000000 +chr15 82040662 82140662 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_L 3000000 +chr15 82534139 82613798 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t1 2389770 +chr15 82614158 82809649 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t2 5864730 +chr15 82827843 83233890 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t3 12181410 +chr15 83233890 83639937 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t4 12181410 +chr15 83639937 84045982 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t5 12181350 +chr15 84596420 84634582 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t6 1144860 +chr15 84636031 84801860 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t7 4974870 +chr15 84803285 85169770 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t8 10994550 +chr15 85526412 85626412 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_R 3000000 +chr15 82040662 82140662 15q25.2_proximal_LCR_A_B-C_L 3000000 +chr15 82534140 82613798 15q25.2_proximal_LCR_A_B-C_t1 2389740 +chr15 82614158 82809649 15q25.2_proximal_LCR_A_B-C_t2 5864730 +chr15 82827843 83233889 15q25.2_proximal_LCR_A_B-C_t3 12181380 +chr15 83233889 83639935 15q25.2_proximal_LCR_A_B-C_t4 12181380 +chr15 83639935 84045981 15q25.2_proximal_LCR_A_B-C_t5 12181380 +chr15 85526412 85626412 15q25.2_proximal_LCR_A_B-C_R 3000000 +chr15 82040662 82140662 15q25.2q25.3_distal_LCR_C-D_L 3000000 +chr15 84596421 84634582 15q25.2q25.3_distal_LCR_C-D_t1 1144830 +chr15 84636031 84801860 15q25.2q25.3_distal_LCR_C-D_t2 4974870 +chr15 84803285 85169770 15q25.2q25.3_distal_LCR_C-D_t3 10994550 +chr15 85526412 85626412 15q25.2q25.3_distal_LCR_C-D_R 3000000 +chr16 14580912 14680912 16p13.11_BP1-BP3_L 3000000 +chr16 15030912 15062395 16p13.11_BP1-BP3_t1 944490 +chr16 15417853 15808131 16p13.11_BP1-BP3_t2 11708340 +chr16 15808131 16198408 16p13.11_BP1-BP3_t3 11708310 +chr16 19002480 19102480 16p13.11_BP1-BP3_R 3000000 +chr16 14580912 14680912 16p13.11_BP1-BP4_L 3000000 +chr16 15030912 15062395 16p13.11_BP1-BP4_t1 944490 +chr16 15417853 15808131 16p13.11_BP1-BP4_t2 11708340 +chr16 15808131 16198409 16p13.11_BP1-BP4_t3 11708340 +chr16 16762938 17093166 16p13.11_BP1-BP4_t4 9906840 +chr16 17096423 17584983 16p13.11_BP1-BP4_t5 14656800 +chr16 17584983 18073542 16p13.11_BP1-BP4_t6 14656770 +chr16 19002480 19102480 16p13.11_BP1-BP4_R 3000000 +chr16 14580912 14680912 16p13.11_BP2-BP3_L 3000000 +chr16 15417854 15808131 16p13.11_BP2-BP3_t1 11708310 +chr16 15808131 16198408 16p13.11_BP2-BP3_t2 11708310 +chr16 19002480 19102480 16p13.11_BP2-BP3_R 3000000 +chr16 14580912 14680912 16p13.11_BP2-BP4_L 3000000 +chr16 15417855 15808132 16p13.11_BP2-BP4_t1 11708310 +chr16 15808132 16198409 16p13.11_BP2-BP4_t2 11708310 +chr16 16762938 17093166 16p13.11_BP2-BP4_t3 9906840 +chr16 17096423 17584983 16p13.11_BP2-BP4_t4 14656800 +chr16 17584983 18073542 16p13.11_BP2-BP4_t5 14656770 +chr16 19002480 19102480 16p13.11_BP2-BP4_R 3000000 +chr16 14580912 14680912 16p12.3_BP3-BP4_L 3000000 +chr16 16762939 17093166 16p12.3_BP3-BP4_t1 9906810 +chr16 17096423 17584983 16p12.3_BP3-BP4_t2 14656800 +chr16 17584983 18073542 16p12.3_BP3-BP4_t3 14656770 +chr16 19002480 19102480 16p12.3_BP3-BP4_R 3000000 +chr16 21192540 21292540 16p12.2_distal_BP1-BP2_L 3000000 +chr16 21558792 21728000 16p12.2_distal_BP1-BP2_t1 5076240 +chr16 30393459 30493459 16p12.2_distal_BP1-BP2_R 3000000 +chr16 21192540 21292540 16p12.2p11.2_L 3000000 +chr16 21558793 21728000 16p12.2p11.2_t1 5076210 +chr16 21937123 22419484 16p12.2p11.2_t2 14470830 +chr16 22613938 22833211 16p12.2p11.2_t3 6578190 +chr16 22834242 23166943 16p12.2p11.2_t4 9981030 +chr16 23166943 23499643 16p12.2p11.2_t5 9981000 +chr16 23500829 23501557 16p12.2p11.2_t6 21840 +chr16 23503728 23667041 16p12.2p11.2_t7 4899390 +chr16 23669861 24115761 16p12.2p11.2_t8 13377000 +chr16 24115761 24561661 16p12.2p11.2_t9 13377000 +chr16 24561661 25007561 16p12.2p11.2_t10 13377000 +chr16 25007561 25453458 16p12.2p11.2_t11 13376910 +chr16 25454717 25743210 16p12.2p11.2_t12 8654790 +chr16 25743210 26031702 16p12.2p11.2_t13 8654760 +chr16 26032921 26111401 16p12.2p11.2_t14 2354400 +chr16 26112444 26452694 16p12.2p11.2_t15 10207500 +chr16 26452694 26792944 16p12.2p11.2_t16 10207500 +chr16 26792944 27133194 16p12.2p11.2_t17 10207500 +chr16 27135251 27503334 16p12.2p11.2_t18 11042490 +chr16 27503334 27871417 16p12.2p11.2_t19 11042490 +chr16 27871417 28239498 16p12.2p11.2_t20 11042430 +chr16 28240648 28327647 16p12.2p11.2_t21 2609970 +chr16 28471892 28590812 16p12.2p11.2_t22 3567600 +chr16 28811313 29035179 16p12.2p11.2_t23 6715980 +chr16 29126773 29127492 16p12.2p11.2_t24 21570 +chr16 29302873 29303936 16p12.2p11.2_t25 31890 +chr16 30393459 30493459 16p12.2p11.2_R 3000000 +chr16 21192540 21292540 16p12.2_proximal_BP2-BP3_L 3000000 +chr16 21937124 22419483 16p12.2_proximal_BP2-BP3_t1 14470770 +chr16 30393459 30493459 16p12.2_proximal_BP2-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_distal_BP1-BP3_L 3000000 +chr16 28471894 28590812 16p11.2_distal_BP1-BP3_t1 3567540 +chr16 28811313 29035178 16p11.2_distal_BP1-BP3_t2 6715950 +chr16 30393459 30493459 16p11.2_distal_BP1-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_BP1-BP4_L 3000000 +chr16 28471894 28590812 16p11.2_proximal_BP1-BP4_t1 3567540 +chr16 28811313 29035179 16p11.2_proximal_BP1-BP4_t2 6715980 +chr16 29126773 29127492 16p11.2_proximal_BP1-BP4_t3 21570 +chr16 29302873 29303936 16p11.2_proximal_BP1-BP4_t4 31890 +chr16 30393459 30493459 16p11.2_proximal_BP1-BP4_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_and_distal_BP1-BP5_L 3000000 +chr16 28471894 28590812 16p11.2_proximal_and_distal_BP1-BP5_t1 3567540 +chr16 28811313 29035179 16p11.2_proximal_and_distal_BP1-BP5_t2 6715980 +chr16 29126773 29127492 16p11.2_proximal_and_distal_BP1-BP5_t3 21570 +chr16 29302873 29303936 16p11.2_proximal_and_distal_BP1-BP5_t4 31890 +chr16 29638675 29903564 16p11.2_proximal_and_distal_BP1-BP5_t5 7946670 +chr16 29903564 30168453 16p11.2_proximal_and_distal_BP1-BP5_t6 7946670 +chr16 30170912 30188531 16p11.2_proximal_and_distal_BP1-BP5_t7 528570 +chr16 30393459 30493459 16p11.2_proximal_and_distal_BP1-BP5_R 3000000 +chr16 21192540 21292540 16p11.2_distal_BP2-BP3_L 3000000 +chr16 28811314 29035178 16p11.2_distal_BP2-BP3_t1 6715920 +chr16 30393459 30493459 16p11.2_distal_BP2-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_BP4-BP5_L 3000000 +chr16 29638676 29903565 16p11.2_proximal_BP4-BP5_t1 7946670 +chr16 29903565 30168453 16p11.2_proximal_BP4-BP5_t2 7946640 +chr16 30170912 30188531 16p11.2_proximal_BP4-BP5_t3 528570 +chr16 30393459 30493459 16p11.2_proximal_BP4-BP5_R 3000000 +chr17 13869962 13969962 17p12_HNPP_CMT1A_L 3000000 +chr17 14194700 14230542 17p12_HNPP_CMT1A_t1 1075260 +chr17 14241379 14667253 17p12_HNPP_CMT1A_t2 12776220 +chr17 14667253 15093127 17p12_HNPP_CMT1A_t3 12776220 +chr17 15093127 15519000 17p12_HNPP_CMT1A_t4 12776190 +chr17 15930698 16030698 17p12_HNPP_CMT1A_R 3000000 +chr17 16486272 16586272 17p11.2_SMS_PLS_L 3000000 +chr17 16906714 16935461 17p11.2_SMS_PLS_t1 862410 +chr17 16935702 16995496 17p11.2_SMS_PLS_t2 1793820 +chr17 17031424 17463581 17p11.2_SMS_PLS_t3 12964710 +chr17 17463581 17895738 17p11.2_SMS_PLS_t4 12964710 +chr17 17895738 18327894 17p11.2_SMS_PLS_t5 12964680 +chr17 18329376 18375628 17p11.2_SMS_PLS_t6 1387560 +chr17 18848096 19020655 17p11.2_SMS_PLS_t7 5176770 +chr17 19237509 19282949 17p11.2_SMS_PLS_t8 1363200 +chr17 19284122 19581999 17p11.2_SMS_PLS_t9 8936310 +chr17 19583330 19587402 17p11.2_SMS_PLS_t10 122160 +chr17 19588723 19598547 17p11.2_SMS_PLS_t11 294720 +chr17 19601764 19856686 17p11.2_SMS_PLS_t12 7647660 +chr17 19856686 20111608 17p11.2_SMS_PLS_t13 7647660 +chr17 20115149 20233524 17p11.2_SMS_PLS_t14 3551250 +chr17 20235188 20309889 17p11.2_SMS_PLS_t15 2241030 +chr17 20986981 21086981 17p11.2_SMS_PLS_R 3000000 +chr17 16486272 16586272 17p11.2_REPD-REPM_L 3000000 +chr17 16906715 16935461 17p11.2_REPD-REPM_t1 862380 +chr17 16935702 16995496 17p11.2_REPD-REPM_t2 1793820 +chr17 17031424 17463581 17p11.2_REPD-REPM_t3 12964710 +chr17 17463581 17895738 17p11.2_REPD-REPM_t4 12964710 +chr17 17895738 18327894 17p11.2_REPD-REPM_t5 12964680 +chr17 18329376 18375627 17p11.2_REPD-REPM_t6 1387530 +chr17 20986981 21086981 17p11.2_REPD-REPM_R 3000000 +chr17 30380647 30480647 17q11.2_Type_1_REPA-REPC_L 3000000 +chr17 30780078 31008336 17q11.2_Type_1_REPA-REPC_t1 6847740 +chr17 31051210 31067523 17q11.2_Type_1_REPA-REPC_t2 489390 +chr17 31072877 31078593 17q11.2_Type_1_REPA-REPC_t3 171480 +chr17 31080033 31138282 17q11.2_Type_1_REPA-REPC_t4 1747470 +chr17 31149888 31189463 17q11.2_Type_1_REPA-REPC_t5 1187250 +chr17 31191722 31193317 17q11.2_Type_1_REPA-REPC_t6 47850 +chr17 31202173 31212016 17q11.2_Type_1_REPA-REPC_t7 295290 +chr17 31236747 31256172 17q11.2_Type_1_REPA-REPC_t8 582750 +chr17 31268283 31345374 17q11.2_Type_1_REPA-REPC_t9 2312730 +chr17 31348726 31642514 17q11.2_Type_1_REPA-REPC_t10 8813640 +chr17 31642514 31936301 17q11.2_Type_1_REPA-REPC_t11 8813610 +chr17 32170047 32270047 17q11.2_Type_1_REPA-REPC_R 3000000 +chr17 30380647 30480647 17q11.2_Type_2_SUZ12P1-SUZ12_L 3000000 +chr17 30780079 31008336 17q11.2_Type_2_SUZ12P1-SUZ12_t1 6847710 +chr17 31051210 31067523 17q11.2_Type_2_SUZ12P1-SUZ12_t2 489390 +chr17 31072877 31078593 17q11.2_Type_2_SUZ12P1-SUZ12_t3 171480 +chr17 31080033 31138282 17q11.2_Type_2_SUZ12P1-SUZ12_t4 1747470 +chr17 31149888 31189463 17q11.2_Type_2_SUZ12P1-SUZ12_t5 1187250 +chr17 31191722 31193317 17q11.2_Type_2_SUZ12P1-SUZ12_t6 47850 +chr17 31202173 31212016 17q11.2_Type_2_SUZ12P1-SUZ12_t7 295290 +chr17 31236747 31256172 17q11.2_Type_2_SUZ12P1-SUZ12_t8 582750 +chr17 31268283 31345374 17q11.2_Type_2_SUZ12P1-SUZ12_t9 2312730 +chr17 31348726 31642514 17q11.2_Type_2_SUZ12P1-SUZ12_t10 8813640 +chr17 31642514 31936301 17q11.2_Type_2_SUZ12P1-SUZ12_t11 8813610 +chr17 32170047 32270047 17q11.2_Type_2_SUZ12P1-SUZ12_R 3000000 +chr17 30380647 30480647 17q11.2_Type_3_REPB-REPC_L 3000000 +chr17 31051211 31067523 17q11.2_Type_3_REPB-REPC_t1 489360 +chr17 31072877 31078593 17q11.2_Type_3_REPB-REPC_t2 171480 +chr17 31080033 31138282 17q11.2_Type_3_REPB-REPC_t3 1747470 +chr17 31149888 31189463 17q11.2_Type_3_REPB-REPC_t4 1187250 +chr17 31191722 31193317 17q11.2_Type_3_REPB-REPC_t5 47850 +chr17 31202173 31212016 17q11.2_Type_3_REPB-REPC_t6 295290 +chr17 31236747 31256172 17q11.2_Type_3_REPB-REPC_t7 582750 +chr17 31268283 31345374 17q11.2_Type_3_REPB-REPC_t8 2312730 +chr17 31348726 31642514 17q11.2_Type_3_REPB-REPC_t9 8813640 +chr17 31642514 31936301 17q11.2_Type_3_REPB-REPC_t10 8813610 +chr17 32170047 32270047 17q11.2_Type_3_REPB-REPC_R 3000000 +chr17 35818633 35918633 17q12_RCAD_syndrome_L 3000000 +chr17 36458167 36800704 17q12_RCAD_syndrome_t1 10276110 +chr17 36800704 37143240 17q12_RCAD_syndrome_t2 10276080 +chr17 37144370 37488620 17q12_RCAD_syndrome_t3 10327500 +chr17 37488620 37832870 17q12_RCAD_syndrome_t4 10327500 +chr17 37836264 37854616 17q12_RCAD_syndrome_t5 550560 +chr17 38364969 38464969 17q12_RCAD_syndrome_R 3000000 +chr17 45264926 45364926 17q21.31_L 3000000 +chr17 45627800 46087514 17q21.31_t1 13791420 +chr17 46757124 46857124 17q21.31_R 3000000 +chr17 59771943 59871943 17q23.1q23.2_L 3000000 +chr17 60035641 60096247 17q23.1q23.2_t1 1818180 +chr17 60127341 60166682 17q23.1q23.2_t2 1180230 +chr17 60171137 60172722 17q23.1q23.2_t3 47550 +chr17 60221224 60374694 17q23.1q23.2_t4 4604100 +chr17 60382053 60434344 17q23.1q23.2_t5 1568730 +chr17 60435541 60851263 17q23.1q23.2_t6 12471660 +chr17 60852649 61301249 17q23.1q23.2_t7 13458000 +chr17 61301249 61749849 17q23.1q23.2_t8 13458000 +chr17 61749849 62198448 17q23.1q23.2_t9 13457970 +chr17 62618463 62718463 17q23.1q23.2_R 3000000 +chr22 16912063 16922834 22q11.21_CES_t1 323130 +chr22 16928158 17014088 22q11.21_CES_t2 2577900 +chr22 17038567 17214090 22q11.21_CES_t3 5265690 +chr22 17215248 17419415 22q11.21_CES_t4 6125010 +chr22 17420675 17763709 22q11.21_CES_t5 10291020 +chr22 17765328 17984579 22q11.21_CES_t6 6577530 +chr22 17998239 18008719 22q11.21_CES_t7 314400 +chr22 18010388 18159723 22q11.21_CES_t8 4480050 +chr22 24773137 24873137 22q11.21_CES_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-B_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-B_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-B_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-B_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-B_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-B_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-B_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-B_t8 2856540 +chr22 20290267 20299685 22q11.2_DGS_proximal_A-B_t9 282540 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-B_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-C_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-C_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-C_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-C_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-C_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-C_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-C_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-C_t8 2856540 +chr22 20290267 20299686 22q11.2_DGS_proximal_A-C_t9 282570 +chr22 20377695 20388431 22q11.2_DGS_proximal_A-C_t10 322080 +chr22 20390146 20451509 22q11.2_DGS_proximal_A-C_t11 1840890 +chr22 20453240 20482745 22q11.2_DGS_proximal_A-C_t12 885150 +chr22 20484148 20656624 22q11.2_DGS_proximal_A-C_t13 5174280 +chr22 20658941 20667579 22q11.2_DGS_proximal_A-C_t14 259140 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-C_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-D_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-D_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-D_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-D_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-D_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-D_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-D_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-D_t8 2856540 +chr22 20290267 20299686 22q11.2_DGS_proximal_A-D_t9 282570 +chr22 20377695 20388431 22q11.2_DGS_proximal_A-D_t10 322080 +chr22 20390146 20451509 22q11.2_DGS_proximal_A-D_t11 1840890 +chr22 20453240 20482745 22q11.2_DGS_proximal_A-D_t12 885150 +chr22 20484148 20656624 22q11.2_DGS_proximal_A-D_t13 5174280 +chr22 20658941 20667580 22q11.2_DGS_proximal_A-D_t14 259170 +chr22 20738049 21010365 22q11.2_DGS_proximal_A-D_t15 8169480 +chr22 21018352 21042050 22q11.2_DGS_proximal_A-D_t16 710940 +chr22 21050489 21057379 22q11.2_DGS_proximal_A-D_t17 206700 +chr22 21062735 21111383 22q11.2_DGS_proximal_A-D_t18 1459440 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-D_R 3000000 +chr22 20377696 20388431 22q11.2_central_B-D_t1 322050 +chr22 20390146 20451509 22q11.2_central_B-D_t2 1840890 +chr22 20453240 20482745 22q11.2_central_B-D_t3 885150 +chr22 20484148 20656624 22q11.2_central_B-D_t4 5174280 +chr22 20658941 20667580 22q11.2_central_B-D_t5 259170 +chr22 20738049 21010365 22q11.2_central_B-D_t6 8169480 +chr22 21018352 21042050 22q11.2_central_B-D_t7 710940 +chr22 21050489 21057379 22q11.2_central_B-D_t8 206700 +chr22 21062735 21111383 22q11.2_central_B-D_t9 1459440 +chr22 24773137 24873137 22q11.2_central_B-D_R 3000000 +chr22 20738050 21010365 22q11.2_central_C-D_t1 8169450 +chr22 21018352 21042050 22q11.2_central_C-D_t2 710940 +chr22 21050489 21057379 22q11.2_central_C-D_t3 206700 +chr22 21062735 21111383 22q11.2_central_C-D_t4 1459440 +chr22 24773137 24873137 22q11.2_central_C-D_R 3000000 +chr22 20738051 21010365 22q11.2_C-E_t1 8169420 +chr22 21018352 21042050 22q11.2_C-E_t2 710940 +chr22 21050489 21057379 22q11.2_C-E_t3 206700 +chr22 21062735 21111384 22q11.2_C-E_t4 1459470 +chr22 21562827 21954496 22q11.2_C-E_t5 11750070 +chr22 21958494 22114615 22q11.2_C-E_t6 4683630 +chr22 22120610 22122809 22q11.2_C-E_t7 65970 +chr22 22125415 22125942 22q11.2_C-E_t8 15810 +chr22 22126195 22133984 22q11.2_C-E_t9 233670 +chr22 22139837 22220568 22q11.2_C-E_t10 2421930 +chr22 22224566 22249758 22q11.2_C-E_t11 755760 +chr22 22315122 22320797 22q11.2_C-E_t12 170250 +chr22 22327981 22402552 22q11.2_C-E_t13 2237130 +chr22 22410447 22620607 22q11.2_C-E_t14 6304800 +chr22 24773137 24873137 22q11.2_C-E_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_I_D-E_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_I_D-E_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_I_D-E_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_I_D-E_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_I_D-E_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_I_D-E_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_I_D-E_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_I_D-E_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_I_D-E_t9 2237130 +chr22 22410447 22620607 22q11.2_distal_type_I_D-E_t10 6304800 +chr22 24773137 24873137 22q11.2_distal_type_I_D-E_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_I_D-F_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_I_D-F_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_I_D-F_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_I_D-F_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_I_D-F_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_I_D-F_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_I_D-F_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_I_D-F_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_I_D-F_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_I_D-F_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_I_D-F_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_I_D-F_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_I_D-F_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_I_D-F_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_I_D-F_t15 10711530 +chr22 23264988 23306924 22q11.2_distal_type_I_D-F_t16 1258080 +chr22 24773137 24873137 22q11.2_distal_type_I_D-F_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_III_D-H_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_III_D-H_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_III_D-H_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_III_D-H_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_III_D-H_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_III_D-H_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_III_D-H_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_III_D-H_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_III_D-H_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_III_D-H_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_III_D-H_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_III_D-H_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_D-H_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_D-H_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_D-H_t15 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_D-H_t16 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_D-H_t17 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_D-H_t18 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_D-H_t19 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_D-H_t20 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_D-H_t21 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_D-H_t22 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_D-H_t23 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_D-H_t24 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_D-H_t25 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_D-H_t26 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_D-H_t27 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_D-H_t28 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_D-H_t29 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_D-H_t30 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_D-H_t31 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_D-H_t32 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_D-H_t33 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_D-H_R 3000000 +chr22 21562829 21954496 22q11.2_distal_type_III_D-G_t1 11750010 +chr22 21958494 22114615 22q11.2_distal_type_III_D-G_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_III_D-G_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_III_D-G_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_III_D-G_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_III_D-G_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_III_D-G_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_III_D-G_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_III_D-G_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_III_D-G_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_III_D-G_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_III_D-G_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_D-G_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_D-G_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_D-G_t15 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_D-G_t16 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_D-G_t17 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_D-G_t18 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_D-G_t19 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_D-G_t20 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_D-G_t21 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_D-G_t22 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_D-G_t23 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_D-G_t24 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_D-G_t25 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_D-G_t26 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_D-G_t27 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_D-G_t28 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_D-G_t29 5337540 +chr22 24184938 24236853 22q11.2_distal_type_III_D-G_t30 1557450 +chr22 24773137 24873137 22q11.2_distal_type_III_D-G_R 3000000 +chr22 22776924 22789775 22q11.2_distal_type_II_E-F_t1 385530 +chr22 22794414 22820054 22q11.2_distal_type_II_E-F_t2 769200 +chr22 22823467 22897363 22q11.2_distal_type_II_E-F_t3 2216880 +chr22 22901536 22902711 22q11.2_distal_type_II_E-F_t4 35250 +chr22 22906902 23263953 22q11.2_distal_type_II_E-F_t5 10711530 +chr22 23264988 23306924 22q11.2_distal_type_II_E-F_t6 1258080 +chr22 24773137 24873137 22q11.2_distal_type_II_E-F_R 3000000 +chr22 22776925 22789775 22q11.2_distal_type_III_E-H_t1 385500 +chr22 22794414 22820054 22q11.2_distal_type_III_E-H_t2 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_E-H_t3 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_E-H_t4 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_E-H_t5 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_E-H_t6 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_E-H_t7 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_E-H_t8 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_E-H_t9 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_E-H_t10 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_E-H_t11 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_E-H_t12 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_E-H_t13 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_E-H_t14 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_E-H_t15 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_E-H_t16 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_E-H_t17 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_E-H_t18 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_E-H_t19 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_E-H_t20 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_E-H_t21 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_E-H_t22 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_E-H_t23 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_E-H_R 3000000 +chr22 23489015 23493585 22q11.2_distal_type_III_F-G_t1 137100 +chr22 23510388 23511082 22q11.2_distal_type_III_F-G_t2 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_F-G_t3 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_F-G_t4 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_F-G_t5 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_F-G_t6 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_F-G_t7 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_F-G_t8 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_F-G_t9 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_F-G_t10 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_F-G_t11 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_F-G_t12 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_F-G_t13 5337540 +chr22 24184938 24236853 22q11.2_distal_type_III_F-G_t14 1557450 +chr22 24773137 24873137 22q11.2_distal_type_III_F-G_R 3000000 +chr22 23489016 23493585 22q11.2_distal_type_III_F-H_t1 137070 +chr22 23510388 23511082 22q11.2_distal_type_III_F-H_t2 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_F-H_t3 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_F-H_t4 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_F-H_t5 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_F-H_t6 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_F-H_t7 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_F-H_t8 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_F-H_t9 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_F-H_t10 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_F-H_t11 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_F-H_t12 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_F-H_t13 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_F-H_t14 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_F-H_t15 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_F-H_t16 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_F-H_t17 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_F-H_R 3000000 +chrX 6377561 6477561 Xp22.31_L 1500000 +chrX 6537771 6709739 Xp22.31_t1 2579520 +chrX 6712094 6934350 Xp22.31_t2 3333840 +chrX 6935400 7003885 Xp22.31_t3 1027275 +chrX 7044785 7321723 Xp22.31_t4 4154070 +chrX 7321723 7598660 Xp22.31_t5 4154055 +chrX 7599946 7752196 Xp22.31_t6 2283750 +chrX 7761567 7840853 Xp22.31_t7 1189290 +chrX 7852088 7973686 Xp22.31_t8 1823970 +chrX 7984426 8000352 Xp22.31_t9 238890 +chrX 8006947 8156913 Xp22.31_t10 2249490 +chrX 8231347 8331347 Xp22.31_R 1500000 +chrX 47703983 47803983 Xp11.22p11.23_L 1500000 +chrX 48447780 48801813 Xp11.22p11.23_t1 5310495 +chrX 48801813 49155846 Xp11.22p11.23_t2 5310495 +chrX 49157442 49162798 Xp11.22p11.23_t3 80340 +chrX 49164374 49300509 Xp11.22p11.23_t4 2042025 +chrX 49606699 49631626 Xp11.22p11.23_t5 373905 +chrX 49633394 49663751 Xp11.22p11.23_t6 455355 +chrX 49668689 49836185 Xp11.22p11.23_t7 2512440 +chrX 49837392 49877729 Xp11.22p11.23_t8 605055 +chrX 49878948 50174945 Xp11.22p11.23_t9 4439955 +chrX 50175512 50204972 Xp11.22p11.23_t10 441900 +chrX 50209960 50210790 Xp11.22p11.23_t11 12450 +chrX 50228964 50278964 Xp11.22p11.23_t12 750000 +chrX 50293964 50599659 Xp11.22p11.23_t13 4585425 +chrX 50599659 50905354 Xp11.22p11.23_t14 4585425 +chrX 50906368 50990225 Xp11.22p11.23_t15 1257855 +chrX 50993330 50996121 Xp11.22p11.23_t16 41865 +chrX 51040256 51077839 Xp11.22p11.23_t17 563745 +chrX 51081476 51154075 Xp11.22p11.23_t18 1088985 +chrX 51184010 51290913 Xp11.22p11.23_t19 1603545 +chrX 51296426 51314554 Xp11.22p11.23_t20 271920 +chrX 51334611 51494772 Xp11.22p11.23_t21 2402415 +chrX 51526125 51617596 Xp11.22p11.23_t22 1372065 +chrX 51619037 51667324 Xp11.22p11.23_t23 724305 +chrX 51692963 51700363 Xp11.22p11.23_t24 111000 +chrX 51725664 51869486 Xp11.22p11.23_t25 2157330 +chrX 51873046 51903394 Xp11.22p11.23_t26 455220 +chrX 51904535 51919767 Xp11.22p11.23_t27 228480 +chrX 51922376 52037052 Xp11.22p11.23_t28 1720140 +chrX 52068673 52185421 Xp11.22p11.23_t29 1751220 +chrX 52213786 52262297 Xp11.22p11.23_t30 727665 +chrX 52266091 52266756 Xp11.22p11.23_t31 9975 +chrX 52268901 52336321 Xp11.22p11.23_t32 1011300 +chrX 52340891 52444264 Xp11.22p11.23_t33 1550595 +chrX 53026821 53126821 Xp11.22p11.23_R 1500000 +chrX 153905935 154005935 Xq28_L 1500000 +chrX 154396223 154555683 Xq28_t1 2391900 +chrX 154704527 154804527 Xq28_R 1500000 +chrX 154710814 154810814 Xq28_int22h1_int22h2-flanked_L 1500000 +chrX 154890328 155006106 Xq28_int22h1_int22h2-flanked_t1 1736670 +chrX 155007147 155147780 Xq28_int22h1_int22h2-flanked_t2 2109495 +chrX 155148821 155288173 Xq28_int22h1_int22h2-flanked_t3 2090280 +chrX 155291419 155304000 Xq28_int22h1_int22h2-flanked_t4 188715 +chrX 155304900 155335092 Xq28_int22h1_int22h2-flanked_t5 452880 +chrX 155506728 155606728 Xq28_int22h1_int22h2-flanked_R 1500000 +chr1 898702 955921 1p36_terminal_t1 1716570 +chr1 960557 1075856 1p36_terminal_t2 3458970 +chr1 1079032 1444514 1p36_terminal_t3 10964460 +chr1 1448930 1464079 1p36_terminal_t4 454470 +chr1 1502033 1504522 1p36_terminal_t5 74670 +chr1 1534825 1632163 1p36_terminal_t6 2920140 +chr1 1730936 1733195 1p36_terminal_t7 67770 +chr1 1752193 2150309 1p36_terminal_t8 11943480 +chr1 2150309 2548425 1p36_terminal_t9 11943480 +chr1 2550688 2551369 1p36_terminal_t10 20430 +chr1 2554047 2567635 1p36_terminal_t11 407640 +chr1 2570293 2576603 1p36_terminal_t12 189300 +chr1 2578586 2652036 1p36_terminal_t13 2203500 +chr1 2702781 2746290 1p36_terminal_t14 1305270 +chr1 2777361 3235654 1p36_terminal_t15 13748790 +chr1 3235654 3693946 1p36_terminal_t16 13748760 +chr1 3695002 3722655 1p36_terminal_t17 829590 +chr1 3723188 4043198 1p36_terminal_t18 9600300 +chr1 4045223 4447638 1p36_terminal_t19 12072450 +chr1 4447638 4850053 1p36_terminal_t20 12072450 +chr1 4850053 5252468 1p36_terminal_t21 12072450 +chr1 5252468 5654881 1p36_terminal_t22 12072390 +chr1 5657240 5665835 1p36_terminal_t23 257850 +chr1 5673788 5929408 1p36_terminal_t24 7668600 +chr1 5929408 6185028 1p36_terminal_t25 7668600 +chr1 6186818 6229913 1p36_terminal_t26 1292850 +chr1 6279913 6379913 1p36_terminal_R 3000000 +chrX 153858528 153958528 Xq28_MECP2_L 1500000 +chrX 154008528 154110279 Xq28_MECP2_t1 1526265 +chrX 154660279 154760279 Xq28_MECP2_R 1500000 +chr13 20073037 20173037 GJB6_D13S1830_L 3000000 +chr13 20223037 20531806 GJB6_D13S1830_t1 9263070 +chr13 20581806 20681806 GJB6_D13S1830_R 3000000 +chr7 117348312 117448312 CFTR_dele2_3_L 3000000 +chr7 117498312 117519392 CFTR_dele2_3_t1 632400 +chr7 117569392 117669392 CFTR_dele2_3_R 3000000 +chr11 5075254 5175254 HBB_exon3_del_L 3000000 +chr11 5225254 5225875 HBB_exon3_del_t1 18630 +chr11 5275875 5375875 HBB_exon3_del_R 3000000 +chr11 5075388 5175388 HBB_619bp_del_L 3000000 +chr11 5225388 5226007 HBB_619bp_del_t1 18570 +chr11 5276007 5376007 HBB_619bp_del_R 3000000 +chr11 5069877 5169877 HBD_HBB_Sicilian_del_L 3000000 +chr11 5219877 5233280 HBD_HBB_Sicilian_del_t1 402090 +chr11 5283280 5383280 HBD_HBB_Sicilian_del_R 3000000 +chrY 2935200 2982579 AZFa_c1 710685 +chrY 12537649 12860852 AZFa_t1 4848045 +chrY 12904107 12920478 AZFa_t2 245565 +chrY 14522572 14845654 AZFa_c2 4846230 +chrY 2935200 2982579 AZFb_c1 710685 +chrY 14522572 14845654 AZFb_c2 4846230 +chrY 19702297 19745148 AZFb_t1 642765 +chrY 20575693 20593154 AZFb_t2 261915 +chrY 2935200 2982579 AZFc_DAZ_c1 710685 +chrY 14522572 14845654 AZFc_DAZ_c2 4846230 +chrY 23129354 23199010 AZFc_DAZ_t1 1044840 +chrY 23219446 23291356 AZFc_DAZ_t2 1078650 +chrY 24763068 24813492 AZFc_DAZ_t3 756360 +chrY 24833842 24907040 AZFc_DAZ_t4 1097970 diff --git a/tests/parity/inputs/cn/kiv-notseen.pileup b/tests/parity/inputs/cn/kiv-notseen.pileup new file mode 100644 index 000000000..b993050ec --- /dev/null +++ b/tests/parity/inputs/cn/kiv-notseen.pileup @@ -0,0 +1,7 @@ +chr6 160646214 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160640667 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160635123 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160629574 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160624030 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160618484 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160612932 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III diff --git a/tests/parity/inputs/cn/kiv-present.pileup b/tests/parity/inputs/cn/kiv-present.pileup new file mode 100644 index 000000000..b4131112c --- /dev/null +++ b/tests/parity/inputs/cn/kiv-present.pileup @@ -0,0 +1,7 @@ +chr6 160646214 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTTt+1A III +chr6 160640667 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTT+1A III +chr6 160635123 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTT+1A III +chr6 160629574 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160624030 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160618484 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160612932 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III diff --git a/tests/parity/inputs/cn/lce0.bedcov.tsv b/tests/parity/inputs/cn/lce0.bedcov.tsv new file mode 100644 index 000000000..0418598db --- /dev/null +++ b/tests/parity/inputs/cn/lce0.bedcov.tsv @@ -0,0 +1,3 @@ +chr1 152520000 152560000 LCE3_FLANK_L 1200000 +chr1 152600233 152614098 LCE3BC_DEL 0 +chr1 152640000 152680000 LCE3_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/lce0.q1.tsv b/tests/parity/inputs/cn/lce0.q1.tsv new file mode 100644 index 000000000..0418598db --- /dev/null +++ b/tests/parity/inputs/cn/lce0.q1.tsv @@ -0,0 +1,3 @@ +chr1 152520000 152560000 LCE3_FLANK_L 1200000 +chr1 152600233 152614098 LCE3BC_DEL 0 +chr1 152640000 152680000 LCE3_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/lpa-floor.bedcov.tsv b/tests/parity/inputs/cn/lpa-floor.bedcov.tsv new file mode 100644 index 000000000..fa5c8e268 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-floor.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 34 +chr6 160612931 160613113 kiv2_e15 39 +chr6 160617116 160617276 kiv2_e14 34 +chr6 160618483 160618665 kiv2_e13 39 +chr6 160622662 160622822 kiv2_e12 34 +chr6 160624029 160624211 kiv2_e11 39 +chr6 160628206 160628366 kiv2_e10 34 +chr6 160629573 160629755 kiv2_e09 39 +chr6 160633752 160633912 kiv2_e08 34 +chr6 160635122 160635304 kiv2_e07 39 +chr6 160639299 160639459 kiv2_e06 34 +chr6 160640666 160640848 kiv2_e05 39 +chr6 160644846 160645006 kiv2_e04 34 +chr6 160646213 160646395 kiv2_e03 39 diff --git a/tests/parity/inputs/cn/lpa-floor.q1.tsv b/tests/parity/inputs/cn/lpa-floor.q1.tsv new file mode 100644 index 000000000..db866d7e5 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-floor.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 3 +chr6 160612931 160613113 kiv2_e15 4 +chr6 160617116 160617276 kiv2_e14 3 +chr6 160618483 160618665 kiv2_e13 4 +chr6 160622662 160622822 kiv2_e12 3 +chr6 160624029 160624211 kiv2_e11 4 +chr6 160628206 160628366 kiv2_e10 3 +chr6 160629573 160629755 kiv2_e09 4 +chr6 160633752 160633912 kiv2_e08 3 +chr6 160635122 160635304 kiv2_e07 4 +chr6 160639299 160639459 kiv2_e06 3 +chr6 160640666 160640848 kiv2_e05 4 +chr6 160644846 160645006 kiv2_e04 3 +chr6 160646213 160646395 kiv2_e03 4 diff --git a/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv b/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv new file mode 100644 index 000000000..da76d893c --- /dev/null +++ b/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 564 +chr6 160547788 160547937 flank_e32 894 +chr6 160548477 160548659 flank_e31 1092 +chr6 160556024 160556184 flank_e30 960 +chr6 160557389 160557571 flank_e29 1092 +chr6 160577135 160577295 flank_e28 960 +chr6 160578522 160578704 flank_e27 1092 +chr6 160585045 160585205 flank_e26 960 +chr6 160586448 160586630 flank_e25 1092 +chr6 160589552 160589712 flank_e24 960 +chr6 160590943 160591101 flank_e23 948 +chr6 160593957 160594117 flank_e22 960 +chr6 160595353 160595535 flank_e21 1092 +chr6 160599499 160599659 flank_e20 960 +chr6 160600916 160601098 flank_e19 1092 +chr6 160611561 160611721 kiv2_e16 2194 +chr6 160612931 160613113 kiv2_e15 2496 +chr6 160617116 160617276 kiv2_e14 2194 +chr6 160618483 160618665 kiv2_e13 2496 +chr6 160622662 160622822 kiv2_e12 2194 +chr6 160624029 160624211 kiv2_e11 2496 +chr6 160628206 160628366 kiv2_e10 2194 +chr6 160629573 160629755 kiv2_e09 2496 +chr6 160633752 160633912 kiv2_e08 2194 +chr6 160635122 160635304 kiv2_e07 2496 +chr6 160639299 160639459 kiv2_e06 2194 +chr6 160640666 160640848 kiv2_e05 2496 +chr6 160644846 160645006 kiv2_e04 2194 +chr6 160646213 160646395 kiv2_e03 2496 diff --git a/tests/parity/inputs/cn/lpa-lowflank.q1.tsv b/tests/parity/inputs/cn/lpa-lowflank.q1.tsv new file mode 100644 index 000000000..f1f092779 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-lowflank.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 564 +chr6 160547788 160547937 flank_e32 894 +chr6 160548477 160548659 flank_e31 1092 +chr6 160556024 160556184 flank_e30 960 +chr6 160557389 160557571 flank_e29 1092 +chr6 160577135 160577295 flank_e28 960 +chr6 160578522 160578704 flank_e27 1092 +chr6 160585045 160585205 flank_e26 960 +chr6 160586448 160586630 flank_e25 1092 +chr6 160589552 160589712 flank_e24 960 +chr6 160590943 160591101 flank_e23 948 +chr6 160593957 160594117 flank_e22 960 +chr6 160595353 160595535 flank_e21 1092 +chr6 160599499 160599659 flank_e20 960 +chr6 160600916 160601098 flank_e19 1092 +chr6 160611561 160611721 kiv2_e16 219 +chr6 160612931 160613113 kiv2_e15 250 +chr6 160617116 160617276 kiv2_e14 219 +chr6 160618483 160618665 kiv2_e13 250 +chr6 160622662 160622822 kiv2_e12 219 +chr6 160624029 160624211 kiv2_e11 250 +chr6 160628206 160628366 kiv2_e10 219 +chr6 160629573 160629755 kiv2_e09 250 +chr6 160633752 160633912 kiv2_e08 219 +chr6 160635122 160635304 kiv2_e07 250 +chr6 160639299 160639459 kiv2_e06 219 +chr6 160640666 160640848 kiv2_e05 250 +chr6 160644846 160645006 kiv2_e04 219 +chr6 160646213 160646395 kiv2_e03 250 diff --git a/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv b/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv new file mode 100644 index 000000000..cf3ca6f12 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10971 +chr6 160612931 160613113 kiv2_e15 12480 +chr6 160617116 160617276 kiv2_e14 10971 +chr6 160618483 160618665 kiv2_e13 12480 +chr6 160622662 160622822 kiv2_e12 10971 +chr6 160624029 160624211 kiv2_e11 12480 +chr6 160628206 160628366 kiv2_e10 10971 +chr6 160629573 160629755 kiv2_e09 12480 +chr6 160633752 160633912 kiv2_e08 10971 +chr6 160635122 160635304 kiv2_e07 12480 +chr6 160639299 160639459 kiv2_e06 10971 +chr6 160640666 160640848 kiv2_e05 12480 +chr6 160644846 160645006 kiv2_e04 10971 +chr6 160646213 160646395 kiv2_e03 12480 diff --git a/tests/parity/inputs/cn/lpa-mapq0.q1.tsv b/tests/parity/inputs/cn/lpa-mapq0.q1.tsv new file mode 100644 index 000000000..9039bd44a --- /dev/null +++ b/tests/parity/inputs/cn/lpa-mapq0.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10423 +chr6 160612931 160613113 kiv2_e15 11856 +chr6 160617116 160617276 kiv2_e14 10423 +chr6 160618483 160618665 kiv2_e13 11856 +chr6 160622662 160622822 kiv2_e12 10423 +chr6 160624029 160624211 kiv2_e11 11856 +chr6 160628206 160628366 kiv2_e10 10423 +chr6 160629573 160629755 kiv2_e09 11856 +chr6 160633752 160633912 kiv2_e08 10423 +chr6 160635122 160635304 kiv2_e07 11856 +chr6 160639299 160639459 kiv2_e06 10423 +chr6 160640666 160640848 kiv2_e05 11856 +chr6 160644846 160645006 kiv2_e04 10423 +chr6 160646213 160646395 kiv2_e03 11856 diff --git a/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv b/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv new file mode 100644 index 000000000..cf3ca6f12 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10971 +chr6 160612931 160613113 kiv2_e15 12480 +chr6 160617116 160617276 kiv2_e14 10971 +chr6 160618483 160618665 kiv2_e13 12480 +chr6 160622662 160622822 kiv2_e12 10971 +chr6 160624029 160624211 kiv2_e11 12480 +chr6 160628206 160628366 kiv2_e10 10971 +chr6 160629573 160629755 kiv2_e09 12480 +chr6 160633752 160633912 kiv2_e08 10971 +chr6 160635122 160635304 kiv2_e07 12480 +chr6 160639299 160639459 kiv2_e06 10971 +chr6 160640666 160640848 kiv2_e05 12480 +chr6 160644846 160645006 kiv2_e04 10971 +chr6 160646213 160646395 kiv2_e03 12480 diff --git a/tests/parity/inputs/cn/lpa-share1996.q1.tsv b/tests/parity/inputs/cn/lpa-share1996.q1.tsv new file mode 100644 index 000000000..fff6ba5cd --- /dev/null +++ b/tests/parity/inputs/cn/lpa-share1996.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 8782 +chr6 160612931 160613113 kiv2_e15 9989 +chr6 160617116 160617276 kiv2_e14 8782 +chr6 160618483 160618665 kiv2_e13 9989 +chr6 160622662 160622822 kiv2_e12 8782 +chr6 160624029 160624211 kiv2_e11 9989 +chr6 160628206 160628366 kiv2_e10 8782 +chr6 160629573 160629755 kiv2_e09 9989 +chr6 160633752 160633912 kiv2_e08 8782 +chr6 160635122 160635304 kiv2_e07 9989 +chr6 160639299 160639459 kiv2_e06 8782 +chr6 160640666 160640848 kiv2_e05 9989 +chr6 160644846 160645006 kiv2_e04 8782 +chr6 160646213 160646395 kiv2_e03 9989 diff --git a/tests/parity/inputs/cn/lpa-table.tsv b/tests/parity/inputs/cn/lpa-table.tsv new file mode 100644 index 000000000..48563d9dd --- /dev/null +++ b/tests/parity/inputs/cn/lpa-table.tsv @@ -0,0 +1,5 @@ +metric value +status not assessed +kiv2_copies 28.8 +mapq0_share 0.199 +message x diff --git a/tests/parity/inputs/cn/lpa12.bedcov.tsv b/tests/parity/inputs/cn/lpa12.bedcov.tsv new file mode 100644 index 000000000..184185374 --- /dev/null +++ b/tests/parity/inputs/cn/lpa12.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 4800 +chr6 160612931 160613113 kiv2_e15 5460 +chr6 160617116 160617276 kiv2_e14 4800 +chr6 160618483 160618665 kiv2_e13 5460 +chr6 160622662 160622822 kiv2_e12 4800 +chr6 160624029 160624211 kiv2_e11 5460 +chr6 160628206 160628366 kiv2_e10 4800 +chr6 160629573 160629755 kiv2_e09 5460 +chr6 160633752 160633912 kiv2_e08 4800 +chr6 160635122 160635304 kiv2_e07 5460 +chr6 160639299 160639459 kiv2_e06 4800 +chr6 160640666 160640848 kiv2_e05 5460 +chr6 160644846 160645006 kiv2_e04 4800 +chr6 160646213 160646395 kiv2_e03 5460 diff --git a/tests/parity/inputs/cn/lpa12.q1.tsv b/tests/parity/inputs/cn/lpa12.q1.tsv new file mode 100644 index 000000000..823c0dbf9 --- /dev/null +++ b/tests/parity/inputs/cn/lpa12.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 480 +chr6 160612931 160613113 kiv2_e15 546 +chr6 160617116 160617276 kiv2_e14 480 +chr6 160618483 160618665 kiv2_e13 546 +chr6 160622662 160622822 kiv2_e12 480 +chr6 160624029 160624211 kiv2_e11 546 +chr6 160628206 160628366 kiv2_e10 480 +chr6 160629573 160629755 kiv2_e09 546 +chr6 160633752 160633912 kiv2_e08 480 +chr6 160635122 160635304 kiv2_e07 546 +chr6 160639299 160639459 kiv2_e06 480 +chr6 160640666 160640848 kiv2_e05 546 +chr6 160644846 160645006 kiv2_e04 480 +chr6 160646213 160646395 kiv2_e03 546 diff --git a/tests/parity/inputs/cn/lpa29.bedcov.tsv b/tests/parity/inputs/cn/lpa29.bedcov.tsv new file mode 100644 index 000000000..9005e6721 --- /dev/null +++ b/tests/parity/inputs/cn/lpa29.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2115 +chr6 160547788 160547937 flank_e32 5588 +chr6 160548477 160548659 flank_e31 4095 +chr6 160556024 160556184 flank_e30 6000 +chr6 160557389 160557571 flank_e29 4095 +chr6 160577135 160577295 flank_e28 6000 +chr6 160578522 160578704 flank_e27 4095 +chr6 160585045 160585205 flank_e26 6000 +chr6 160586448 160586630 flank_e25 4095 +chr6 160589552 160589712 flank_e24 6000 +chr6 160590943 160591101 flank_e23 3555 +chr6 160593957 160594117 flank_e22 6000 +chr6 160595353 160595535 flank_e21 4095 +chr6 160599499 160599659 flank_e20 6000 +chr6 160600916 160601098 flank_e19 4095 +chr6 160611561 160611721 kiv2_e16 13286 +chr6 160612931 160613113 kiv2_e15 9068 +chr6 160617116 160617276 kiv2_e14 13286 +chr6 160618483 160618665 kiv2_e13 9068 +chr6 160622662 160622822 kiv2_e12 13286 +chr6 160624029 160624211 kiv2_e11 9068 +chr6 160628206 160628366 kiv2_e10 13286 +chr6 160629573 160629755 kiv2_e09 9068 +chr6 160633752 160633912 kiv2_e08 13286 +chr6 160635122 160635304 kiv2_e07 9068 +chr6 160639299 160639459 kiv2_e06 13286 +chr6 160640666 160640848 kiv2_e05 9068 +chr6 160644846 160645006 kiv2_e04 13286 +chr6 160646213 160646395 kiv2_e03 9068 diff --git a/tests/parity/inputs/cn/lpa29.q1.tsv b/tests/parity/inputs/cn/lpa29.q1.tsv new file mode 100644 index 000000000..96a717a9c --- /dev/null +++ b/tests/parity/inputs/cn/lpa29.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2115 +chr6 160547788 160547937 flank_e32 5588 +chr6 160548477 160548659 flank_e31 4095 +chr6 160556024 160556184 flank_e30 6000 +chr6 160557389 160557571 flank_e29 4095 +chr6 160577135 160577295 flank_e28 6000 +chr6 160578522 160578704 flank_e27 4095 +chr6 160585045 160585205 flank_e26 6000 +chr6 160586448 160586630 flank_e25 4095 +chr6 160589552 160589712 flank_e24 6000 +chr6 160590943 160591101 flank_e23 3555 +chr6 160593957 160594117 flank_e22 6000 +chr6 160595353 160595535 flank_e21 4095 +chr6 160599499 160599659 flank_e20 6000 +chr6 160600916 160601098 flank_e19 4095 +chr6 160611561 160611721 kiv2_e16 1329 +chr6 160612931 160613113 kiv2_e15 907 +chr6 160617116 160617276 kiv2_e14 1329 +chr6 160618483 160618665 kiv2_e13 907 +chr6 160622662 160622822 kiv2_e12 1329 +chr6 160624029 160624211 kiv2_e11 907 +chr6 160628206 160628366 kiv2_e10 1329 +chr6 160629573 160629755 kiv2_e09 907 +chr6 160633752 160633912 kiv2_e08 1329 +chr6 160635122 160635304 kiv2_e07 907 +chr6 160639299 160639459 kiv2_e06 1329 +chr6 160640666 160640848 kiv2_e05 907 +chr6 160644846 160645006 kiv2_e04 1329 +chr6 160646213 160646395 kiv2_e03 907 diff --git a/tests/parity/inputs/cn/lpa47.bedcov.tsv b/tests/parity/inputs/cn/lpa47.bedcov.tsv new file mode 100644 index 000000000..b46886e05 --- /dev/null +++ b/tests/parity/inputs/cn/lpa47.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2679 +chr6 160547788 160547937 flank_e32 4694 +chr6 160548477 160548659 flank_e31 5187 +chr6 160556024 160556184 flank_e30 5040 +chr6 160557389 160557571 flank_e29 5187 +chr6 160577135 160577295 flank_e28 5040 +chr6 160578522 160578704 flank_e27 5187 +chr6 160585045 160585205 flank_e26 5040 +chr6 160586448 160586630 flank_e25 5187 +chr6 160589552 160589712 flank_e24 5040 +chr6 160590943 160591101 flank_e23 4503 +chr6 160593957 160594117 flank_e22 5040 +chr6 160595353 160595535 flank_e21 5187 +chr6 160599499 160599659 flank_e20 5040 +chr6 160600916 160601098 flank_e19 5187 +chr6 160611561 160611721 kiv2_e16 18480 +chr6 160612931 160613113 kiv2_e15 17199 +chr6 160617116 160617276 kiv2_e14 18480 +chr6 160618483 160618665 kiv2_e13 17199 +chr6 160622662 160622822 kiv2_e12 18480 +chr6 160624029 160624211 kiv2_e11 17199 +chr6 160628206 160628366 kiv2_e10 18480 +chr6 160629573 160629755 kiv2_e09 17199 +chr6 160633752 160633912 kiv2_e08 18480 +chr6 160635122 160635304 kiv2_e07 17199 +chr6 160639299 160639459 kiv2_e06 18480 +chr6 160640666 160640848 kiv2_e05 17199 +chr6 160644846 160645006 kiv2_e04 18480 +chr6 160646213 160646395 kiv2_e03 17199 diff --git a/tests/parity/inputs/cn/lpa47.q1.tsv b/tests/parity/inputs/cn/lpa47.q1.tsv new file mode 100644 index 000000000..6df1103e9 --- /dev/null +++ b/tests/parity/inputs/cn/lpa47.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2679 +chr6 160547788 160547937 flank_e32 4694 +chr6 160548477 160548659 flank_e31 5187 +chr6 160556024 160556184 flank_e30 5040 +chr6 160557389 160557571 flank_e29 5187 +chr6 160577135 160577295 flank_e28 5040 +chr6 160578522 160578704 flank_e27 5187 +chr6 160585045 160585205 flank_e26 5040 +chr6 160586448 160586630 flank_e25 5187 +chr6 160589552 160589712 flank_e24 5040 +chr6 160590943 160591101 flank_e23 4503 +chr6 160593957 160594117 flank_e22 5040 +chr6 160595353 160595535 flank_e21 5187 +chr6 160599499 160599659 flank_e20 5040 +chr6 160600916 160601098 flank_e19 5187 +chr6 160611561 160611721 kiv2_e16 1848 +chr6 160612931 160613113 kiv2_e15 1720 +chr6 160617116 160617276 kiv2_e14 1848 +chr6 160618483 160618665 kiv2_e13 1720 +chr6 160622662 160622822 kiv2_e12 1848 +chr6 160624029 160624211 kiv2_e11 1720 +chr6 160628206 160628366 kiv2_e10 1848 +chr6 160629573 160629755 kiv2_e09 1720 +chr6 160633752 160633912 kiv2_e08 1848 +chr6 160635122 160635304 kiv2_e07 1720 +chr6 160639299 160639459 kiv2_e06 1848 +chr6 160640666 160640848 kiv2_e05 1720 +chr6 160644846 160645006 kiv2_e04 1848 +chr6 160646213 160646395 kiv2_e03 1720 diff --git a/tests/parity/inputs/cn/oracle.sh b/tests/parity/inputs/cn/oracle.sh new file mode 100644 index 000000000..09cfdb7d7 --- /dev/null +++ b/tests/parity/inputs/cn/oracle.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# The oracle of og cn (tests/parity/oracle.ref): bin/copy_number.py with the +# modules it imported (staged by gen/cn-oracle.sh), each og cn verb turned +# into the old command line. +# sh ./oracle.sh PROG VERB [ARGS] +# call ... --marker LPA_KIV2_4925 --marker-pileup P --marker-out O +# copy_number.py call ..., then lpa_kiv2.py null-allele --table T +# marker-regions NAME traits_report.py --marker-regions NAME +# marker-read NAME --pileup P [--copy-table T] --out O +# traits_report.py --marker NAME ... --marker-out O +d=$(mktemp -d) || exit 70 +trap 'rm -rf "$d"' EXIT +cp "$1" "${d}/copy_number.py" +cp hba_copy_number.py lpa_kiv2.py traits_report.py cn_loci.tsv "$d"/ +shift +verb=$1 +shift +case "$verb" in + marker-regions) python3 -B "${d}/traits_report.py" --marker-regions "$@"; exit $? ;; + marker-read) + name=$1 + shift + set -- "$@" --end + while [ "$1" != --end ]; do + case "$1" in + --out) set -- "$@" --marker-out "$2" ;; + --catalog) ;; + *) set -- "$@" "$1" "$2" ;; + esac + shift 2 + done + shift + python3 -B "${d}/traits_report.py" --marker "$name" "$@" + exit $? + ;; + call) + pileup="" out="" table="" + set -- "$@" --end + while [ "$1" != --end ]; do + case "$1" in + --marker) ;; + --marker-pileup) pileup=$2 ;; + --marker-out) out=$2 ;; + --table) table=$2; set -- "$@" "$1" "$2" ;; + *) set -- "$@" "$1" "$2" ;; + esac + shift 2 + done + shift + python3 -B "${d}/copy_number.py" call "$@" || exit $? + [ -z "$out" ] || python3 -B "${d}/lpa_kiv2.py" null-allele --pileup "$pileup" --table "$table" --out "$out" + exit $? + ;; + *) python3 -B "${d}/copy_number.py" "$verb" "$@"; exit $? ;; +esac diff --git a/tests/parity/inputs/cn/rhd0.bedcov.tsv b/tests/parity/inputs/cn/rhd0.bedcov.tsv new file mode 100644 index 000000000..1783b7afc --- /dev/null +++ b/tests/parity/inputs/cn/rhd0.bedcov.tsv @@ -0,0 +1,13 @@ +chr1 25199999 25260000 RHD_FLANK_UP 1800030 +chr1 25272392 25330445 RHD_GENE 0 +chr1 25272508 25272695 RHD_EX1 0 +chr1 25284572 25284759 RHD_EX2 0 +chr1 25290640 25290791 RHD_EX3 0 +chr1 25300945 25301093 RHD_EX4 0 +chr1 25301519 25301686 RHD_EX5 0 +chr1 25303321 25303459 RHD_EX6 0 +chr1 25306595 25306729 RHD_EX7 0 +chr1 25316999 25317079 RHD_EX8 0 +chr1 25321888 25321962 RHD_EX9 0 +chr1 25328897 25330445 RHD_EX10 0 +chr1 25334999 25360000 RHD_FLANK_DOWN 750030 diff --git a/tests/parity/inputs/cn/rhd1.bedcov.tsv b/tests/parity/inputs/cn/rhd1.bedcov.tsv new file mode 100644 index 000000000..4547d8942 --- /dev/null +++ b/tests/parity/inputs/cn/rhd1.bedcov.tsv @@ -0,0 +1,13 @@ +chr1 25199999 25260000 RHD_FLANK_UP 1800030 +chr1 25272392 25330445 RHD_GENE 870795 +chr1 25272508 25272695 RHD_EX1 5610 +chr1 25284572 25284759 RHD_EX2 5610 +chr1 25290640 25290791 RHD_EX3 4530 +chr1 25300945 25301093 RHD_EX4 4440 +chr1 25301519 25301686 RHD_EX5 5010 +chr1 25303321 25303459 RHD_EX6 4140 +chr1 25306595 25306729 RHD_EX7 4020 +chr1 25316999 25317079 RHD_EX8 2400 +chr1 25321888 25321962 RHD_EX9 2220 +chr1 25328897 25330445 RHD_EX10 46440 +chr1 25334999 25360000 RHD_FLANK_DOWN 750030 diff --git a/tests/parity/inputs/cn/smn-lowq.tsv b/tests/parity/inputs/cn/smn-lowq.tsv new file mode 100644 index 000000000..0d52f91be --- /dev/null +++ b/tests/parity/inputs/cn/smn-lowq.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 LowQual 4 12 LowQual 2,2 12,12 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/smn-notseen.pileup b/tests/parity/inputs/cn/smn-notseen.pileup new file mode 100644 index 000000000..55f07ef90 --- /dev/null +++ b/tests/parity/inputs/cn/smn-notseen.pileup @@ -0,0 +1 @@ +chr5 70952074 N 30 TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT III diff --git a/tests/parity/inputs/cn/smn-present.pileup b/tests/parity/inputs/cn/smn-present.pileup new file mode 100644 index 000000000..b4375e16d --- /dev/null +++ b/tests/parity/inputs/cn/smn-present.pileup @@ -0,0 +1,3 @@ +chr5 70952074 N 30 TTTTTTTTTTTTTTTTTTTTTTTTTGGGGG III +chr5 70076654 N 30 tttttttttttttttttttttttttttttt*<> III +chr5 70000000 N 30 GGGGGGGGGGGGGGGGGGGGGGGGGGGGGG III diff --git a/tests/parity/inputs/cn/smn-two-rows.tsv b/tests/parity/inputs/cn/smn-two-rows.tsv new file mode 100644 index 000000000..9e0f68c72 --- /dev/null +++ b/tests/parity/inputs/cn/smn-two-rows.tsv @@ -0,0 +1,3 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70951000 70953000 SMN1 PASS 4 30 LowQual 1,3 12,40 chr5:70075000-70078000 +chr5 70951900 70952100 SMN1 PASS 5 30 PASS 2,2 40,40 chr5:70076000-70076700 diff --git a/tests/parity/inputs/cn/smn1.tsv b/tests/parity/inputs/cn/smn1.tsv new file mode 100644 index 000000000..b5fe09c78 --- /dev/null +++ b/tests/parity/inputs/cn/smn1.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 PASS 3 30 PASS 1,2 40,40 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/smn2.tsv b/tests/parity/inputs/cn/smn2.tsv new file mode 100644 index 000000000..a8858aa7d --- /dev/null +++ b/tests/parity/inputs/cn/smn2.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 PASS 4 30 PASS 2,2 40,40 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/stdin b/tests/parity/inputs/cn/stdin new file mode 100644 index 000000000..7a4cb7036 --- /dev/null +++ b/tests/parity/inputs/cn/stdin @@ -0,0 +1,2 @@ +chr1 +chr2_KI270769v1_alt diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index b36653ef1..3f1623af8 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -39,3 +39,4 @@ viral-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/viral_report.py p y-str-bed 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/ystr_report.py sh -c 'exec python3 "$0" bed "$@"' y str-bed y-str-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/ystr_report.py sh -c 'exec python3 "$0" report "$@"' y str-report exomiser 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/exomiser_report.py python3 exomiser +cn 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/copy_number.py sh ./oracle.sh cn diff --git a/tests/test_callability.py b/tests/test_callability.py index f5048e717..c0696d9b7 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -23,11 +23,12 @@ unreadable, not a reading; a chromosome with no row at all (a BAM whose reference lacks it, as the e2e fixture's) puts its genes under "not on this reference"; - 5. bin/copy_number.py check-bam and call --bam-header refuse a header with - an ALT or an HLA contig (exit 4, a REFUSED line) and pass a header - without one (the positive control). + 5. og cn check-bam and call --bam-header refuse a header with an ALT or + an HLA contig (exit 4, a REFUSED line) and pass a header without one + (the positive control). They need og: OG names the binary; without it + part 5 is skipped, and with OG_REQUIRED set too the test fails. -Run: python3 tests/test_callability.py +Run: OG=rust/target/release/og python3 tests/test_callability.py """ import csv import gzip @@ -203,33 +204,50 @@ def count_matches(reader, tsv): # --- 5. the ALT contig refusal --------------------------------------------------------- +OG = os.environ.get("OG", "") +if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): + # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job + # builds og in a temporary folder, so the tests run a copy that sits next + # to the catalog, as bin/.og/og-* does in a checkout. + import atexit, shutil # noqa: E401 + _og_dir = tempfile.mkdtemp(prefix="og-cn-") + atexit.register(shutil.rmtree, _og_dir, True) + shutil.copy2(OG, os.path.join(_og_dir, "og")) + shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) + OG = os.environ["OG"] = os.path.join(_og_dir, "og") + + def copy_number(*args, stdin=None): - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "copy_number.py"), *args], - capture_output=True, text=True, input=stdin) + p = subprocess.run([OG, "cn", *args], capture_output=True, text=True, input=stdin) return p.returncode, p.stderr -with tempfile.TemporaryDirectory() as work: - noalt = os.path.join(work, "noalt.txt") - with open(noalt, "w") as f: - f.write("@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n") - alt = os.path.join(work, "alt.txt") - with open(alt, "w") as f: - f.write("@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n") - rc, err = copy_number("check-bam", "--header", noalt) - check("5. positive control: a header without ALT contigs passes", rc == 0 and not err, (rc, err)) - rc, err = copy_number("check-bam", "--header", alt) - check("5. a header with an ALT contig is refused (exit 4, a REFUSED line naming it)", - rc == 4 and err.startswith("REFUSED:") and "chr22_KI270879v1_alt" in err, (rc, err)) - rc, err = copy_number("check-bam", "--header", "-", stdin="chr1\nHLA-A*01:01:01:01\n") - check("5. contig names on stdin with an HLA contig are refused", rc == 4 and "HLA-A" in err, (rc, err)) - bed = os.path.join(work, "bedcov.tsv") - open(bed, "w").close() - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", alt) - check("5. call --bam-header refuses before it reads any depth", rc == 4 and err.startswith("REFUSED:"), - (rc, err)) - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", noalt) - check("5. call --bam-header without ALT contigs goes on (two empty tables: not assessed)", rc == 0, (rc, err)) +if not OG: + print("skip: OG is not set (OG=rust/target/release/og): part 5 did not run") + if os.environ.get("OG_REQUIRED"): + FAILS += 1 +else: + with tempfile.TemporaryDirectory() as work: + noalt = os.path.join(work, "noalt.txt") + with open(noalt, "w") as f: + f.write("@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n") + alt = os.path.join(work, "alt.txt") + with open(alt, "w") as f: + f.write("@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n") + rc, err = copy_number("check-bam", "--header", noalt) + check("5. positive control: a header without ALT contigs passes", rc == 0 and not err, (rc, err)) + rc, err = copy_number("check-bam", "--header", alt) + check("5. a header with an ALT contig is refused (exit 4, a REFUSED line naming it)", + rc == 4 and err.startswith("REFUSED:") and "chr22_KI270879v1_alt" in err, (rc, err)) + rc, err = copy_number("check-bam", "--header", "-", stdin="chr1\nHLA-A*01:01:01:01\n") + check("5. contig names on stdin with an HLA contig are refused", rc == 4 and "HLA-A" in err, (rc, err)) + bed = os.path.join(work, "bedcov.tsv") + open(bed, "w").close() + rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", alt) + check("5. call --bam-header refuses before it reads any depth", rc == 4 and err.startswith("REFUSED:"), + (rc, err)) + rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", noalt) + check("5. call --bam-header without ALT contigs goes on (two empty tables: not assessed)", rc == 0, (rc, err)) print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") sys.exit(1 if FAILS else 0) diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 30b7ccb87..36cb9bc80 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1666,16 +1666,23 @@ def parascopy_section(work, schema): and "not a carrier by copy number" in card_s and "CYP21A2 and the gene-dosage loci have no Parascopy model" in card_s and "step 35 counts them from read depth against flanking sequence" in card_s and "Not checked" not in html, card_s) - # SMN1 g.27134T>G beside the psCN (step 35's _smn1_marker.tsv, bin/traits_report.py smn1_marker) + # SMN1 g.27134T>G beside the psCN (step 35's _smn1_marker.tsv, og cn marker-read SMN1_27134) caveat = render_report.SMN1_MARKER_CAVEAT after_cn = " SMN1 copies: 2 (quality 33, filter PASS)\n SMN1 g.27134T>G silent-carrier marker (rs143838139): " check("parascopy: without step 35's marker table the marker line says not assessed, beside the psCN, with why", (pd.get("smn1_marker") or {}).get("status") == "not assessed" and after_cn + "not assessed\n" in txt and "step 35 wrote no S_smn1_marker.tsv" in txt and f" SMN1 marker caveat: {caveat}\n" in txt, txt) - import traits_report mk = f"{d}/paralogs/S_smn1_marker.tsv" - traits_report.write_metric_table(mk, traits_report.smn1_marker((35, 5, 0), f"{d}/paralogs/S_smn_copy_number.tsv")) + # What og cn marker-read SMN1_27134 writes for 5 of 40 reads with G and this SMN table. + put(mk, "metric\tvalue\nmarker\tSMN1 g.27134T>G\nrsid\trs143838139\nstatus\tpresent\nalt_reads\t5\n" + "depth\t40\nfraction\t0.1250\ncopies\t0.5\nsmn1_pscn\t2 (quality 33, filter PASS)\nagcn\t4\n" + "message\tSMN1 g.27134T>G (rs143838139): present, 5 of 40 reads carry G at the SMN1 and SMN2 copies of the " + "site together (about 0.5 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or " + "SMN2. With two SMN1 copies, this marker goes with a higher chance that both sit on one chromosome (2+0, a " + "silent carrier): in Luo 2014 it raised the residual carrier risk of a two-copy result (in African Americans " + "from about 1 in 121 to 1 in 34); the figures depend on ancestry. This marker does not prove 2+0, and its " + "absence does not rule 2+0 out.\n") summ_m, txt_m, html_m = render(d) card_m = html_m.split("

SMN1/SMN2 Copy Number (Parascopy)

", 1)[-1].split("

", 1)[0] check("parascopy: a marker table with 5 of 40 reads reads as present beside the psCN in the text report, " @@ -1702,7 +1709,7 @@ def parascopy_section(work, schema): any(x.startswith("HBA1/HBA2 copy number") for x in summ["not_assessed"]) and "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count" in txt and "HBA1 and HBA2 together" not in txt, summ["not_assessed"]) - # Step 35's HBA table (bin/hba_copy_number.py): four genes, then a planted three. + # Step 35's HBA table (og cn call --locus HBA): four genes, then a planted three. hba_head = "metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t" put(f"{d}/paralogs/S_hba_copy_number.tsv", hba_head + "4\nestimate\t4.02\nhba2_depth\t30.1\n" "hba1_depth\t30.2\nflank_left_depth\t30.0\nflank_right_depth\t30.0\n" diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 9665db09b..b00a00302 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 -"""bin/copy_number.py and bin/cn_loci.tsv: one depth-ratio engine for every -copy-number locus. +"""og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number +locus, and what bin/copy_number.py keeps (the catalog reader and assess(), +which step 41's report imports). + +Without og (OG unset) the checks of the catalog and of assess() run, and the +wiring; with OG naming the og binary (OG=rust/target/release/og, or bin/og) +also those of og cn: the bedcov reader, parity, the CLI, and the blocks of +step 35 cut out and run. With OG_REQUIRED set, a run without og fails. 1. the catalog: every row loads with its columns checked; a broken row (a bad kind, a segment without a span, a class call without an @@ -9,19 +15,19 @@ (main or paralog copy) of a model the pinned Parascopy data holds (assets/parascopy_models_v1.7.txt), except the pairs EXEMPT lists; a planted AMY1C row and a planted row on C4A's copy (C4B) are caught; - 3. parity: the depths of tests/test_hba_copy_number.py and - tests/test_lpa_kiv2.py give, through the engine, the very tables the - code before the engine wrote (GOLD, frozen from it); + 3. parity: made-up HBA and LPA depths give, through og cn call, the very + tables the code before the engine wrote (GOLD, frozen from it); 4. synthetic bedcov tables at 0, 1, 2, 3 and 4 copies of a made-up gene row are each called right; a MAPQ 0 share above the row's ceiling is not assessed; a kind=interval row with pooled depths works the same; 5. a control segment with no read while the other controls are covered raises BrokenControl, and `call` exits 3 naming BROKEN_CONTROL; with no read anywhere it is 'not assessed', not an error; - 6. the CLI: `bed` prints every bedcov locus, `call --json` writes one block; - 7. the wiring: steps 35, 41 and 46 and their Nextflow modules call - copy_number.py, never hba_copy_number.py or lpa_kiv2.py, with the - bedcov options of the catalog row. UGT2B17 has no row: pypgx calls it + 6. the CLI: `bed` prints every bedcov locus, `call --table` writes step + 46's table; og's catalog reader refuses the broken rows the Python + refuses, with the same message; + 7. the wiring: steps 35, 41 and 46 and their Nextflow modules call og cn, + never the old Python, with the bedcov options of the catalog row. UGT2B17 has no row: pypgx calls it in BAM mode (assets/pgx_genes.tsv), and tests/test_pypgx_genes.sh checks that no catalog row names a gene pypgx calls from the BAM; 8. the gene-dosage rows (HP, LCE3BC, CCL3L1): made-up depths at 0 @@ -56,7 +62,7 @@ the same commands; the couple screen counts the deletion and the conversion as CYP21A2 findings and the unmodified table as none. -Run: python3 tests/test_copy_number.py +Run: OG=rust/target/release/og python3 tests/test_copy_number.py """ import json import os @@ -66,17 +72,25 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "copy_number.py") +OG = os.environ.get("OG", "") +if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): + # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job + # builds og in a temporary folder, so the tests run a copy that sits next + # to the catalog, as bin/.og/og-* does in a checkout. + import atexit, shutil # noqa: E401 + _og_dir = tempfile.mkdtemp(prefix="og-cn-") + atexit.register(shutil.rmtree, _og_dir, True) + shutil.copy2(OG, os.path.join(_og_dir, "og")) + shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) + OG = os.environ["OG"] = os.path.join(_og_dir, "og") CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") MODELS = os.path.join(REPO, "assets", "parascopy_models_v1.7.txt") sys.path.insert(0, os.path.join(REPO, "bin")) import copy_number as cn # noqa: E402 -import hba_copy_number as hba # noqa: E402 -import lpa_kiv2 as lpa # noqa: E402 FAILS = 0 # Frozen from bin/hba_copy_number.py and bin/lpa_kiv2.py as they were before the -# engine (main at 6056452): inputs [depths..., MAPQ 0 share] and the dict assess() gave. +# engine (main at 6056452): inputs [depths..., MAPQ 0 share] and the table rows they wrote. GOLD = json.loads(''.join(( '{"hba":[[[30,30,30,30,0.25],{"hba2_depth":30,"hba1_depth":30,"flank_left_depth":30,"flank_right_depth":30,"map' 'q0_share":"0.250","estimate":"4.00","alpha_globin_genes":"4","status":"called","message":"four alpha-globin ge' @@ -192,10 +206,40 @@ def write_catalog(path, rows): f.write("\t".join(r[c] for c in cn.COLUMNS) + "\n") -def bedcov(path, prow, depth): +def bedcov(path, prow, depth, exact=False): + """samtools bedcov over a row's segments at these mean depths: whole sums, + or with exact the sums themselves, so each mean comes back as given.""" with open(path, "w") as f: for c, s, e, n in cn.regions(prow): - f.write(f"{c}\t{s}\t{e}\t{n}\t{round(depth[n] * (e - s))}\n") + v = depth[n] * (e - s) + f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") + + +def og_cn(*args, **kw): + """og cn with these arguments.""" + return subprocess.run([OG, "cn", *args], capture_output=True, text=True, **kw) + + +def table(path): + """{metric: value} of a metricvalue table, {} when it is not there.""" + if not os.path.exists(path): + return {} + return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) + + +# Step 46's table, in its order (what bin/lpa_kiv2.py wrote). +LPA_ORDER = ["status", "kiv2_copies", "ci95_low", "ci95_high", "class", "kiv2_depth", "flank_depth", "mapq0_share", + "message", "advice"] +# Step 35's interval table (what copy_number.py intervals wrote). +INTERVAL_COLUMNS = ["locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", + "reason"] + + +def change(b): + """The change column of an interval row for an assess() block.""" + if b["copies"] is None or b["expected"] is None: + return "." + return "deletion" if b["copies"] < b["expected"] else "duplication" if b["copies"] > b["expected"] else "no change" # The only rows allowed to overlap a Parascopy model, each for a stated reason. @@ -255,6 +299,7 @@ def main(): want = {"low", "typical", "high"} if mode == "class" else {str(i) for i in range(int(len(vals) + 1))} \ if mode == "cut" else None check(f"catalog {loc}: a reading for every call", want is None or want <= set(readings), readings) + broken = [] with tempfile.TemporaryDirectory() as t: p = os.path.join(t, "c.tsv") for desc, kw, err in (("a bad kind", {"kind": "exon"}, "kind"), @@ -272,6 +317,7 @@ def main(): except cn.CatalogError as e: got = str(e) check(f"catalog: {desc} is refused", err in got, got) + broken.append((desc, [row(**kw)], got)) write_catalog(p, [row(), row(locus="TESTH", targets="TESTG:80000-81000", controls="fl_c:1-2;fl_d:3-4")]) try: cn.load_catalog(p) @@ -280,30 +326,6 @@ def main(): got = str(e) check("catalog: one segment name in two loci is refused", "used by two loci" in got, got) - g0 = cn.parse_row(row()) - b = os.path.join(t, "dup.bedcov") - bedcov(b, g0, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0}) - with open(b) as f: - first = f.readline() - with open(b, "a") as f: - f.write(first.rsplit("\t", 1)[0] + "\t0\n") - try: - cn.read_bedcov(b, g0) - got = "read" - except ValueError as e: - got = str(e) - check("bedcov: a region listed twice is refused, not overwritten", "appears twice" in got, got) - with open(b, "w") as f: - f.write(first.split("\t")[0] + "\t50000\t50000\tTESTG\t0\n") - try: - cn.read_bedcov(b, g0) - got = "read" - except ValueError as e: - got = str(e) - except ZeroDivisionError as e: - got = f"ZeroDivisionError: {e}" - check("bedcov: an empty region is refused with its name", "is empty or reversed" in got, got) - # 2. one owner per locus models = {} with open(MODELS) as f: @@ -355,23 +377,6 @@ def main(): check("owner: RHD overlaps RHCE's model only (a paralog copy), the exempt pair", {m for m, _ in overlaps(cat["RHD"], models)} == {"RHCE"}, overlaps(cat["RHD"], models)) - # 3. parity with the code before the engine - for (h2, h1, left, right, share), want in GOLD["hba"]: - m = {"flank_left": left, "HBA2": h2, "HBA1": h1, "flank_right": right} - q = {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in m.items()} - got = hba.assess(m, q) - check(f"parity HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {want['status']} {want['alpha_globin_genes']}", - got == want, (got, want)) - kiv = [n for *_, n in lpa.REGIONS if n.startswith("kiv2_")] - fl = [n for *_, n in lpa.REGIONS if n.startswith("flank_")] - for (n, flank, ks, fs, share), want in GOLD["lpa"]: - k = flank * (n + 2) / 14 - m = {x: k * (1 + (ks if i % 2 == 0 else -ks)) for i, x in enumerate(kiv)} - m.update({x: flank * (1 + (fs if i % 2 == 0 else -fs)) for i, x in enumerate(fl)}) - q = {x: v * (1 - share) if x.startswith("kiv2_") else v for x, v in m.items()} - got = lpa.assess(m, q) - check(f"parity LPA {n} copies at {flank}x: {want['status']} {want['kiv2_copies']}", got == want, (got, want)) - # 4. synthetic copies, the MAPQ 0 ceiling, a kind=interval row g = cn.parse_row(row()) for copies in (0, 1, 2, 3, 4): @@ -430,43 +435,138 @@ def main(): (b["status"], b["reason"]) == ("not assessed", "low_control_depth"), b) b = cn.assess(g, {"fl_a": 0.0, "fl_b": 5.0, "TESTG": 5.0}, {}) check("one flank empty beside a thin one (5x): too little depth, not an error", b["reason"] == "low_control_depth", b) - lrow = cat["LPA_KIV2"] + # 7. the wiring + def read(p): + with open(os.path.join(REPO, p)) as f: + return f.read() + for path, loci in (("scripts/35-paralogs.sh", ["HBA"]), ("modules/local/parascopy/main.nf", ["HBA"]), + ("scripts/41-bloodtype.sh", ["RHD"]), ("modules/local/bloodtype/main.nf", ["RHD"]), + ("scripts/46-lpa.sh", ["LPA_KIV2"]), ("modules/local/lpa/main.nf", ["LPA_KIV2"])): + text = read(path) + for loc in loci: + check(f"{path}: og cn bed --locus {loc}", f"og cn bed --locus {loc}" in text) + if loc != "RHD": # step 41's report imports the engine + check(f"{path}: og cn call --locus {loc}", f"og cn call --locus {loc}" in text) + passes = ["" if p == "." else p for p in cat[loc]["_passes"]] + if loc == "HBA": # and the dosage loop's two passes (section 8), the interval pass (section 9) + passes += ["", "-Q 1", "", ""] # and the CYP21A2 passes (section 10) + passes += ["-Q 1"] + # the commands only: options, then a path, a quoted argument or a variable + calls = [o for o, arg in re.findall(r"samtools bedcov((?: -[A-Za-z](?: \d+)?)*) (\S+)", text) + if re.match(r'["$/]|\S+\.bed\b', arg)] + check(f"{path}: its bedcov passes are the catalog's ({' and '.join(repr(p) for p in passes)})", + sorted(c.strip() for c in calls) == sorted(passes), calls) + check(f"{path}: no call of the old Python", + not re.search(r"(copy_number|hba_copy_number|lpa_kiv2)\.py |traits_report\.py --marker", text)) + + dosage(cat, read) + intervals(cat, read) + psv(cat, read) + if OG: + og_engine(cat, broken) + else: + print("skip: OG is not set (OG=rust/target/release/og): the og cn checks did not run") + + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") + return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 + + +def og_engine(cat, broken): + """Sections 1, 3, 5 and 6 through og cn: its catalog reader, bedcov + reader, the frozen tables, the broken control and the CLI.""" with tempfile.TemporaryDirectory() as t: - a, q, j = (os.path.join(t, x) for x in ("a.tsv", "q.tsv", "o.json")) + p = os.path.join(t, "c.tsv") + for desc, rows, want in broken: + write_catalog(p, rows) + out = og_cn("bed", "--catalog", p) + check(f"og catalog: {desc} is refused with the Python's message", + out.returncode == 2 and out.stderr == f"ERROR: {want}\n", (out.returncode, out.stderr, want)) + + write_catalog(p, [row()]) + g0 = cn.parse_row(row()) + b = os.path.join(t, "dup.bedcov") + bedcov(b, g0, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0}) + with open(b) as f: + first = f.readline() + with open(b, "a") as f: + f.write(first.rsplit("\t", 1)[0] + "\t0\n") + out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) + check("bedcov: a region listed twice is refused, not overwritten", + out.returncode == 2 and "appears twice" in out.stderr, out) + with open(b, "w") as f: + f.write(first.split("\t")[0] + "\t50000\t50000\tTESTG\t0\n") + out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) + check("bedcov: an empty region is refused with its name", + out.returncode == 2 and "is empty or reversed" in out.stderr, out) + + # 3. parity with the code before the engine: every row of the frozen + # table, the depths as numbers + def same(got, want): + def eq(k, v): + if isinstance(v, (int, float)): + try: + return abs(float(got.get(k)) - v) < 1e-9 + except (TypeError, ValueError): + return False + return got.get(k) == v + return all(eq(k, v) for k, v in want.items()) + + a, q, tab = (os.path.join(t, x) for x in ("a.tsv", "q.tsv", "t.tsv")) + hrow = cat["HBA"] + for (h2, h1, left, right, share), want in GOLD["hba"]: + m = {"flank_left": left, "HBA2": h2, "HBA1": h1, "flank_right": right} + bedcov(a, hrow, m, exact=True) + bedcov(q, hrow, {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in m.items()}, exact=True) + out = og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", a, "--bedcov-mapq1", q, + "--table", tab) + got = table(tab) + check(f"parity HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {want['status']} " + f"{want['alpha_globin_genes']}", out.returncode == 0 and same(got, want), (out.stderr, got, want)) + lrow = cat["LPA_KIV2"] + # in catalog order, as bin/lpa_kiv2.py's REGIONS listed them + kiv = [n for n, *_ in lrow["_targets"]] + fl = [n for n, *_ in lrow["_controls"]] + for (n, flank, ks, fs, share), want in GOLD["lpa"]: + k = flank * (n + 2) / 14 + m = {x: k * (1 + (ks if i % 2 == 0 else -ks)) for i, x in enumerate(kiv)} + m.update({x: flank * (1 + (fs if i % 2 == 0 else -fs)) for i, x in enumerate(fl)}) + bedcov(a, lrow, m, exact=True) + bedcov(q, lrow, {x: v * (1 - share) if x.startswith("kiv2_") else v for x, v in m.items()}, exact=True) + out = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, + "--table", tab) + got = table(tab) + check(f"parity LPA {n} copies at {flank}x: {want['status']} {want['kiv2_copies']}", + out.returncode == 0 and same(got, want), (out.stderr, got, want)) + + # 5. a broken control + os.remove(tab) depth = {n: 30.0 for *_, n in cn.regions(lrow)} depth["flank_e19"] = 0.0 bedcov(a, lrow, depth) bedcov(q, lrow, depth) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, - "--bedcov-mapq1", q, "--json", j], capture_output=True, text=True) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) check("broken control: `call` exits 3 with BROKEN_CONTROL and writes nothing", out.returncode == 3 and "BROKEN_CONTROL: control segment flank_e19 of LPA_KIV2" in out.stderr - and not os.path.exists(j), out) + and not os.path.exists(tab), out) # 6. the CLI - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed") names = [x.split("\t")[3] for x in out.stdout.splitlines()] want = sum(len(cn.regions(r)) for r in cat.values() if r["input"] == "bedcov") - check("bed: every segment of the bedcov loci, once", out.returncode == 0 and len(names) == len(set(names)) == want, - (out.returncode, len(names), want)) + check("bed: every segment of the bedcov loci, once (the catalog beside og)", + out.returncode == 0 and len(names) == len(set(names)) == want, (out.returncode, out.stderr, len(names), want)) depth = {n: 30.0 for *_, n in cn.regions(lrow)} for n in depth: if n.startswith("kiv2_"): depth[n] = 30.0 * 22 / 14 bedcov(a, lrow, depth) bedcov(q, lrow, {n: v * (0.1 if n.startswith("kiv2_") else 1) for n, v in depth.items()}) - tab = os.path.join(t, "t.tsv") - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, - "--bedcov-mapq1", q, "--json", j, "--table", tab], capture_output=True, text=True) - block = json.load(open(j)) if os.path.exists(j) else {} - check("call --json: one block, 20 KIV-2 repeats, called", - out.returncode == 0 and block.get("locus") == "LPA_KIV2" and block.get("status") == "called" - and abs(block.get("estimate", 0) - 20) < 0.01, (out, block)) - tab_rows = dict(x.split("\t", 1) for x in open(tab).read().splitlines()[1:]) if os.path.exists(tab) else {} - check("call --table: step 46's table", tab_rows.get("kiv2_copies") == "20.0" and list(tab_rows) == list(lpa.ORDER), - tab_rows) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, "--json", j], - capture_output=True, text=True) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) + tab_rows = table(tab) + check("call --table: step 46's table, 20 KIV-2 repeats, in its order", + out.returncode == 0 and tab_rows.get("status") == "estimated" and tab_rows.get("kiv2_copies") == "20.0" + and list(tab_rows) == LPA_ORDER, (out, tab_rows)) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a) check("call: a row with a MAPQ 0 guard needs --bedcov-mapq1", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) empty = os.path.join(t, "empty.tsv") @@ -474,44 +574,12 @@ def main(): for loc, key in (("HBA", "alpha_globin_genes"), ("LPA_KIV2", "kiv2_copies")): if os.path.exists(tab): os.remove(tab) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", loc, "--bedcov", empty, - "--bedcov-mapq1", empty, "--table", tab], capture_output=True, text=True) - tab_rows = dict(x.split("\t", 1) for x in open(tab).read().splitlines()[1:]) if os.path.exists(tab) else {} + out = og_cn("call", "--locus", loc, "--bedcov", empty, "--bedcov-mapq1", empty, "--table", tab) + tab_rows = table(tab) check(f"call {loc} on two empty tables (no reads on the contig): exit 0, not assessed", out.returncode == 0 and tab_rows.get("status") == "not assessed" and tab_rows.get(key) == "." and "no chr" in tab_rows.get("message", ""), (out, tab_rows)) - # 7. the wiring - def read(p): - with open(os.path.join(REPO, p)) as f: - return f.read() - for path, loci in (("scripts/35-paralogs.sh", ["HBA"]), ("modules/local/parascopy/main.nf", ["HBA"]), - ("scripts/41-bloodtype.sh", ["RHD"]), ("modules/local/bloodtype/main.nf", ["RHD"]), - ("scripts/46-lpa.sh", ["LPA_KIV2"]), ("modules/local/lpa/main.nf", ["LPA_KIV2"])): - text = read(path) - for loc in loci: - check(f"{path}: copy_number.py bed --locus {loc}", f"copy_number.py bed --locus {loc}" in text) - if loc != "RHD": # step 41's report imports the engine - check(f"{path}: copy_number.py call --locus {loc}", f"copy_number.py call --locus {loc}" in text) - passes = ["" if p == "." else p for p in cat[loc]["_passes"]] - if loc == "HBA": # and the dosage loop's two passes (section 8), the interval pass (section 9) - passes += ["", "-Q 1", "", ""] # and the CYP21A2 passes (section 10) - passes += ["-Q 1"] - # the commands only: options, then a path, a quoted argument or a variable - calls = [o for o, arg in re.findall(r"samtools bedcov((?: -[A-Za-z](?: \d+)?)*) (\S+)", text) - if re.match(r'["$/]|\S+\.bed\b', arg)] - check(f"{path}: its bedcov passes are the catalog's ({' and '.join(repr(p) for p in passes)})", - sorted(c.strip() for c in calls) == sorted(passes), calls) - check(f"{path}: no call of the old per-gene scripts", - not re.search(r"(hba_copy_number|lpa_kiv2)\.py (bed|check|estimate)", text)) - - dosage(cat, read) - intervals(cat, read) - psv(cat, read) - - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - def dosage_depths(r, copies, q1_share=0.5): """Mean depths of a dosage row at a copy number: controls at 30x (two @@ -522,14 +590,14 @@ def dosage_depths(r, copies, q1_share=0.5): return m, {n: v * q1_share if n in tg else v for n, v in m.items()} -# run_in and cpath for the dosage block of step 35: copy_number.py runs for -# real, samtools bedcov prints tests/fixtures/cn/[.q1].bedcov.tsv. +# run_in and cpath for the dosage block of step 35: og cn runs for real (OG), +# samtools bedcov prints tests/fixtures/cn/[.q1].bedcov.tsv. STUB = r""" run_in() { while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") shift 2; q=""; if [ "$1" = -Q ]; then q=.q1; shift 2; fi d=$(dirname "$1"); cat "${FIX}/${d##*/}${q}.bedcov.tsv" ;; *) return 1 ;; @@ -549,8 +617,8 @@ def dosage(cat, read): m and m.group(1)) for path, text, q in (("scripts/35-paralogs.sh", step, '"'), ("modules/local/parascopy/main.nf", module, '"\\')): for cmd in ("bed", "call"): - check(f"{path}: copy_number.py {cmd} --locus on each dosage row", - f"copy_number.py {cmd} --locus {q}$loc" in text) + check(f"{path}: og cn {cmd} --locus on each dosage row", + f"og cn {cmd} --locus {q}$loc" in text) for loc in DOSAGE: r = cat[loc] check(f"dosage {loc}: two bedcov passes, all reads then MAPQ >= 1 (what the loop runs)", @@ -580,6 +648,8 @@ def dosage(cat, read): # The block of step 35, cut out and run on the fixtures. blk = re.search(r'\n(DOSAGE_LOCI=.*?\nrm -rf "\$DOSAGE_WORK"\n)', step, re.S) check("step 35 holds the dosage block", blk is not None) + if not OG: + return fix = os.path.join(REPO, "tests", "fixtures", "cn") with tempfile.TemporaryDirectory() as t: env = dict(os.environ, REPO=REPO, FIX=fix, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", @@ -601,16 +671,16 @@ def dosage(cat, read): got[1:3] == ["called", str(n)] and got[5:] == [cn.reading(cat[loc], n)], got) # A row shifted by 1 kb no longer matches the fixture made for its place. - for loc in DOSAGE: - r = dict(cat[loc]) - r["targets"] = ";".join(f"{n}:{s + 1000}-{e + 1000}" for n, s, e, _ in r["_targets"]) - shifted = cn.parse_row({k: v for k, v in r.items() if not k.startswith("_")}) - try: - cn.read_bedcov(os.path.join(fix, f"{loc}.bedcov.tsv"), shifted) - got = "read" - except ValueError as e: - got = str(e) - check(f"dosage {loc}: shifted 1 kb, its fixture is refused", "regions differ" in got, got) + with tempfile.TemporaryDirectory() as t: + p = os.path.join(t, "c.tsv") + for loc in DOSAGE: + r = {k: v for k, v in cat[loc].items() if not k.startswith("_")} + r["targets"] = ";".join(f"{n}:{s + 1000}-{e + 1000}" for n, s, e, _ in cat[loc]["_targets"]) + write_catalog(p, [r]) + f = os.path.join(fix, f"{loc}.bedcov.tsv") + out = og_cn("call", "--catalog", p, "--locus", loc, "--bedcov", f, "--bedcov-mapq1", f) + check(f"dosage {loc}: shifted 1 kb, its fixture is refused", + out.returncode == 2 and "regions differ" in out.stderr, out) # Section 9: the interval rows. @@ -630,7 +700,7 @@ def dosage(cat, read): while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") shift 2; python3 "${FAKECOV}" "$1" ;; *) return 1 ;; esac @@ -709,12 +779,11 @@ def intervals(cat, read): not hits, hits) a_d, p17 = cat["22q11.2_DGS_proximal_A-D"], cat["17p12_HNPP_CMT1A"] - for r, target, want, change in ((a_d, 15.0, 1, "deletion"), (p17, 45.0, 3, "duplication"), + for r, target, want, change_ in ((a_d, 15.0, 1, "deletion"), (p17, 45.0, 3, "duplication"), (a_d, 30.0, 2, "no change"), (p17, 30.0, 2, "no change")): b = cn.assess(r, depths(r, target)) - line = cn.interval_line(r, b) - check(f"interval {r['locus']}: targets at {target}x over 30x controls is {want} copies, {change}", - (b["status"], b["copies"], line[8]) == ("called", want, change), (b, line)) + check(f"interval {r['locus']}: targets at {target}x over 30x controls is {want} copies, {change_}", + (b["status"], b["copies"], change(b)) == ("called", want, change_), b) az = cat["AZFc_DAZ"] m = depths(az, 15.0 / 4, 15.0) check("AZFc on a male: four DAZ copies at a single copy's depth in all is 4 copies", @@ -723,12 +792,12 @@ def intervals(cat, read): half = {n: (7.5 if n in {x for x, *_ in az["_targets"]} else 15.0) for n in m} b = cn.assess(az, half, None, "male") check("AZFc on a male at half its DAZ depth: 2 copies, a deletion (gr/gr or b2/b3)", - (b["copies"], cn.interval_line(az, b)[8]) == (2, "deletion"), b) + (b["copies"], change(b)) == (2, "deletion"), b) b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 15.0, 15.0), None, "male") check("AZFa on a male: one copy, no change", (b["copies"], b["expected"]) == (1, 1), b) b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 0.0, 15.0), None, "male") check("AZFa on a male with no read over it: 0 copies, a deletion", - (b["copies"], cn.interval_line(cat["AZFa"], b)[8]) == (0, "deletion"), b) + (b["copies"], change(b)) == (0, "deletion"), b) for loc in ("AZFa", "AZFb", "AZFc_DAZ"): b = cn.assess(cat[loc], depths(cat[loc], 15.0, 15.0), None, "female") check(f"{loc} on a female: not applicable", (b["status"], b["reason"]) == ("not applicable", "no_y"), b) @@ -740,12 +809,20 @@ def intervals(cat, read): b = cn.assess(xp, depths(xp, 0.0, 15.0), None, "male") check("Xp22.31 on a male with no read there: 0 copies, a deletion (the STS deletion)", b["copies"] == 0, b) + step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") + for path, text in (("scripts/35-paralogs.sh", step), ("modules/local/parascopy/main.nf", module)): + for cmd in ("og cn bed --kind interval", "og cn intervals --bedcov", "og cn intervals --failed"): + check(f"{path}: {cmd}", cmd in text) + check("PARASCOPY module: the samplesheet sex goes to the interval rows", + "def sex = meta.sex in ['male', 'female'] ? meta.sex : 'unknown'" in module and "--sex ${sex}" in module) + if not OG: + return with tempfile.TemporaryDirectory() as t: fake = os.path.join(t, "fakecov.py") with open(fake, "w") as f: f.write(FAKECOV) bed = os.path.join(t, "r.bed") - out = subprocess.run([sys.executable, SCRIPT, "bed", "--kind", "interval"], capture_output=True, text=True) + out = og_cn("bed", "--kind", "interval") open(bed, "w").write(out.stdout) check("bed --kind interval: every segment of every interval row, once", out.returncode == 0 and len(out.stdout.splitlines()) == sum(len(cn.regions(r)) for r in iv), out.stderr) @@ -755,12 +832,11 @@ def intervals(cat, read): with open(cov, "w") as f: subprocess.run([sys.executable, fake, bed], stdout=f, env=env, check=True) tab = os.path.join(t, "iv.tsv") - out = subprocess.run([sys.executable, SCRIPT, "intervals", "--bedcov", cov, "--sex", "female", "--out", tab], - capture_output=True, text=True) + out = og_cn("intervals", "--bedcov", cov, "--sex", "female", "--out", tab) lines = open(tab).read().splitlines() if os.path.exists(tab) else [] rows = {x.split("\t")[0]: x.split("\t") for x in lines[1:]} check("intervals: exit 0, the header, one line per interval row", - out.returncode == 0 and lines[:1] == ["\t".join(cn.INTERVAL_COLUMNS)] and list(rows) == names, + out.returncode == 0 and lines[:1] == ["\t".join(INTERVAL_COLUMNS)] and list(rows) == names, (out.returncode, out.stderr[-400:], lines[:1])) check("intervals: the 22q11.2 A-D deletion at 1 copy", rows.get(a_d["locus"], [""] * 11)[5:9] == ["called", "1", "2", "deletion"], rows.get(a_d["locus"])) @@ -769,12 +845,11 @@ def intervals(cat, read): and rows.get("3q29", [""] * 11)[5:9] == ["called", "2", "2", "no change"], rows.get(p17["locus"])) check("intervals: chrY rows on a female are not applicable", all(rows.get(x, [""] * 11)[5] == "not applicable" for x in ("AZFa", "AZFb", "AZFc_DAZ"))) - out = subprocess.run([sys.executable, SCRIPT, "intervals", "--failed", "--out", tab], capture_output=True, text=True) + out = og_cn("intervals", "--failed", "--out", tab) lines = open(tab).read().splitlines()[1:] check("intervals --failed: every row not assessed, check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines)) - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") blk = re.search(r'\n(INTERVAL_OUT=.*?\nrm -rf "\$INTERVAL_WORK"\n)', step, re.S) check("step 35 holds the interval block", blk is not None) env = dict(os.environ, REPO=REPO, FAKECOV=fake, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", @@ -799,10 +874,6 @@ def intervals(cat, read): check("step 35 interval block with a failed bedcov: exit 0, every row check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines), (out.returncode, out.stderr[-400:])) - for path, text in (("scripts/35-paralogs.sh", step), ("modules/local/parascopy/main.nf", module)): - for cmd in ("copy_number.py bed --kind interval", "copy_number.py intervals --bedcov", - "copy_number.py intervals --failed"): - check(f"{path}: {cmd}", cmd in text) # collect_summary: step 35's section carries the interval rows with a change import collect_summary as cs sd = os.path.join(t, "sample") @@ -821,8 +892,6 @@ def intervals(cat, read): ci = (data or {}).get("cn_intervals") or {} check("collect_summary: the interval rows stay when step 35 wrote no other table", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) - check("PARASCOPY module: the samplesheet sex goes to the interval rows", - "def sex = meta.sex in ['male', 'female'] ? meta.sex : 'unknown'" in module and "--sex ${sex}" in module) # Section 10: the CYP21A2 row, call psv. @@ -866,26 +935,42 @@ def cyp_fixture(r, total, a2): def cyp_call(r, bedcov_text, pileup_text, mapq0=0.1): - """The block of a CYP21A2 genome; mapq0 is the share of every segment's + """og cn call on a CYP21A2 genome; mapq0 is the share of every segment's depth from MAPQ 0 reads (0.1 as HG002's BWA-MEM and minimap2 reads give - over the two targets; 0 for an aligner that leaves those reads out).""" + over the two targets; 0 for an aligner that leaves those reads out). The + table read back: status, copies, module_total, finding and reason (None + for '.'), reading, conversion_sites, each site's copies estimate and + ratio, and the table and stdout line themselves.""" with tempfile.TemporaryDirectory() as t: - b, p = os.path.join(t, "b.tsv"), os.path.join(t, "p.tsv") + b, q, p, tab = (os.path.join(t, x) for x in ("b.tsv", "q.tsv", "p.tsv", "t.tsv")) open(b, "w").write(bedcov_text) + open(q, "w").write("".join( + "\t".join(c[:4] + [repr(float(c[4]) * (1 - mapq0))]) + "\n" + for c in (ln.split("\t") for ln in bedcov_text.splitlines()))) open(p, "w").write(pileup_text) - m = cn.read_bedcov(b, r) - q1 = {n: v * (1 - mapq0) for n, v in m.items()} - return cn.psv_assess(r, cn.assess(r, m, q1), cn.read_pileup(p, r)) - - -# run_in and cpath for the CYP21A2 block of step 35: copy_number.py runs for real, + out = og_cn("call", "--catalog", CATALOG, "--locus", "CYP21A2", "--bedcov", b, "--bedcov-mapq1", q, + "--pileup", p, "--table", tab) + rows = table(tab) + val = (lambda k, f=str: None if rows.get(k, ".") == "." else f(rows[k])) + sites = {} + for k, v in rows.items(): + m = re.search(r"copies \S+ \(([0-9]+\.[0-9]+)\); ratio ([0-9]+\.[0-9]+)", v) if k.startswith("psv_") else None + if m: + sites[k[4:]] = {"estimate": float(m.group(1)), "ratio": float(m.group(2))} + return {"status": rows.get("status"), "copies": val("cyp21a2_copies", int), "module_total": val("module_total", int), + "finding": val("finding"), "reason": val("reason"), "reading": rows.get("reading", ""), + "conversion_sites": (val("conversion_sites") or "").split(",") if val("conversion_sites") else [], + "sites": sites, "table": rows, "line": out.stdout.strip(), "exit": out.returncode} + + +# run_in and cpath for the CYP21A2 block of step 35: og cn runs for real (OG), # samtools bedcov and mpileup print tests/fixtures/cn/CYP21A2_${CYP_CASE}.*. STUB_CYP = r""" run_in() { while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") # the MAPQ >= 1 pass: 90% of each depth (HG002's share), or all of it for # an aligner that left the MAPQ 0 reads out (CYP_NO_MAPQ0) @@ -938,16 +1023,29 @@ def psv(cat, read): got = str(e) check(f"CYP21A2: {desc} is refused", err in got, got) - got = cn.count_bases("^]C.,c$+2AGg-1tG*<>N", "C") - check("pileup: read starts, ends and indels are not bases; '.' and ',' are the reference base", - got == {"A": 0, "C": 4, "G": 2, "T": 0}, got) + step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") + for path, text, q in (("scripts/35-paralogs.sh", step, True), ("modules/local/parascopy/main.nf", module, False)): + for cmd in ("og cn bed --locus CYP21A2", "og cn bed --psv --locus CYP21A2", + "--bedcov-mapq1", "cyp21a2_bedcov_q1.tsv" if not q else 'bedcov_q1.tsv")"', + "samtools mpileup -A -B -q 0 -Q 13 -d 100000", "og cn call --locus CYP21A2 --bedcov", + "--pileup", "_cyp21a2_psv.tsv"): + check(f"{path}: {cmd}", cmd in text) + check("PARASCOPY module: its CYP21A2 table is an output", 'path("${meta.id}_cyp21a2_psv.tsv"), emit: cyp21a2' + in module) + check("CYP21A2 row: a MAPQ 0 guard (two bedcov passes, all reads and MAPQ >= 1; minimum share 0.02)", + r["_passes"] == [".", "-Q 1"] and r["_mapq0_min"] == 0.02 and r["_mapq0_max"] is None, + (r["_passes"], r["_mapq0_min"], r["_mapq0_max"])) + if not OG: + return - out = subprocess.run([sys.executable, SCRIPT, "bed", "--psv", "--locus", "CYP21A2"], capture_output=True, text=True) + # og's pileup reader (read starts, ends and indels are not bases) is + # checked by the crate's own test and by the parity cases. + out = og_cn("bed", "--psv", "--locus", "CYP21A2") lines = out.stdout.splitlines() check("bed --psv: twelve one-base positions (each site on CYP21A2 and on CYP21A1P), sorted", out.returncode == 0 and len(lines) == 12 and all(int(x.split("\t")[2]) - int(x.split("\t")[1]) == 1 for x in lines) and [int(x.split("\t")[1]) for x in lines] == sorted(int(x.split("\t")[1]) for x in lines), out) - out = subprocess.run([sys.executable, SCRIPT, "bed", "--psv", "--locus", "HBA"], capture_output=True, text=True) + out = og_cn("bed", "--psv", "--locus", "HBA") check("bed --psv on a row without PSV sites is refused", out.returncode == 2 and "no psv locus" in out.stderr, out) # The committed fixtures are cyp_fixture's, so a change to either shows. @@ -1008,9 +1106,6 @@ def psv(cat, read): # The MAPQ 0 guard: an aligner that left out the reads fitting both copies lowers # the module total, so the deletion's tables from such a BAM (no MAPQ 0 read) read # not assessed, not a deletion; with HG002's 10% share they still give the deletion. - check("CYP21A2 row: a MAPQ 0 guard (two bedcov passes, all reads and MAPQ >= 1; minimum share 0.02)", - r["_passes"] == [".", "-Q 1"] and r["_mapq0_min"] == 0.02 and r["_mapq0_max"] is None, - (r["_passes"], r["_mapq0_min"], r["_mapq0_max"])) total, a2 = CYP_CASES["deletion"] bed, pile = cyp_fixture(r, total, a2) for share, want in ((0.0, ("not assessed", "low_mapq0_share", None, None)), @@ -1020,19 +1115,17 @@ def psv(cat, read): check(f"CYP21A2: the deletion's tables with a MAPQ 0 share of {share}: {want[0]} ({want[1] or want[2]})", (b["status"], b["reason"], b["finding"], b["copies"]) == want, b) b = cyp_call(r, bed, pile, 0.0) - rows, line = cn.psv_table(r, b) - tab = dict(rows) + tab, line = b["table"], b["line"] check("CYP21A2 without MAPQ 0 reads: the table says so (message, share 0.000), no copies, module total or finding", tab["message"].startswith("not assessed: the BAM has few or no multi-mapping reads here") and "CYP21A2 and CYP21A1P" in tab["message"] and tab["mapq0_share"] == "0.000" and (tab["cyp21a2_copies"], tab["module_total"], tab["finding"]) == (".", ".", ".") and "low_mapq0_share" in line, (tab, line)) bed, pile = cyp_fixture(r, 4, {}) - tab = dict(cn.psv_table(r, cyp_call(r, bed, pile))[0]) + tab = cyp_call(r, bed, pile)["table"] check("CYP21A2 unmodified with HG002's MAPQ 0 share: called, share 0.100, message '.'", (tab["status"], tab["finding"], tab["mapq0_share"], tab["message"]) == ("called", "none", "0.100", "."), tab) - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") blk = re.search(r'\n(CYP21_OUT=.*?\nrm -rf "\$CYP21_WORK"\n)', step, re.S) check("step 35 holds the CYP21A2 block", blk is not None) tabs = {} @@ -1071,14 +1164,6 @@ def psv(cat, read): (nl.get("mapq0_share"), nl.get("message")) == ("0.100", "."), nl) check("step 35 CYP21A2 block with a failed mpileup: the rows of a called table but the sites'", list(rd("fail")) == [k for k in nl if not k.startswith("psv_")], (list(rd("fail")), list(nl))) - for path, text, q in (("scripts/35-paralogs.sh", step, True), ("modules/local/parascopy/main.nf", module, False)): - for cmd in ("copy_number.py bed --locus CYP21A2", "copy_number.py bed --psv --locus CYP21A2", - "--bedcov-mapq1", "cyp21a2_bedcov_q1.tsv" if not q else 'bedcov_q1.tsv")"', - "samtools mpileup -A -B -q 0 -Q 13 -d 100000", "copy_number.py call --locus CYP21A2 --bedcov", - "--pileup", "_cyp21a2_psv.tsv"): - check(f"{path}: {cmd}", cmd in text) - check("PARASCOPY module: its CYP21A2 table is an output", 'path("${meta.id}_cyp21a2_psv.tsv"), emit: cyp21a2' - in module) # the couple screen genes = {"CYP21A2": {"gene": "CYP21A2"}} diff --git a/tests/test_hba_copy_number.py b/tests/test_hba_copy_number.py index 973821e72..a31afb5cd 100755 --- a/tests/test_hba_copy_number.py +++ b/tests/test_hba_copy_number.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -"""bin/hba_copy_number.py: step 35's HBA1/HBA2 count from short-read depth. +"""og cn call --locus HBA: step 35's HBA1/HBA2 count from short-read depth. +It needs og: OG names the binary (OG=rust/target/release/og, or bin/og); +without it the test is skipped, and with OG_REQUIRED set too it fails. On `samtools bedcov` tables made up for each case (summed depth = mean depth x length), at 30x in the flanks, with a quarter of the gene-body depth from @@ -17,11 +19,12 @@ 15.5x, HBA1 25.7x, flanks 30x and 31x, which main counted as three or left between two and three) is 'not assessed', naming the cause; 1% is too, 3% is called; no gene at all (no reads to judge by) is still 0; - 7. the CLI: `bed` prints the four regions, `check` writes the metric table - and exits 0; a table with other regions, or a short row, in either - input exits 2, and so does a missing --bedcov-mapq1. + 7. the CLI: `og cn bed --locus HBA` prints the four regions, `og cn call + --table` writes the metric table and exits 0; a table with other + regions, or a short row, in either input exits 2, and so does a missing + --bedcov-mapq1. -Run: python3 tests/test_hba_copy_number.py +Run: OG=rust/target/release/og python3 tests/test_hba_copy_number.py """ import os import subprocess @@ -29,10 +32,15 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "hba_copy_number.py") +CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import hba_copy_number as hba # noqa: E402 +import copy_number as cn # noqa: E402 +REGIONS = cn.regions(cn.load_catalog(CATALOG)["HBA"]) +# Step 35's HBA table, in its order (what bin/hba_copy_number.py wrote). +ORDER = ["status", "alpha_globin_genes", "estimate", "hba2_depth", "hba1_depth", "flank_left_depth", + "flank_right_depth", "mapq0_share", "message"] FAILS = 0 @@ -46,7 +54,7 @@ def check(desc, ok, detail=""): def bedcov(path, depth): """A bedcov table with the given mean depth per region name.""" with open(path, "w") as f: - for c, s, e, n in hba.REGIONS: + for c, s, e, n in REGIONS: f.write(f"{c}\t{s}\t{e}\t{n}\t{int(depth[n] * (e - s))}\n") @@ -56,12 +64,18 @@ def mapq1(depth, share=0.25): return {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in depth.items()} +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + def run(depth, share=0.25): + """Step 35's HBA table for these depths, as og cn call writes it.""" with tempfile.TemporaryDirectory() as t: - p, q = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv") + p, q, o = (os.path.join(t, x) for x in ("cov.tsv", "cov_q1.tsv", "hba.tsv")) bedcov(p, depth) bedcov(q, mapq1(depth, share)) - return hba.assess(hba.read_bedcov(p), hba.read_bedcov(q)) + og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", p, "--bedcov-mapq1", q, "--table", o) + return dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) if os.path.exists(o) else {} def depths(hba2, hba1, left=30.0, right=30.0): @@ -69,6 +83,9 @@ def depths(hba2, hba1, left=30.0, right=30.0): def main(): + if not OG: + print("skip: OG is not set (OG=rust/target/release/og)") + return 1 if os.environ.get("OG_REQUIRED") else 0 r = run(depths(30, 30)) check("no deletion: four alpha-globin genes, called", (r["status"], r["alpha_globin_genes"], r["estimate"]) == ("called", "4", "4.00") @@ -113,33 +130,45 @@ def main(): (r["status"], r["alpha_globin_genes"]) == ("called", "0") and "Bart" in r["message"], r) with tempfile.TemporaryDirectory() as t: - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed", "--catalog", CATALOG, "--locus", "HBA") rows = [line.split("\t") for line in out.stdout.splitlines()] check("bed: four regions, the two genes between the flanks", out.returncode == 0 and [x[3] for x in rows] == ["flank_left", "HBA2", "HBA1", "flank_right"], out) cov, covq, res = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv"), os.path.join(t, "hba.tsv") bedcov(cov, depths(15, 30)) bedcov(covq, mapq1(depths(15, 30))) - cli = [sys.executable, SCRIPT, "check", "--bedcov", cov, "--bedcov-mapq1", covq, "--out", res] + cli = [OG, "cn", "call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", cov, "--bedcov-mapq1", covq, + "--table", res] out = subprocess.run(cli, capture_output=True, text=True) table = dict(line.split("\t", 1) for line in open(res).read().splitlines()[1:]) if os.path.exists(res) else {} - check("check: exit 0, the table has the status, the count and the message", + check("call: exit 0, the table has the status, the count and the message", out.returncode == 0 and table.get("status") == "called" and table.get("alpha_globin_genes") == "3" and table.get("mapq0_share") == "0.250" - and list(table) == list(hba.ORDER) and "HBA1/HBA2: three" in out.stdout, (out, table)) - out = subprocess.run(cli[:4] + cli[6:], capture_output=True, text=True) - check("check: without --bedcov-mapq1 exits 2", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) + and list(table) == ORDER and "HBA1/HBA2: three" in out.stdout, (out, table)) + out = subprocess.run(cli[:9] + cli[11:], capture_output=True, text=True) + check("call: without --bedcov-mapq1 exits 2", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) for name, path in (("all reads", cov), ("MAPQ 1 or more", covq)): bedcov(cov, depths(15, 30)) bedcov(covq, mapq1(depths(15, 30))) with open(path, "a") as f: f.write("chr16\t1\t100\tother\t500\n") out = subprocess.run(cli, capture_output=True, text=True) - check(f"check: a table ({name}) with other regions exits 2", out.returncode == 2 and "differ" in out.stderr, out) + check(f"call: a table ({name}) with other regions exits 2", out.returncode == 2 and "differ" in out.stderr, out) with open(path, "w") as f: f.write("chr16\t100000\t140000\n") out = subprocess.run(cli, capture_output=True, text=True) - check(f"check: a short row ({name}) exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + check(f"call: a short row ({name}) exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + + # The demo's HBA table (tests/demo/make_demo_sample.py) is what og writes. + sys.path.insert(0, os.path.join(REPO, "tests", "demo")) + import make_demo_sample as demo + with tempfile.TemporaryDirectory() as t: + cov, covq, res = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv"), os.path.join(t, "hba.tsv") + bedcov(cov, demo.HBA_DEPTH) + bedcov(covq, {n: v * demo.HBA_Q1[n] for n, v in demo.HBA_DEPTH.items()}) + og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", cov, "--bedcov-mapq1", covq, "--table", res) + got = open(res).read() if os.path.exists(res) else "" + check("the demo's HBA table is the one og cn call writes for its depths", got == demo.HBA_TABLE, got) print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") return 1 if FAILS else 0 diff --git a/tests/test_lpa_fallback.sh b/tests/test_lpa_fallback.sh index acb0f7e2f..c16dc7e53 100755 --- a/tests/test_lpa_fallback.sh +++ b/tests/test_lpa_fallback.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# Step 46 [4/4]: a KIV-2 4925G>A reader that crashes never costs the KIV-2 -# count. scripts/46-lpa.sh then writes lpa/_lpa_kiv2_4925.tsv saying +# Step 46 [3/3]: a KIV-2 4925G>A reader that crashes never costs the KIV-2 +# count (one og cn call reads both; it writes the count table first). scripts/46-lpa.sh then writes lpa/_lpa_kiv2_4925.tsv saying # 'not assessed', exits 0 and keeps lpa/_lpa_kiv2.tsv; the LPA Nextflow # process (modules/local/lpa/main.nf) writes the same fallback table, byte for # byte. The e2e case genotype-rules-2-lpa-null runs the module's block with a # crashing reader in its image. # -# A fake `docker` on PATH stands in for the containers: the count call writes -# a table, the 4925G>A call exits 1 (or 0 for the control, writing a -# 'present' table), everything else succeeds. +# A fake `docker` on PATH stands in for the containers: the og cn call writes +# the count table, then exits 1 at --marker-out (or 0 for the control, +# writing a 'present' table), everything else succeeds. # # Controls: with the reader working the table is the reader's ('present'), so # the not-assessed check can fail; with the fallback removed from a copy of @@ -97,7 +97,7 @@ mkdir -p "${WORK}/scripts/lib" cp "${REPO}/scripts/lib/"* "${WORK}/scripts/lib/" cp "${REPO}/versions.env" "${WORK}/" 2>/dev/null || true # shellcheck disable=SC2016 # the $(cpath ...) is text of the step, not to expand here -sed -e 's/^if ! run_in -v /run_in -v /' -e 's/--marker-out "$(cpath "$NULL")"; then$/--marker-out "$(cpath "$NULL")"\nif false; then/' \ +sed -e 's/--marker-out "$(cpath "$NULL")" || rc=\$?$/--marker-out "$(cpath "$NULL")"/' \ "${REPO}/scripts/46-lpa.sh" > "${WORK}/scripts/46-lpa.sh" bash -n "${WORK}/scripts/46-lpa.sh" || fail "control: the copy of step 46 without the fallback is not valid bash" if cmp -s "${REPO}/scripts/46-lpa.sh" "${WORK}/scripts/46-lpa.sh"; then diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index a60ed5fba..c1aeccf10 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 -"""bin/lpa_kiv2.py: step 46's LPA KIV-2 repeat count from short-read depth, -and its Lp(a) section in both reports. +"""og cn call --locus LPA_KIV2: step 46's LPA KIV-2 repeat count from +short-read depth, with KIV-2 4925G>A beside it, and its Lp(a) section in both +reports. Part 6 runs without og; the rest needs it: OG names the binary +(OG=rust/target/release/og, or bin/og); without it they are skipped, and with +OG_REQUIRED set too the test fails. On `samtools bedcov` tables made up for each case (summed depth = mean depth x exon length), 30x over the single-copy LPA exons, with 90% of the KIV-2 @@ -19,9 +22,10 @@ count; no read of MAPQ 0 over KIV-2 (novoalign's default) is 'not assessed', naming the cause; two empty tables (a BAM with no chr6) is 'not assessed'; - 5. the CLI: `bed` prints the 29 regions, 14 KIV-2 exons of 2,394 bp and 15 - single-copy exons; `estimate` writes the metric table and exits 0; a - table with other regions or a short row exits 2; + 5. the CLI: `og cn bed --locus LPA_KIV2` prints the 29 regions, 14 KIV-2 + exons of 2,394 bp and 15 single-copy exons; `og cn call --table` writes + the metric table and exits 0; a table with other regions or a short row + exits 2; 6. assets/traits.tsv has the two LPA risk variants with the alleles bin/collect_summary.py counts, and the blood-test advice; 7. both reports: a low count with one risk allele is flagged with both @@ -42,9 +46,11 @@ column are not counted; when step 46's table shows the aligner left out the reads of MAPQ 0, 'not assessed'; the reading goes to its own table and step 46's table is left byte for byte; without a count the copies are '.'; an - output that cannot be written exits 2. + output that cannot be written exits 2. `og cn marker-read` on step 46's + table and `og cn call --marker` (what step 46 and the LPA process run) + write the same 4925G>A table. -Run: python3 tests/test_lpa_kiv2.py +Run: OG=rust/target/release/og python3 tests/test_lpa_kiv2.py """ import math import os @@ -53,15 +59,32 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "lpa_kiv2.py") +CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") +OG = os.environ.get("OG", "") +if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): + # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job + # builds og in a temporary folder, so the tests run a copy that sits next + # to the catalog, as bin/.og/og-* does in a checkout. + import atexit, shutil # noqa: E401 + _og_dir = tempfile.mkdtemp(prefix="og-cn-") + atexit.register(shutil.rmtree, _og_dir, True) + shutil.copy2(OG, os.path.join(_og_dir, "og")) + shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) + OG = os.environ["OG"] = os.path.join(_og_dir, "og") sys.path.insert(0, os.path.join(REPO, "bin")) -import lpa_kiv2 as lpa # noqa: E402 +import copy_number as cn # noqa: E402 FAILS = 0 # The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. R1, R2 = "rs10455872", "rs3798220" -KIV = [n for *_, n in lpa.REGIONS if n.startswith("kiv2_")] -FLANK = [n for *_, n in lpa.REGIONS if n.startswith("flank_")] +ROW = cn.load_catalog(CATALOG)["LPA_KIV2"] +REGIONS = cn.regions(ROW) +# in catalog order: the KIV-2 block, then the single-copy exons +KIV = [n for n, *_ in ROW["_targets"]] +FLANK = [n for n, *_ in ROW["_controls"]] +# The advice row of step 46's table. +ADVICE = ("A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 European " + "Atherosclerosis Society consensus advises measuring it at least once in every adult.") def check(desc, ok, detail=""): @@ -87,10 +110,41 @@ def q1(m, share=0.9): return {n: v * (1 - share) if n.startswith("kiv2_") else v for n, v in m.items()} -def bedcov(path, m): +def bedcov(path, m, exact=False): + """samtools bedcov at these mean depths: whole sums, or with exact the + sums themselves, so each mean comes back as given.""" with open(path, "w") as f: - for c, s, e, n in lpa.REGIONS: - f.write(f"{c}\t{s}\t{e}\t{n}\t{round(m[n] * (e - s))}\n") + for c, s, e, n in REGIONS: + v = m[n] * (e - s) + f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") + + +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + +def read_table(path): + return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) if os.path.exists(path) else {} + + +def estimate(a, b, out): + """og cn call on step 46's two bedcov tables, writing its table.""" + return og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, + "--table", out) + + +def assess(m, q): + """Step 46's table for mean depths m (every read) and q (MAPQ >= 1).""" + with tempfile.TemporaryDirectory() as t: + a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) + if m: + bedcov(a, m, exact=True) + bedcov(b, q, exact=True) + else: + open(a, "w").close() + open(b, "w").close() + estimate(a, b, o) + return read_table(o) def cv(vals): @@ -98,22 +152,52 @@ def cv(vals): return math.sqrt(sum((v - mu) ** 2 for v in vals) / (len(vals) - 1)) / mu +def pileup(path, per_site, extra=""): + """mpileup lines at the seven copies of 4925G>A (traits_report.MARKERS); + per_site is (C reads, T reads) at each.""" + import traits_report + with open(path, "w") as f: + for p in traits_report.MARKERS["LPA_KIV2_4925"]["copies"]: + c, t_ = per_site + bases = "^]" + "C" * c + "t" * t_ + "$" + f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") + f.write(extra) + + +def traits(): + """6. assets/traits.tsv holds the two LPA risk variants the report counts.""" + import collect_summary + import traits_report + table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} + for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): + r = table.get(rs, {}) + check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", + r.get("gene") == "LPA" and r.get("alt") == alt, r) + check(f"traits.tsv: {rs} says the actionable test is a blood Lp(a) measurement", + "measure" in (r.get("note") or "").lower() and "blood" in (r.get("note") or ""), r) + + def main(): + if not OG: + traits() + print("skip: OG is not set (OG=rust/target/release/og): only part 6 ran") + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") + return 1 if FAILS or os.environ.get("OG_REQUIRED") else 0 # 1. arithmetic for n in (12, 20, 40): - r = lpa.assess(depths(n), q1(depths(n))) + r = assess(depths(n), q1(depths(n))) check(f"arithmetic: a block at 30 x {n + 2} / 14 is {n} KIV-2 copies", r["status"] == "estimated" and r["kiv2_copies"] == f"{n:.1f}", r) # 2. interval - r = lpa.assess(depths(20), q1(depths(20))) + r = assess(depths(20), q1(depths(20))) check("interval: equal depths give a zero-width interval", r["ci95_low"] == r["ci95_high"] == "20.0", r) m = depths(20, kiv_spread=0.10, flank_spread=0.15) - r = lpa.assess(m, q1(m)) + r = assess(m, q1(m)) rel = math.sqrt(cv([m[n] for n in KIV]) ** 2 / 14 + cv([m[n] for n in FLANK]) ** 2 / 15) - klen = sum(e - s for _, s, e, n in lpa.REGIONS if n in KIV) - flen = sum(e - s for _, s, e, n in lpa.REGIONS if n in FLANK) - k = sum(m[n] * (e - s) for _, s, e, n in lpa.REGIONS if n in KIV) / klen - f = sum(m[n] * (e - s) for _, s, e, n in lpa.REGIONS if n in FLANK) / flen + klen = sum(e - s for _, s, e, n in REGIONS if n in KIV) + flen = sum(e - s for _, s, e, n in REGIONS if n in FLANK) + k = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in KIV) / klen + f = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in FLANK) / flen total = 14 * k / f want = (f"{total - 2:.1f}", f"{total * math.exp(-1.96 * rel) - 2:.1f}", f"{total * math.exp(1.96 * rel) - 2:.1f}") got = (r["kiv2_copies"], r["ci95_low"], r["ci95_high"]) @@ -121,38 +205,38 @@ def main(): check("interval: it contains the estimate", float(r["ci95_low"]) < float(r["kiv2_copies"]) < float(r["ci95_high"]), r) m2 = depths(20, kiv_spread=0.20, flank_spread=0.30) - r2 = lpa.assess(m2, q1(m2)) + r2 = assess(m2, q1(m2)) check("interval: twice the spread is wider", float(r2["ci95_high"]) - float(r2["ci95_low"]) > float(r["ci95_high"]) - float(r["ci95_low"]), (r, r2)) # 3. class for n, sp, cls in ((20, 0.03, "low"), (29, 0.03, "low"), (29, 0.25, "typical"), (38, 0.03, "typical"), (47, 0.25, "typical"), (60, 0.03, "high")): m = depths(n, kiv_spread=sp, flank_spread=sp) - r = lpa.assess(m, q1(m)) + r = assess(m, q1(m)) check(f"class: {n} copies ({r['ci95_low']} to {r['ci95_high']}) is {cls}", r["class"] == cls, r) - r = lpa.assess(depths(20), q1(depths(20))) + r = assess(depths(20), q1(depths(20))) check("class: the low message says what a low count means", "a low count goes with small apo(a) isoforms and a higher Lp(a) level" in r["message"], r) # 4. not assessed - r = lpa.assess(depths(20, flank=8.0), q1(depths(20, flank=8.0))) + r = assess(depths(20, flank=8.0), q1(depths(20, flank=8.0))) check("8x over the single-copy exons: too little depth, no count", r["status"] == "too little depth" and r["kiv2_copies"] == "." and "8.0x" in r["message"], r) - r = lpa.assess(depths(20, flank=10.0), q1(depths(20, flank=10.0))) + r = assess(depths(20, flank=10.0), q1(depths(20, flank=10.0))) check("10x: estimated", r["status"] == "estimated", r) - r = lpa.assess(depths(20), q1(depths(20), share=0.0)) + r = assess(depths(20), q1(depths(20), share=0.0)) check("no read of MAPQ 0 over KIV-2: not assessed, naming the cause", r["status"] == "not assessed" and r["kiv2_copies"] == "." and "novoalign" in r["message"], r) - r = lpa.assess(depths(20), q1(depths(20), share=0.25)) + r = assess(depths(20), q1(depths(20), share=0.25)) check("a quarter of the depth from MAPQ 0: estimated", r["status"] == "estimated", r) - r = lpa.assess({}, {}) + r = assess({}, {}) check("no chr6: not assessed", r["status"] == "not assessed" and "no chr6" in r["message"], r) - r = lpa.assess(depths(-1.5), q1(depths(-1.5))) + r = assess(depths(-1.5), q1(depths(-1.5))) check("KIV-2 depth under the floor of any count: not assessed, never a negative count", r["status"] == "not assessed" and r["kiv2_copies"] == "." and "too little for any repeat count" in r["message"], r) # 5. CLI - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed", "--catalog", CATALOG, "--locus", "LPA_KIV2") rows = [x.split("\t") for x in out.stdout.splitlines()] check("bed: 29 regions", out.returncode == 0 and len(rows) == 29, out) check("bed: 14 KIV-2 exons of 2,394 bp in total", @@ -163,42 +247,37 @@ def main(): a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) bedcov(a, depths(20)) bedcov(b, q1(depths(20))) - cli = [sys.executable, SCRIPT, "estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", o] + cli = [OG, "cn", "call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, + "--table", o] out = subprocess.run(cli, capture_output=True, text=True) tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) if os.path.exists(o) else {} - check("estimate: exit 0 and the metric table", out.returncode == 0 and tab.get("kiv2_copies") == "20.0" + check("call: exit 0 and the metric table", out.returncode == 0 and tab.get("kiv2_copies") == "20.0" and tab.get("status") == "estimated" and "blood" in tab.get("advice", ""), (out, tab)) open(a, "w").close() open(b, "w").close() out = subprocess.run(cli, capture_output=True, text=True) tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) - check("estimate: two empty tables (no chr6) exit 0, not assessed", + check("call: two empty tables (no chr6) exit 0, not assessed", out.returncode == 0 and tab.get("status") == "not assessed", (out, tab)) bedcov(a, depths(20)) out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: an empty MAPQ 1 table beside a full one exits 2", + check("call: an empty MAPQ 1 table beside a full one exits 2", out.returncode == 2 and "is empty but the other table is not" in out.stderr, out) bedcov(a, depths(20)) with open(a, "a") as f: f.write("chr6\t1\t100\tother\t500\n") out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: other regions exit 2", out.returncode == 2 and "differ" in out.stderr, out) + check("call: other regions exit 2", out.returncode == 2 and "differ" in out.stderr, out) with open(a, "w") as f: f.write("chr6\t160646213\t160646395\n") out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: a short row exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + check("call: a short row exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) # 6. the traits table + traits() import collect_summary import render_report import traits_report - table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} - for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): - r = table.get(rs, {}) - check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", - r.get("gene") == "LPA" and r.get("alt") == alt, r) - check(f"traits.tsv: {rs} says the actionable test is a blood Lp(a) measurement", - "measure" in (r.get("note") or "").lower() and "blood" in (r.get("note") or ""), r) # 7. reports def sample(t, name, n_copies, flank=30.0, genotypes=None): @@ -208,7 +287,7 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): m = depths(n_copies, flank=flank, kiv_spread=0.03, flank_spread=0.03) bedcov(a, m) bedcov(b, q1(m)) - lpa.main(["estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", os.path.join(d, "lpa", "S_lpa_kiv2.tsv")]) + estimate(a, b, os.path.join(d, "lpa", "S_lpa_kiv2.tsv")) if genotypes: os.makedirs(os.path.join(d, "traits")) with open(os.path.join(d, "traits", "S_traits.tsv"), "w") as f: @@ -231,10 +310,10 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): check("report: the text section has the count, both variants and the advice", "## Lp(a): LPA KIV-2 Repeat Count and Risk Variants" in txt and "KIV-2 repeats (both chromosomes): 20.0" in txt and f"{R1}: A/G (1 risk allele)" in txt and f"{R2}: T/T (0 risk alleles)" in txt - and lpa.ADVICE in txt, txt) + and ADVICE in txt, txt) check("report: the HTML card has the flag and the advice", "

Lp(a): LPA KIV-2 Repeat Count

" in html and "Genetic sign of high Lp(a)" in html - and render_report.E(lpa.ADVICE) in html, html[-3000:]) + and render_report.E(ADVICE) in html, html[-3000:]) summ, txt, html = sample(t, "typ", 38, genotypes={R1: "A/A", R2: "T|T"}) check("report: a typical count with no risk allele is not flagged", summ["sections"]["lpa"]["data"]["flag"] == "." and "No genetic sign of a high Lp(a)" in txt @@ -261,8 +340,9 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): and "KIV-2 4925G>A null allele (rs1484919544): not assessed\n" in txt.split(count, 1)[-1].split(R1, 1)[0] and "step 46 wrote no S_lpa_kiv2_4925.tsv" in txt and cav in txt, txt) d = os.path.join(t, "nul0", "S", "lpa") - traits_report.write_metric_table(os.path.join(d, "S_lpa_kiv2_4925.tsv"), - lpa.null_allele((665, 35, 0), "38.0")) + pileup(os.path.join(t, "nul0.pileup"), (95, 5)) + og_cn("marker-read", "LPA_KIV2_4925", "--pileup", os.path.join(t, "nul0.pileup"), "--copy-table", + os.path.join(d, "S_lpa_kiv2.tsv"), "--out", os.path.join(d, "S_lpa_kiv2_4925.tsv")) summ = collect_summary.collect("S", os.path.join(t, "nul0", "S")) txt, html = render_report.text_report(summ), render_report.html_report(summ) beside = txt.split(count, 1)[-1].split(R1, 1)[0] @@ -291,16 +371,6 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): "

Lp(a)" not in render_report.html_report(nothing) and "## Lp(a)" not in render_report.text_report(nothing)) # 8. KIV-2 4925G>A beside the count - sites = traits_report.MARKERS["LPA_KIV2_4925"]["copies"] - - def pileup(path, per_site, extra=""): - """mpileup lines at the seven copies; per_site is (C reads, T reads) at each.""" - with open(path, "w") as f: - for p in sites: - c, t_ = per_site - bases = "^]" + "C" * c + "t" * t_ + "$" - f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") - f.write(extra) def table_after(t, name, per_site, n_copies=38, extra=""): """Step 46's table, a pileup, then null-allele; returns (rc, the 4925G>A table, LPA table, pileup, out).""" @@ -309,10 +379,9 @@ def table_after(t, name, per_site, n_copies=38, extra=""): a, b, o, p, n = (os.path.join(d, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv", "p.tsv", "S_lpa_kiv2_4925.tsv")) bedcov(a, depths(n_copies)) bedcov(b, q1(depths(n_copies))) - lpa.main(["estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", o]) + estimate(a, b, o) pileup(p, per_site, extra) - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, "--out", n], - capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) if os.path.exists(n) else {} return rc, tab, o, p, n @@ -333,15 +402,17 @@ def table_after(t, name, per_site, n_copies=38, extra=""): a0, b0, o0 = (os.path.join(d0, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv")) bedcov(a0, depths(38)) bedcov(b0, q1(depths(38))) - lpa.main(["estimate", "--bedcov", a0, "--bedcov-mapq1", b0, "--out", o0]) + estimate(a0, b0, o0) check("4925G>A: step 46's table is left byte for byte", open(o).read() == open(o0).read(), open(o).read()) - n3 = os.path.join(t, "viatr.tsv") - res = subprocess.run([sys.executable, os.path.join(REPO, "bin", "traits_report.py"), "--marker", "LPA_KIV2_4925", - "--pileup", p, "--copy-table", o, "--marker-out", n3], capture_output=True, text=True) - check("4925G>A: traits_report.py --marker LPA_KIV2_4925 (what step 46 and the LPA process run) writes " - "the same table as null-allele", res.returncode == 0 and open(n3).read() == open(n).read(), res) - res = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", - os.path.join(t, "none.tsv"), "--out", n3], capture_output=True, text=True) + n3, o3 = os.path.join(t, "viacall.tsv"), os.path.join(t, "viacall_lpa.tsv") + res = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a0, "--bedcov-mapq1", b0, + "--table", o3, "--marker", "LPA_KIV2_4925", "--marker-pileup", p, "--marker-out", n3) + check("4925G>A: og cn call --marker LPA_KIV2_4925 (what step 46 and the LPA process run) writes the same " + "count table and the same 4925G>A table as marker-read on that table", + res.returncode == 0 and open(o3).read() == open(o0).read() and open(n3).read() == open(n).read() + and res.stdout.splitlines()[-1].startswith("LPA LPA KIV-2 4925G>A"), res) + res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", os.path.join(t, "none.tsv"), + "--out", n3) tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) check("4925G>A: without step 46's table the reading stands and the copies are '.'", res.returncode == 0 and tab.get("status") == "present" and tab.get("copies") == "." @@ -350,8 +421,7 @@ def table_after(t, name, per_site, n_copies=38, extra=""): lt = os.path.join(t, f"share{share}.tsv") with open(lt, "w") as f: f.write(f"metric\tvalue\nstatus\tnot assessed\nkiv2_copies\t.\nmapq0_share\t{share}\n") - res = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", lt, "--out", n3], - capture_output=True, text=True) + res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", lt, "--out", n3) tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) check(f"4925G>A: a MAPQ 0 share of {share} in step 46's table gives {want} on the same reads", res.returncode == 0 and tab.get("status") == want @@ -374,15 +444,26 @@ def table_after(t, name, per_site, n_copies=38, extra=""): rc, tab, o, p, n = table_after(t, "bad", (95, 5)) with open(p, "w") as f: f.write("not a pileup\n") - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, "--out", n], - capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) check("4925G>A: an unreadable pileup is not assessed, exit 0", rc.returncode == 0 and tab.get("status") == "not assessed", (rc, tab)) - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, - "--out", os.path.join(t, "no", "such", "dir.tsv")], capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, + "--out", os.path.join(t, "no", "such", "dir.tsv")) check("4925G>A: an output that cannot be written exits 2", rc.returncode == 2, rc) + # The demo's step 46 table (tests/demo/make_demo_sample.py) is what og writes. + sys.path.insert(0, os.path.join(REPO, "tests", "demo")) + import make_demo_sample as demo + m, q = demo.lpa_depth(ROW) + with tempfile.TemporaryDirectory() as t: + a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) + bedcov(a, m, exact=True) + bedcov(b, q, exact=True) + estimate(a, b, o) + got = open(o).read() if os.path.exists(o) else "" + check("the demo's step 46 table is the one og cn call writes for its depths", got == demo.LPA_TABLE, got) + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") return 1 if FAILS else 0 diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index 375926cb2..cd9b75470 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -50,8 +50,10 @@ 11. the copy-aware markers (MARKERS: LPA KIV-2 4925G>A, SMN1 g.27134T>G): each pinned dbSNP record is one of the marker's copies with its ref and alt, and a planted swap, a minus-strand pair and a position off the - copies are each caught; the context's middle base is the ref; - --marker-regions prints one region per copy; on mpileup lines made up + copies are each caught; the context's middle base is the ref. With OG + naming the og binary (skipped without it, a failure with OG_REQUIRED + set too): og cn marker-regions prints one region per copy of MARKERS; + og cn marker-read SMN1_27134 on mpileup lines made up for each case at the SMN1 and SMN2 copies: a quarter of the reads with G is 'present' (copies = fraction x agCN), T only (the reference) 'not seen', 2 reads with G 'uncertain', under 10 reads 'too little depth', an @@ -78,6 +80,9 @@ RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") SCRIPT = os.path.join(REPO, "bin", "traits_report.py") CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") +OG = os.environ.get("OG", "") +# The sentence every SMN1 marker message ends with. +SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." FAILS = 0 # The sha256 of the APOE line (its OUT_COLUMNS joined by tabs) the hand code of @@ -147,6 +152,10 @@ def run(*args): return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + def dbsnp_checks(rules): """10. scripts/ci/check-rule-alleles.sh on the rules and on planted copies. Runs first, so a rules table that breaks the other checks still shows what dbSNP says.""" @@ -294,10 +303,14 @@ def marker_checks(): e = tr.marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) check(f"marker {name} control: a context whose middle base is not the ref is caught", any("middle base" in x for x in e), e) - res = run("--marker-regions", name) - check(f"marker {name}: --marker-regions prints one region per copy", - res.returncode == 0 and res.stdout.split() == [f"{chrom}:{p}-{p}" for p in m["copies"]], res) check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) + if not OG: + print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") + return + for name, m in tr.MARKERS.items(): + res = og_cn("marker-regions", name) + check(f"marker {name}: og cn marker-regions prints one region per copy of MARKERS", + res.returncode == 0 and res.stdout.split() == [f"{m['chrom']}:{p}-{p}" for p in m["copies"]], res) sm = tr.MARKERS["SMN1_27134"] work = tempfile.mkdtemp() @@ -315,7 +328,9 @@ def smn(pscn="2,2", pqual="30,30", pfilt="PASS", agcn="4", homolog="chr5:7004900 def table(p, smn_path): out = os.path.join(work, "o.tsv") - res = run("--marker", "SMN1_27134", "--pileup", p, "--copy-table", smn_path, "--marker-out", out) + if os.path.exists(out): + os.remove(out) + res = og_cn("marker-read", "SMN1_27134", "--pileup", p, "--copy-table", smn_path, "--out", out) return res, dict(x.split("\t", 1) for x in open(out).read().splitlines()[1:]) if os.path.exists(out) else {} res, t = table(pileup(15, 5), smn()) @@ -362,13 +377,20 @@ def table(p, smn_path): # Every status with every psCN: the message says the marker does not prove 2+0. msgs = [] - for counts in [(30, 10, 0), (40, 0, 0), (38, 2, 0), (3, 1, 0), None]: - for path in (smn(), smn(pscn="1,3"), smn(pqual="5,5"), os.path.join(work, "missing.tsv")): - msgs.append(dict(tr.smn1_marker(counts, path))["message"]) + for counts in [(30, 10), (40, 0), (38, 2), (3, 1), None]: + if counts is None: + p = bad + else: # every read at the SMN1 copy + p = os.path.join(work, "all.tsv") + write(p, f"chr5\t{sm['copies'][0]}\tN\t{sum(counts)}\t{'T' * counts[0]}{'G' * counts[1]}\t.\n") + for path in ("2,2", "1,3", "5,5", None): + smn_path = (os.path.join(work, "missing.tsv") if path is None + else smn(pqual=path) if path == "5,5" else smn(pscn=path)) + msgs.append(table(p, smn_path)[1].get("message", "")) check(f"SMN1 marker: every message ({len(msgs)}) says it does not prove 2+0", - all(says_not_proof(x) for x in msgs), [x for x in msgs if not says_not_proof(x)]) + len(msgs) == 20 and all(says_not_proof(x) for x in msgs), [x for x in msgs if not says_not_proof(x)]) check("SMN1 marker control: a message without the sentence fails that test", - not says_not_proof(msgs[0].replace(tr.SMN1_NOT_PROOF, ""))) + not says_not_proof(msgs[0].replace(SMN1_NOT_PROOF, ""))) finally: shutil.rmtree(work) @@ -601,7 +623,7 @@ def rule(rid, *lines, phased=None): marker_checks() print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 + return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 if __name__ == "__main__":