From 63721408f44d3adee5f292260958f5c14eade95c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:55:47 +0100 Subject: [PATCH 01/53] test(live): a child's closed result must stay off the family page Red first: the demo family's compound-het pair in KCNH2 names the son's planted result to act on, and the family page shows it open. The new browser checks also plant a de novo candidate and a grandparent stretch over it, and check the son's copy-number dot in a closed gene. The unit checks want results.json to carry the closed results (veil.spans), facts and drawing items tagged, and core.js veilId() to answer as veil_id(). Bead: pgp-iqon.24 --- tests/demo/make_demo_sample.py | 3 +- .../live/demo/DEMO-FAMILY/compound_het.tsv | 1 + tests/fixtures/live/demo/README.md | 2 + tests/test_live_demo.py | 219 +++++++++++++++++- tests/test_live_results.py | 93 +++++++- 5 files changed, 312 insertions(+), 6 deletions(-) diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 9def678d8..352eb0b93 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -107,7 +107,8 @@ def own_chroms(): FINDINGS = {"DEMO-002": {"fmr1", "chip", "mocha"}, "DEMO-003": {"annotsv", "act"}} # The son's results to act on ("act"): a pathogenic ClinVar hit in KCNH2, an ACMG secondary-findings gene that # is on no carrier list, and an AnnotSV class 5 deletion over it. Public gene name, invented positions and -# alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose. +# alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose, and that the +# family page keeps closed the compound-het pair in KCNH2 of tests/fixtures/live/demo/DEMO-FAMILY too. PLANTED = {"gene": "KCNH2", "hit": ("chr7", 150960000, "C", "T"), "sv": ("chr7", 150955000, 150965000)} # Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv index a5927bf30..5a68b490d 100644 --- a/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv +++ b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv @@ -1,3 +1,4 @@ child gene variant_a variant_b a_from b_from phase DEMO-003 G0412 chr2:73481166:A>G chr2:73492310:C>T mother . unknown DEMO-003 G0187 chr7:94371020:G>A chr7:94398114:T>C father mother in trans +DEMO-003 KCNH2 chr7:150960000:C>T chr7:150962400:G>A father mother in trans diff --git a/tests/fixtures/live/demo/README.md b/tests/fixtures/live/demo/README.md index 62ca22167..d08c2eade 100644 --- a/tests/fixtures/live/demo/README.md +++ b/tests/fixtures/live/demo/README.md @@ -14,5 +14,7 @@ sample DEMO-001 that `tests/demo/make_demo_sample.py` writes. No real sample, pe - `DEMO-FAMILY/family_plan.tsv`, `relationships.tsv`, `ibd_pairs.tsv`, `DEMO-003_segments.tsv`, `de_novo.tsv`, `compound_het.tsv`: step 38's files for the trio, invented, in the columns step 38 writes, so the family page draws its grid, the son's painted stretches and his tables. + `compound_het.tsv` also holds a pair in KCNH2 whose first variant is the son's planted result to act on + (`make_demo_sample.py` PLANTED): the family page keeps that row closed until the result is opened there. The run itself is `../demo-001.weblog.jsonl`, written by `tests/demo/make_demo_events.py`. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 805606cfb..8f10fe575 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -53,7 +53,15 @@ marks and ranking only, and a reload closes them again; the home page and every finding page of the son (an open finding's rail and related list too) name them nowhere, and the symptom ranking's own page shows the gene - once opened on purpose; DEMO-001's page + once opened on purpose; on the family page the son's rows that name a planted + result (the compound-het pair in KCNH2 of the fixture, and a de novo + candidate and a stretch painted from a grandparent planted in family.json) + name its gene, positions and that grandparent in no row, link, title or + count until "Show the result" is clicked there, the click brings each row + back, and a reload closes them again; with the son's results.json + unavailable every row of his that could name one stays closed; on his page + a CHIP call and a copy-number dot in the gene of a closed result stay out + of their drawings until it is opened; DEMO-001's page mid-replay and at the end leads with the report (a short run band above the lead while it runs, none once finished), ends with the step timeline folded, keeps the wire to at most eight one-sentence lines with the plain @@ -64,7 +72,8 @@ the veil, person.js and finding.js that read a missing state as a standalone adult, and search.js without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag -and finding.js whose findings ignore their tags), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil +and core.js whose veilId() tags nothing; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -314,6 +323,8 @@ def browser_checks(out): fold_checks(b, base, out) veil_checks(b, base, out) planted_checks(b, base, out) + family_veil_checks(b, base, out) + widget_veil_checks(b, base, out) finally: b.close() srv.shutdown() @@ -408,7 +419,8 @@ def family_depth(b, base): b.go("about:blank") b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{FAM}/") - return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && JSON.stringify([" + return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "(document.documentElement.dataset.pgpKids || '').indexOf('DEMO-003:known') >= 0 && JSON.stringify([" "document.querySelectorAll('table.rel-grid td:not(.self):not(.none)').length, " "document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length, " "document.querySelectorAll('details[data-child=\"DEMO-003\"] table.rows').length, " @@ -716,6 +728,207 @@ def write(p, text): write(p, text) +# The family page: the son's rows that name his planted result to act on. +FAMILY_SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('[href],[title],[aria-label]," + "[alt]'), function (e) { return ['href', 'title', 'aria-label', 'alt'].map(function (k) { " + "return e.getAttribute(k) || ''; }).join(' '); })).join(' ')") +GRANDPARENT = "DEMO-GF" +PAIR_B = "150962400" # the second variant of the fixture's compound-het pair in KCNH2 + + +def no_cache(b): + """Chrome keeps a script it fetched a moment ago without asking again: the controls below swap a script, so + every request goes to the server from here on.""" + b.call("Network.enable", {}, b.session) + b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) + + +def family_page(b, base): + """The family page once it knows the son's results (or after 10 s, for a page that never learns them).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{FAM}/") + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "/DEMO-003:(known|failed)/.test(document.documentElement.dataset.pgpKids || '') && 'ready'", 10) + time.sleep(0.3) + + +def family_closed(b, base): + """[problems] of the family page before any click: the son's planted gene, its positions or the grandparent of + the stretch over it named anywhere; the rows that name nothing closed gone too.""" + gene, hit, sv = planted_words() + family_page(b, base) + shown = b.eval(FAMILY_SHOWN_JS) or "" + out = [f"{w} shown on the family page while closed" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + closed = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed').length") + # The stretches' table, the 2 de novo rows of the fixture and its 2 other pairs stay; one closed box each. + if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or segs != 1: + out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} closed stretches") + return out + + +def family_problems(b, base): + """[problems] of the family page: closed, then opened with one click, then reloaded.""" + gene, hit, _ = planted_words() + out = family_closed(b, base) + if out: + return out + b.eval("document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true") + time.sleep(0.5) + shown = b.eval(FAMILY_SHOWN_JS) or "" + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + if not all(w in shown for w in [gene, GRANDPARENT, hit[0]]) or json.loads(rows or "[0,0,0]")[1:] != [3, 3] or left: + out.append(f"one click on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") + out += [f"after a reload: {x}" for x in family_closed(b, base)] + return out + + +def family_unknown(b, base): + """[problems] of the family page with the son's results.json unavailable: a row of his named, or any of his de + novo or compound-het rows open.""" + gene, hit, sv = planted_words() + family_page(b, base) + shown = b.eval(FAMILY_SHOWN_JS) or "" + out = [f"{w} shown with the son's results unavailable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + if rows is None or len(json.loads(rows)) != 1: + out.append(f"the son's de novo or compound-het rows open with his results unavailable: tables {rows}") + return out + + +def family_veil_checks(b, base, out): + """On the family page a child's rows that name a closed result stay closed as on the child's page, until that + result is opened there; a page that cannot read the child's results keeps every row of his that could closed.""" + no_cache(b) + import make_demo_sample + fj = os.path.join(out, "runs", FAM, "family.json") + rj = os.path.join(out, "runs", OTHERS[1], "results.json") + fp, cj = (os.path.join(out, "assets", x) for x in ("family-page.js", "core.js")) + keep = {} + for p in (fj, fp, cj): + with open(p) as f: + keep[p] = f.read() + + def write(p, text): + with open(p, "w") as f: + f.write(text) + try: + # A de novo candidate at the planted hit, and the father's copy over it painted from a grandparent. + fam = json.loads(keep[fj]) + c, pos, ref, alt = make_demo_sample.PLANTED["hit"] + row = dict(fam["de_novo"][0], chrom=c, pos=str(pos), ref=ref, alt=alt) + fam["de_novo"].append(row) + segs = fam["segments"]["DEMO-003"] + over = [s_ for s_ in segs if s_["chrom"] == c and s_["side"] == "paternal" and int(s_["start"]) <= pos <= int(s_["end"])] + if len(over) != 1: + check("browser: the son's stretches hold one paternal stretch over the planted hit", False, over) + return + over[0]["ancestor"] = GRANDPARENT + write(fj, json.dumps(fam)) + keep["planted"] = json.dumps(fam) + bad = family_problems(b, base) + check("browser: on the family page the son's rows that name his planted result (a compound-het pair, a de novo " + "candidate, a stretch from a grandparent) name its gene, positions and grandparent nowhere until 'Show the " + "result' is clicked there, one click brings each row back, and a reload closes them", not bad, bad) + for p, a, z, desc in ( + (fp, "function isClosed(child, tag) { return !!tag && ", "function isClosed(child, tag) { return false && ", + "family-page.js whose rows ignore the veil"), + (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): + if a not in keep[p]: + check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) + continue + write(p, keep[p].replace(a, z)) + control(desc, not family_closed(b, base)) + write(p, keep[p]) + os.replace(rj, rj + ".away") + try: + bad = family_unknown(b, base) + finally: + os.replace(rj + ".away", rj) + check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " + "a result to act on closed", not bad, bad) + finally: + keep.pop("planted", None) + for p, text in keep.items(): + write(p, text) + + +def widget_veil_checks(b, base, out): + """On the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of sight until that + result is opened; the other dot stays drawn. A CHIP call is also a finding of its card that names its gene, so + the CHIP card closes whole with the result; SMN1 at two copies is no finding, so only its dot's tag closes it.""" + no_cache(b) + rj = os.path.join(out, "runs", OTHERS[1], "results.json") + js = os.path.join(out, "assets", "person.js") + with open(rj) as f: + keep = f.read() + with open(js) as f: + keep_js = f.read() + + def drawn(): + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/") + b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " + "&& document.querySelector('[data-widget=copies]') && document.querySelector('.veil') && 'drawn'", 15) + return shown() + + def shown(): + return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || ''])") + + def problems(text, opened): + page, cops = json.loads(text) if text else ["", ""] + out_ = [] + if "SMN2" not in cops: + out_.append(f"the open dot is not drawn: {cops[:120]}") + if ("p.R175H" in page) != opened or ("SMN1" in cops) != opened: + out_.append(f"the CHIP call or the SMN1 dot of the closed results {'not ' if opened else ''}shown: " + f"{'p.R175H' in page} | {cops[:200]}") + return out_ + try: + with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: + summary = json.load(f) + S = summary.setdefault("sections", {}) + S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ + {"gene": "TP53", "variant": "c.524G>A", "final": "Pathogenic", "clinvar": "Pathogenic"}, + {"gene": "SMN1", "variant": "c.1A>G", "final": "Pathogenic", "clinvar": "Pathogenic"}]}} + S["chip"] = {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ + {"gene": g, "protein_change": ch, "vaf": v, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"} + for g, ch, v in (("DNMT3A", "p.R882H", 0.12), ("TP53", "p.R175H", 0.08))], + "genes": [{"gene": g, "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"} for g in ("DNMT3A", "TP53")]}} + S["paralogs"] = {"state": "ok", "data": {"genes": [{"gene": g, "copy_number": n, "status": "called", "detail": "."} + for g, n in (("SMN1", 2), ("SMN2", 2))]}} + res = live_results.build(OTHERS[1], summary) + with open(rj, "w") as f: + json.dump(res, f) + bad = problems(drawn(), False) + if not bad: + for _ in range(2): # the CPSR card holds both results: one click opens one + b.eval("document.querySelector('#card-cpsr .veil button').click(); true") + time.sleep(0.5) + bad = problems(shown(), True) + check("browser: on the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of " + "sight until it is opened", not bad, bad) + a = "add(text, widget(of[1]));" + if a not in keep_js: + check("control: person.js that draws every item (the planted line is in person.js)", False, a) + else: + with open(js, "w") as f: + f.write(keep_js.replace(a, "add(text, widget(c.widget));")) + control("person.js that draws every item", not problems(drawn(), False)) + finally: + with open(rj, "w") as f: + f.write(keep) + with open(js, "w") as f: + f.write(keep_js) + + # finding.js as it was: a missing state or one with no plugins read as a standalone adult. OLD_FINDING = ("if (!st || !st.plugins) { return Promise.resolve(UNKNOWN); }\n var fam = st.plugins.family;", "var fam = st && st.plugins && st.plugins.family;") diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 8213596a2..7a403b68c 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -42,12 +42,17 @@ the ids and no other row does, the ranking's finding carries the secondary finding's id, and so does a segment of the lead that names the gene; the home page's view of the run (overview) lists no such finding among its - first ones and its closed lead names none. + first ones and its closed lead names none; a CHIP call, a copy-number dot + and a fact that name a closed result's gene carry its id (the CHIP chart's + panel of genes never does), results.json carries the closed results as + veil.spans, and core.js veilId(), which tags the family page's rows of a + child, answers each row as veil_id() does, under node. Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows and lines) and one that tags every row. A check +8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row, and +veilId() with its gene rule or its overlap rule dropped. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -402,6 +407,75 @@ def veil_lines_problems(): return out +def widget_problems(): + """What is wrong with the veil's tags of a card's facts and drawing: a CHIP call and a copy-number dot, and a + fact, that name a closed result's gene carry its id; the CHIP chart's list of genes, the same panel for + everyone, is never tagged; nothing else is.""" + found = [{"id": "cpsr-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53 c.524G>A"}, + {"id": "two-hits-smn1-test", "level": "act", "gene": "SMN1"}] + S = {"chip": {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ + {"gene": "DNMT3A", "protein_change": "p.R882H", "vaf": 0.12, "alt_reads": 9, "depth": 75, "status": "pass", "note": "a clone"}, + {"gene": "TP53", "protein_change": "p.R175H", "vaf": 0.08, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"}], + "genes": [{"gene": "DNMT3A", "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"}, + {"gene": "TP53", "detection_limit_vaf": 0.06, "mean_depth": 70, "status": "ok"}]}}, + "paralogs": {"state": "ok", "data": {"genes": [ + {"gene": "SMN1", "copy_number": 1, "status": "called", "detail": "."}, + {"gene": "SMN2", "copy_number": 2, "status": "called", "detail": "."}]}}} + summ = {"sections": S} + chip = dict(live_results.a_chip(S, summ), id="chip") + par = dict(live_results.a_paralogs(S, summ), id="paralogs") + par["facts"] = [["SMN1 copies", "1"], ["Genes read", "2"]] + v = live_results.veil(summ, found, [{"sections": [chip, par]}]) + out = [] + want = {"chip": [None, "cpsr-tp53-test"], "paralogs": ["two-hits-smn1-test", None]} + if v.get("widget") != want: + out.append(f"drawings tagged {v.get('widget')}, want {want}") + if (v.get("facts") or {}) != {"paralogs": ["two-hits-smn1-test", None]}: + out.append(f"facts tagged {v.get('facts')}") + if v.get("spans") != [list(x) for x in live_results.closable(found)]: + out.append(f"spans {v.get('spans')}") + return out + + +# Rows as the family page tags them (family-page.js): [text, chrom, pos, end], against two closed results. +VEIL_SPANS = [["sf-kcnh2", "KCNH2", "chr7", 150960000, 150960000], ["sv-del", "KCNH2", "chr7", 150955000, 150965000], + ["two-hits-hla", "HLA-A", None, None, None]] +VEIL_ROWS = [["chr7:150960000 C>T", "chr7", "150960000", None], ["chr7:150961000 G>A", "chr7", "150961000", None], + ["chr7:150970000 G>A", "chr7", "150970000", None], ["chr3:48211904 C>T", "chr3", "48211904", None], + ["KCNH2 chr7:94371020:G>A chr7:94398114:T>C", None, None, None], + ["G0187 chr7:150,962,000:G>A chr7:1:T>C", None, None, None], ["G0412 chr2:1:A>G", None, None, None], + ["chr7:61600000-159285973", "chr7", "61600000", "159285973"], ["in KCNH2.", None, None, None], + ["HLA-A*02:01", None, None, None], ["HLA-AB", None, None, None], ["kcnh2 lower case", None, None, None], + ["7:150960000", "7", 150960000, None], ["chrX:1-2", "chrX", "x", None], ["", "chr7", "150955000", "150965000"]] +CORE_JS = os.path.join(ASSETS, "core.js") +NODE_VEIL = ("const vm=require('vm'),fs=require('fs');const w={};" + "const doc={documentElement:{setAttribute(){}},addEventListener(){}};" + "vm.runInNewContext(fs.readFileSync(process.argv[1],'utf8'),{window:w,document:doc});" + "const sp=JSON.parse(process.argv[2]);" + "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3]))))") + + +def node_veil(js_path): + """veilId() of core.js at JS_PATH on VEIL_ROWS, under node: [ids] or the error.""" + node = shutil.which("node") + if not node: + return "no node" + p = subprocess.run([node, "-e", NODE_VEIL, js_path, json.dumps(VEIL_SPANS), json.dumps(VEIL_ROWS)], + capture_output=True, text=True) + return json.loads(p.stdout) if p.returncode == 0 else p.stderr + + +def veil_js_problems(js_path): + """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS.""" + want = [live_results.veil_id([tuple(x) for x in VEIL_SPANS], *r) for r in VEIL_ROWS] + got = node_veil(js_path) + if got != want: + return [f"veilId {got} != veil_id {want}"] + if len({w for w in want}) < 4: + return [f"the rows exercise too few answers: {want}"] + return [] + + def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -519,6 +593,20 @@ def tagged(sample_dir): bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) + bad = widget_problems() + check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " + "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) + check("8. node is on the PATH (ubuntu-latest has it)", shutil.which("node") is not None) + bad = veil_js_problems(CORE_JS) + check("8. core.js veilId() tags each family-page row as veil_id() does (under node)", not bad, bad) + with open(CORE_JS) as f: + core = f.read() + for a, z, desc in (("if (s[1] && words[s[1]]) { return s[0]; }", "", "veilId() without its gene rule"), + ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule")): + planted = os.path.join(work, "core-planted.js") + with open(planted, "w") as f: + f.write(core.replace(a, z)) + control(desc, a in core and not veil_js_problems(planted)) bad = veil_problems(*tagged(d3)) check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " @@ -529,6 +617,7 @@ def tagged(sample_dir): live_results.veil_id = lambda spans, *a, **k: None control("a veil rule that tags nothing", not veil_problems(*tagged(d3))) control("a veil rule that tags no line", not veil_lines_problems()) + control("a veil rule that tags no fact or drawing", not widget_problems()) live_results.veil_id = lambda spans, *a, **k: spans[0][0] if spans else None control("a veil rule that tags every row", not veil_problems(*tagged(d3))) finally: From 76354c4481375a2f84625e34d4ab4957006596be Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:05:41 +0100 Subject: [PATCH 02/53] fix(live): tag the closed results for the family page, and close a child's dots and facts with them results.json carries the closed results as veil.spans, and tags each card's facts and the items of its drawing (CHIP calls, copy-number dots). core.js gets veilId(), a mirror of veil_id() for the family page's rows that a unit check runs against the Python rule under node, and the closed box both pages draw. Bead: pgp-iqon.24 --- assets/live/core.js | 55 ++++++++++++++++++++++++++++++++++++++++++- assets/live/person.js | 43 +++++++++++++++++++-------------- bin/live_results.py | 19 +++++++++++++-- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index b6e4ff071..e0aa05e32 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -208,8 +208,61 @@ return isFinite(x) && isFinite(y) && y > x ? (y - x) / 3600e3 : null; } + // --- the veil: a child's results to act on stay closed until someone opens them on purpose --------- + // The closed box, the same on a child's page (person.js) and on the family page (family-page.js): what stays + // closed, and the button that opens it. + function veil(text, button, onOpen) { + var v = el('div', { 'class': 'veil' }); + add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), text)); + var b = el('button', { type: 'button', 'class': 'btn' }, button || 'Show the result'); + b.addEventListener('click', onOpen); + add(v, b); + return v; + } + // The id of the closed result a row belongs to, or null: bin/live_results.py veil_id() word for word, for the + // rows only the family page holds (a child's de novo and compound-het candidates). spans: results.json's + // veil.spans, [[id, gene, chrom, start, end]]. text: what the row shows (its gene names; its loci are read + // too); chrom, pos and end its own position. tests/test_live_results.py runs it under node against veil_id(). + var LOCUS_RE = /(?= s[3]; })) { return s[0]; } + } + return null; + } + window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours }; + hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId }; }()); diff --git a/assets/live/person.js b/assets/live/person.js index 6dcd7490f..c0b582ac8 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -344,6 +344,22 @@ out.rows = rows; return [rows.length ? out : null, t.rows.length - rows.length]; } + // A card's facts and drawing without their closed items (a fact, a CHIP call or a copy-number dot that names a + // closed result's gene): [facts, widget, how many items they left out]. + function openFacts(c) { + var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); }); + var w = c.widget, key = w && (w.kind === 'vaf' ? 'calls' : w.kind === 'copies' ? 'genes' : null); + var wt = veilTags('widget', c.id), shut = (c.facts || []).length - facts.length; + if (key && wt.length && w[key]) { + var items = w[key].filter(function (x, i) { return !(wt[i] && tagClosed(wt[i])); }); + shut += w[key].length - items.length; + var copy = {}; + Object.keys(w).forEach(function (k) { copy[k] = w[k]; }); + copy[key] = items; + w = copy; + } + return [facts, w, shut]; + } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with @@ -398,20 +414,16 @@ } else if (g === 'empty') { add(text, el('p', { 'class': 'notassessed' }, 'Not assessed: its step ended without a result for this section.')); } else if (closedFs.length) { - var v = el('div', { 'class': 'veil' }); // One button opens one result: a card with two results to act on asks twice. - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - (closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + - ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.')); - var vb = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - vb.addEventListener('click', function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(v, vb); - add(text, v); + add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + + ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, + function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } - add(margin, U.facts(c.facts)); - add(text, widget(c.widget)); + var of = openFacts(c); + add(margin, U.facts(of[0])); + add(text, widget(of[1])); Object.keys((results && results.charts) || {}).forEach(function (chart) { if (results.charts[chart] === c.id) { add(text, el('div', { 'class': 'chart-slot', 'data-chart-slot': chart })); } }); @@ -419,7 +431,7 @@ var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length; + var shut = ot[1] + (c.lines || []).length - lines.length + of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } if (shut) { add(text, el('p', { 'class': 'dim veil-rows' }, (shut === 1 ? 'One row stays' : shut + ' rows stay') + @@ -534,13 +546,8 @@ if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); if (veiled(f)) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - 'A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.')); - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(v, b); - add(art, v); + add(art, U.veil('A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.', + null, function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); add(row, art, el('aside', {})); return row; } diff --git a/bin/live_results.py b/bin/live_results.py index ef5d93eb3..a2202011f 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1210,11 +1210,16 @@ def veil(summary, found, chapters, lead_parts=()): intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" - names left out: the page closes those itself). Empty when nothing can be closed.""" + names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items + of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself, so the + family page (assets/live/family-page.js) can tag its own rows of a child, the de novo and compound-het + candidates, by this rule (core.js veilId(), checked against veil_id() under node). Empty when nothing can + be closed.""" spans = closable(found) if not spans: return {} - out = {"rows": {}, "viewer": {}, "lines": {}, "marks": {}, "findings": {}, "lead": []} + out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "widget": {}, "marks": {}, "findings": {}, "lead": [], + "spans": [list(s) for s in spans]} S = summary.get("sections") or {} hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] tags = [veil_id(spans, str(h.get("gene") or ""), h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None @@ -1238,6 +1243,16 @@ def veil(summary, found, chapters, lead_parts=()): tags = [veil_id(spans, ln) for ln in c.get("lines") or []] if any(tags): out["lines"][c["id"]] = tags + tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] + if any(tags): + out["facts"][c["id"]] = tags + # A drawing's items: the CHIP calls (gene and change) and the copy-number dots (gene). The CHIP + # chart's list of genes with their detection limits is the same panel for everyone: never tagged. + w = c.get("widget") or {} + items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None + tags = [veil_id(spans, f"{x.get('gene') or ''} {x.get('change') or ''}") for x in items or []] + if any(tags): + out["widget"][c["id"]] = tags try: specs = rr.chart_specs(summary) except (KeyError, TypeError, ValueError, AttributeError, IndexError, ZeroDivisionError): From 1bf0623a0a529e1b19842d8ab2c98c22e1c1e46b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:05:41 +0100 Subject: [PATCH 03/53] fix(live): keep a child's closed result off the family page The family page reads each child's results.json and keeps closed every de novo candidate, compound-het pair and grandparent stretch that names a closed result, until it is opened there; while it cannot read them, every such row of that child stays closed. Bead: pgp-iqon.24 --- CHANGELOG.md | 2 + assets/live/family-page.js | 131 ++++++++++++++++++++++++++++++++++--- assets/live/live.css | 1 + docs/live-site.md | 6 +- tests/test_live_demo.py | 2 +- 5 files changed, 130 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fcb5480..1ff9e66f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a stretch painted from a grandparent over one, stay behind "Show the result" until that result is opened there; a reload closes them again. While the page cannot read the child's results, every such row of that child stays closed. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. + - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. - The seven `##INFO=` header descriptions end in `(og vcf csq-rank)`, not `(bin/csq_rank.awk)`. No tag, value or column changes. - A ranking floor that is not a number, or a `--keep` other than `all` or `noncommon`, is a usage error (exit 2). awk read such a floor as 0. diff --git a/assets/live/family-page.js b/assets/live/family-page.js index ed5421947..2cbf03b6a 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -6,6 +6,9 @@ // phone), each child's stretches painted by parent, the de novo and compound-het candidates, and the // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. +// A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json +// gives the closed results (veil.spans), and a row of the child that names one (a de novo or compound-het +// candidate, a stretch painted from a grandparent) stays closed until that result is opened on purpose here. (function () { 'use strict'; var U = window.PGPUI; @@ -70,6 +73,73 @@ } function memberUrl(f, s) { return f.runs && f.runs[s] ? f.runs[s].url : null; } + // --- the veil --------------------------------------------------------------------------------------- + // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child + // with no run here, or whose results.json has not answered or could not be read, cannot say which rows name a + // result to act on, so every row of theirs that could stays closed (UNKNOWN) until opened on purpose. + // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. + var UNKNOWN = '?'; + var kids = {}, opened = {}, openedAll = {}, last = null; + function loadKids(f) { + (f.state.children || []).forEach(function (c) { + var url = memberUrl(f, c.child); + var k = kids[c.child]; + if (!url) { kids[c.child] = { status: 'failed' }; return; } + if (k && k.url === url && k.busy) { return; } + kids[c.child] = k && k.url === url ? k : { status: 'pending', url: url }; + kids[c.child].busy = true; + U.getJSON(url + 'results.json').then(function (r) { + var cur = kids[c.child]; + if (!cur || cur.url !== url) { return; } + cur.busy = false; + var v = (r && r.veil) || {}; + var acts = ((r && r.findings) || []).filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }); + var next = r ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; + // An answer that cannot be read keeps what was known: the page never opens on a failed request. + if (!r && cur.status === 'known') { return; } + var key = JSON.stringify([next.status, next.spans, next.acts]); + if (key === cur.key) { return; } + next.key = key; + kids[c.child] = next; + if (last) { redraw(); } + }).catch(function () { + var cur = kids[c.child]; + if (!cur || cur.url !== url) { return; } + cur.busy = false; + if (cur.status !== 'known' && cur.status !== 'failed') { cur.status = 'failed'; cur.key = 'failed'; if (last) { redraw(); } } + }); + }); + } + // The closed result a row of CHILD belongs to: its id, UNKNOWN when this page cannot tell, or null. + function tagOf(child, text, chrom, pos, end) { + var k = kids[child]; + if (!k || k.status !== 'known') { return UNKNOWN; } + if (!k.spans) { return k.acts.length ? UNKNOWN : null; } // results without the tags: cannot tell + return U.veilId(k.spans, text, chrom, pos, end); + } + function isClosed(child, tag) { return !!tag && !(tag === UNKNOWN ? openedAll[child] : opened[tag]); } + // The rows of CHILD split by the veil: { open: rows, closed: [tags of the closed rows] }. + function veiled(child, rows, tagFn) { + var out = { open: [], closed: [] }; + rows.forEach(function (r) { + var t = tagFn(r); + if (isClosed(child, t)) { out.closed.push(t); } else { out.open.push(r); } + }); + return out; + } + // The closed box under a table of CHILD: how many rows stay closed, and one button that opens one result. + function veilBox(child, closed, what, whats) { + if (!closed.length) { return null; } + var n = closed.length === 1 ? 'One ' + what + ' stays' : closed.length + ' ' + whats + ' stay'; + var first = closed[0]; + if (first === UNKNOWN) { + return U.veil(n + ' closed: this page cannot read ' + child + '’s results yet, so it cannot tell which ' + whats + + ' name a result to act on in adult life.', 'Show the ' + whats, function () { openedAll[child] = true; redraw(); }); + } + return U.veil(n + ' closed with the result to act on in adult life ' + (closed.length === 1 ? 'it belongs' : 'they belong') + + ' to, until that result is opened.', null, function () { opened[first] = true; redraw(); }); + } + // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. function pedigree(f) { var L = f.layout, st = f.state; @@ -286,7 +356,13 @@ // A child's stretches, painted: per chromosome the father's copy above the mother's, each stretch // coloured by the grandparent it came from (or by the parent when no grandparent was sequenced). var SIDE = { paternal: 'father', maternal: 'mother' }; - function painting(f, child, rows) { + function painting(f, child, all) { + // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a + // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. + var split = veiled(child, all, function (r) { + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : null; + }); + var rows = split.open; var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -305,6 +381,14 @@ ['paternal', 'maternal'].forEach(function (side) { var track = el('span', { 'class': 'paint-track ' + side }); track.style.setProperty('--len', (100 * cl[1] / longest).toFixed(2) + '%'); + all.filter(function (r) { return r.chrom === chrom && r.side === side && rows.indexOf(r) < 0; }).forEach(function (r) { + var a = parseInt(r.start, 10), b = parseInt(r.end, 10); + if (!isFinite(a) || !isFinite(b) || b < a) { return; } + var seg = el('span', { 'class': 'paint-seg closed', title: 'A closed stretch' }); + seg.style.setProperty('--a', (100 * a / cl[1]).toFixed(3) + '%'); + seg.style.setProperty('--w', Math.max(0.4, 100 * (b - a) / cl[1]).toFixed(3) + '%'); + add(track, seg); + }); rows.filter(function (r) { return r.chrom === chrom && r.side === side; }).forEach(function (r) { var a = parseInt(r.start, 10), b = parseInt(r.end, 10); if (!isFinite(a) || !isFinite(b) || b < a) { return; } @@ -329,8 +413,10 @@ who + (colours[who].basis === 'other copy' ? ' (the copy not from the sequenced parent)' : '')); add(leg, it); }); - add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); + add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).' + + (split.closed.length ? ' Hatched: a closed stretch.' : ''))); add(fig, leg); + add(fig, veilBox(child, split.closed, 'stretch', 'stretches')); var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -361,27 +447,41 @@ if (rows && rows.length) { add(d, painting(f, c.child, rows)); } else if (c.segments) { - add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); + // The drawing as an image cannot close one stretch: while this child has a result to act on that is + // closed (or this page cannot tell), it stays closed whole. + var k = kids[c.child] || {}; + var shut = k.status !== 'known' ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }).filter(function (id) { return !opened[id]; }); + if (!k.spans && k.status === 'known' && k.acts.length) { shut = [UNKNOWN]; } + if (shut.length && !(shut[0] === UNKNOWN && openedAll[c.child])) { + add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + + 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[t] = true; } }); redraw(); })); + } else { + add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); + } } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); } var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'De novo candidates')); if (dn.length) { + var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos); }); add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], - rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + rows: dv.open.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); + add(d, veilBox(c.child, dv.closed, 'row', 'rows')); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { + var cv = veiled(c.child, ch, function (r) { return tagOf(c.child, [r.gene, r.variant_a, r.variant_b].join(' ')); }); add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], - rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + rows: cv.open.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + add(d, veilBox(c.child, cv.closed, 'row', 'rows')); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -431,7 +531,12 @@ } function draw(f) { + last = f; var main = document.getElementById('family'); + // A redraw (a result opened, a child's results answered) keeps each child's fold as the reader left it. + var folds = {}; + Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { folds[x.getAttribute('data-child')] = x.open; }); + var y = window.scrollY; main.textContent = ''; var page = el('section', { 'class': 'family-page', 'data-plugin': 'family', 'aria-label': 'Family ' + f.state.name }); if (f.state.error) { @@ -462,14 +567,22 @@ add(page, related(f), couples(f), children(f), sideBySide(f)); if ((f.state.plan || []).length || f.state.reports.length) { add(page, more(f)); } add(main, page); + Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { + var k = x.getAttribute('data-child'); + if (k in folds) { x.open = folds[k]; } + }); + window.scrollTo(0, y); + // What the page knows of each child's results (for the tests): "DEMO-003:known". + root.setAttribute('data-pgp-kids', Object.keys(kids).sort().map(function (c) { return c + ':' + kids[c].status; }).join(' ')); root.setAttribute('data-pgp-status', 'ready'); } + function redraw() { if (last) { draw(last); } } // The page asks again every 30 s and redraws only when family.json changed (a member's run moved on, // step 39 finished). A failed first request says so, with a retry; a later one keeps what is shown. - var shown = null; + var drawnKey = null; function failed() { - if (shown !== null) { return; } + if (drawnKey !== null) { return; } var main = document.getElementById('family'); main.textContent = ''; var b = el('button', { type: 'button', 'class': 'more' }, 'Try again'); @@ -481,7 +594,9 @@ return U.getJSON('family.json').then(function (f) { if (!f) { failed(); return; } var k = JSON.stringify(f); - if (k !== shown) { shown = k; draw(f); } + // Each child's results are asked again with the family: a result to act on can appear while a run goes. + if (!f.state.error) { loadKids(f); } + if (k !== drawnKey) { drawnKey = k; draw(f); } }).catch(failed); } function start() { diff --git a/assets/live/live.css b/assets/live/live.css index 5dba9f176..4adff6602 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -607,6 +607,7 @@ ul.rel-pairs .chip { white-space: normal; } .anc1 { background: var(--anc1); } .anc2 { background: var(--anc2); } .anc3 { background: var(--anc3); } .anc4 { background: var(--anc4); } .anc5 { background: var(--anc5); } .paint-seg.other, .paint-swatch.other { background-image: repeating-linear-gradient(45deg, transparent 0 2px, rgba(0, 0, 0, .35) 2px 4px); } +.paint-seg.closed { background: repeating-linear-gradient(-45deg, rgba(143, 141, 134, .45) 0 2px, transparent 2px 6px); } .paint-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; font-size: 12.5px; margin-top: 8px; } .paint-key { display: inline-flex; gap: 6px; align-items: center; } .paint-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 3px; } diff --git a/docs/live-site.md b/docs/live-site.md index 11b784bc2..0f6935941 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a copy-number dot (a CHIP call is a finding of its own card, so the CHIP card closes whole), a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, and a reload closes it again. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, or a request that failed), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. @@ -88,7 +88,7 @@ The reads cross the network to the browser that logged in: only the byte ranges `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words` and `databases`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it, and `spans`, the closed results with their gene and position, which the family page tags a child's rows with by the same rule). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 8f10fe575..54431709f 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -408,7 +408,7 @@ def widget_checks(b, base, out): keep = f.read() try: with open(js, "w") as f: - f.write(keep.replace("add(text, widget(c.widget));", "")) + f.write(keep.replace("add(text, widget(of[1]));", "")) control("person.js without its widget line", not widget_problems(widget_counts(b, base))) finally: with open(js, "w") as f: From 0d715a074a53d84149d9ea6b3627610ef7399afc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:42:09 +0100 Subject: [PATCH 04/53] fix(live): keep a family row closed until every closed result it names is opened A compound-het pair or a painted stretch can name two closed results: core.js veilIds() lists each one by veil_id()'s rule, and the row stays closed until all are opened. Opened results are kept per child and run, since two children can carry the same result id. Results the server could not build count as unreadable. On a child's page the phone's card line takes only an open fact, and the closed count names rows and items apart. Bead: pgp-iqon.24 --- assets/live/core.js | 7 +++++- assets/live/family-page.js | 45 ++++++++++++++++++++++---------------- assets/live/person.js | 14 +++++++----- docs/live-site.md | 2 +- tests/test_live_demo.py | 33 ++++++++++++++++++++++------ tests/test_live_results.py | 24 ++++++++++++-------- 6 files changed, 83 insertions(+), 42 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index e0aa05e32..fff06e955 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -260,9 +260,14 @@ } return null; } + // Every closed result a row belongs to, by the same rule taken one result at a time: a compound-het pair can + // name two, and a painted stretch can hold two, so the family page keeps such a row closed until each is opened. + function veilIds(spans, text, chrom, pos, end) { + return (spans || []).filter(function (s) { return veilId([s], text, chrom, pos, end); }).map(function (s) { return s[0]; }); + } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId }; + hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId, veilIds: veilIds }; }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 2cbf03b6a..1253b4d59 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -77,9 +77,11 @@ // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child // with no run here, or whose results.json has not answered or could not be read, cannot say which rows name a // result to act on, so every row of theirs that could stays closed (UNKNOWN) until opened on purpose. - // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. + // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They + // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; var kids = {}, opened = {}, openedAll = {}, last = null; + function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { (f.state.children || []).forEach(function (c) { var url = memberUrl(f, c.child); @@ -93,10 +95,12 @@ if (!cur || cur.url !== url) { return; } cur.busy = false; var v = (r && r.veil) || {}; - var acts = ((r && r.findings) || []).filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }); - var next = r ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; + // Results the server could not build come back without their findings: as unreadable as no answer. + var usable = !!(r && Array.isArray(r.findings)); + var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; + var next = usable ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; // An answer that cannot be read keeps what was known: the page never opens on a failed request. - if (!r && cur.status === 'known') { return; } + if (!usable && cur.status === 'known') { return; } var key = JSON.stringify([next.status, next.spans, next.acts]); if (key === cur.key) { return; } next.key = key; @@ -110,20 +114,23 @@ }); }); } - // The closed result a row of CHILD belongs to: its id, UNKNOWN when this page cannot tell, or null. + // The closed results a row of CHILD belongs to: their ids, [UNKNOWN] when this page cannot tell, or []. function tagOf(child, text, chrom, pos, end) { var k = kids[child]; - if (!k || k.status !== 'known') { return UNKNOWN; } - if (!k.spans) { return k.acts.length ? UNKNOWN : null; } // results without the tags: cannot tell - return U.veilId(k.spans, text, chrom, pos, end); + if (!k || k.status !== 'known') { return [UNKNOWN]; } + if (!k.spans) { return k.acts.length ? [UNKNOWN] : []; } // results without the tags: cannot tell + return U.veilIds(k.spans, text, chrom, pos, end); } - function isClosed(child, tag) { return !!tag && !(tag === UNKNOWN ? openedAll[child] : opened[tag]); } - // The rows of CHILD split by the veil: { open: rows, closed: [tags of the closed rows] }. + // The results of TAGS still closed for CHILD: a row stays closed while one of them is. + function stillClosed(child, tags) { + return tags.filter(function (t) { return t === UNKNOWN ? !openedAll[child] : !opened[openKey(child, t)]; }); + } + // The rows of CHILD split by the veil: { open: rows, closed: [the still closed results of each closed row] }. function veiled(child, rows, tagFn) { var out = { open: [], closed: [] }; rows.forEach(function (r) { - var t = tagFn(r); - if (isClosed(child, t)) { out.closed.push(t); } else { out.open.push(r); } + var t = stillClosed(child, tagFn(r)); + if (t.length) { out.closed.push(t); } else { out.open.push(r); } }); return out; } @@ -131,13 +138,13 @@ function veilBox(child, closed, what, whats) { if (!closed.length) { return null; } var n = closed.length === 1 ? 'One ' + what + ' stays' : closed.length + ' ' + whats + ' stay'; - var first = closed[0]; + var first = closed[0][0]; if (first === UNKNOWN) { return U.veil(n + ' closed: this page cannot read ' + child + '’s results yet, so it cannot tell which ' + whats + ' name a result to act on in adult life.', 'Show the ' + whats, function () { openedAll[child] = true; redraw(); }); } return U.veil(n + ' closed with the result to act on in adult life ' + (closed.length === 1 ? 'it belongs' : 'they belong') + - ' to, until that result is opened.', null, function () { opened[first] = true; redraw(); }); + ' to, until that result is opened.', null, function () { opened[openKey(child, first)] = true; redraw(); }); } // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. @@ -360,7 +367,7 @@ // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : null; + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : []; }); var rows = split.open; var lengths = f.chrom_lengths || []; @@ -450,11 +457,11 @@ // The drawing as an image cannot close one stretch: while this child has a result to act on that is // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; - var shut = k.status !== 'known' ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }).filter(function (id) { return !opened[id]; }); - if (!k.spans && k.status === 'known' && k.acts.length) { shut = [UNKNOWN]; } - if (shut.length && !(shut[0] === UNKNOWN && openedAll[c.child])) { + var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); + var shut = stillClosed(c.child, all); + if (shut.length) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + - 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[t] = true; } }); redraw(); })); + 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[openKey(c.child, t)] = true; } }); redraw(); })); } else { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } diff --git a/assets/live/person.js b/assets/live/person.js index c0b582ac8..4a45383d6 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -431,11 +431,15 @@ var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length + of[2]; + var shut = ot[1] + (c.lines || []).length - lines.length, items = of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } - if (shut) { - add(text, el('p', { 'class': 'dim veil-rows' }, (shut === 1 ? 'One row stays' : shut + ' rows stay') + - ' closed with the result to act on ' + (shut === 1 ? 'it belongs' : 'they belong') + ' to, until that result is opened.')); + if (shut || items) { + // Rows of its table and lines, and items of its facts and drawing, counted apart. + var what = [shut ? (shut === 1 ? 'One row' : shut + ' rows') : '', items ? (items === 1 ? (shut ? 'one item' : 'One item') : items + ' items') + + ' of its numbers or drawing' : ''].filter(Boolean).join(' and '); + var many = shut + items > 1; + add(text, el('p', { 'class': 'dim veil-rows' }, what + (many ? ' stay' : ' stays') + + ' closed with the result to act on ' + (many ? 'they belong' : 'it belongs') + ' to, until that result is opened.')); } add(text, lineList(lines)); if (c.hidden) { @@ -711,7 +715,7 @@ function cardLine(c) { var g = gate(c); return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' - : g === 'show' ? (c.headline || (c.facts && c.facts[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)) + : g === 'show' ? (c.headline || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag diff --git a/docs/live-site.md b/docs/live-site.md index 0f6935941..4634555ab 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, and a reload closes it again. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, or a request that failed), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 54431709f..2fa60be1a 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -57,8 +57,9 @@ result (the compound-het pair in KCNH2 of the fixture, and a de novo candidate and a stretch painted from a grandparent planted in family.json) name its gene, positions and that grandparent in no row, link, title or - count until "Show the result" is clicked there, the click brings each row - back, and a reload closes them again; with the son's results.json + count until "Show the result" is clicked there, a row that names both + planted results stays closed until both are opened (one per click), each + row then comes back, and a reload closes them again; with the son's results.json unavailable every row of his that could name one stays closed; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out of their drawings until it is opened; DEMO-001's page @@ -771,19 +772,28 @@ def family_closed(b, base): def family_problems(b, base): - """[problems] of the family page: closed, then opened with one click, then reloaded.""" + """[problems] of the family page: closed, then opened one result per click, then reloaded. Each planted row + names both of the son's planted results (the secondary finding's position, and the deletion's span around it, + both in KCNH2), so the first click opens one result and every row stays closed for the other.""" gene, hit, _ = planted_words() out = family_closed(b, base) if out: return out - b.eval("document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true") + click = "document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true" + b.eval(click) + time.sleep(0.5) + shown = b.eval(FAMILY_SHOWN_JS) or "" + left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + if any(w in shown for w in [gene, GRANDPARENT, PAIR_B] + hit) or left != 3: + out.append(f"one click opened a row that also names the other closed result ({left} closed boxes left)") + b.eval(click) time.sleep(0.5) shown = b.eval(FAMILY_SHOWN_JS) or "" rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") if not all(w in shown for w in [gene, GRANDPARENT, hit[0]]) or json.loads(rows or "[0,0,0]")[1:] != [3, 3] or left: - out.append(f"one click on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") + out.append(f"two clicks on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") out += [f"after a reload: {x}" for x in family_closed(b, base)] return out @@ -835,9 +845,10 @@ def write(p, text): bad = family_problems(b, base) check("browser: on the family page the son's rows that name his planted result (a compound-het pair, a de novo " "candidate, a stretch from a grandparent) name its gene, positions and grandparent nowhere until 'Show the " - "result' is clicked there, one click brings each row back, and a reload closes them", not bad, bad) + "result' is clicked there, a row naming two closed results stays closed until both are opened, then each " + "row comes back, and a reload closes them", not bad, bad) for p, a, z, desc in ( - (fp, "function isClosed(child, tag) { return !!tag && ", "function isClosed(child, tag) { return false && ", + (fp, "return tags.filter(function (t) {", "return [].filter(function (t) {", "family-page.js whose rows ignore the veil"), (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): if a not in keep[p]: @@ -853,6 +864,14 @@ def write(p, text): os.replace(rj + ".away", rj) check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " "a result to act on closed", not bad, bad) + # The results route answers a summary it cannot build with a note and no findings (owngenome_live_results.py). + with open(rj) as f: + keep[rj] = f.read() + write(rj, json.dumps({"schema_version": 1, "sample": OTHERS[1], "chapters": [], + "note": "The results could not be read (KeyError)."})) + bad = family_unknown(b, base) + check("browser: with the son's results answered as unreadable (a note, no findings) the family page keeps every " + "row of his that could name a result to act on closed", not bad, bad) finally: keep.pop("planted", None) for p, text in keep.items(): diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 7a403b68c..becc5e182 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -52,7 +52,7 @@ and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, 8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row, and -veilId() with its gene rule or its overlap rule dropped. A check +veilId() with its gene rule or its overlap rule dropped and veilIds() keeping only the first result. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -452,11 +452,12 @@ def widget_problems(): "const doc={documentElement:{setAttribute(){}},addEventListener(){}};" "vm.runInNewContext(fs.readFileSync(process.argv[1],'utf8'),{window:w,document:doc});" "const sp=JSON.parse(process.argv[2]);" - "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3]))))") + "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>[w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3])," + "w.PGPUI.veilIds(sp,r[0],r[1],r[2],r[3])])))") def node_veil(js_path): - """veilId() of core.js at JS_PATH on VEIL_ROWS, under node: [ids] or the error.""" + """[veilId(), veilIds()] of core.js at JS_PATH for each of VEIL_ROWS, under node, or the error.""" node = shutil.which("node") if not node: return "no node" @@ -466,12 +467,14 @@ def node_veil(js_path): def veil_js_problems(js_path): - """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS.""" - want = [live_results.veil_id([tuple(x) for x in VEIL_SPANS], *r) for r in VEIL_ROWS] + """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS, and veilIds() from veil_id() taken + one closed result at a time (every result the row belongs to).""" + spans = [tuple(x) for x in VEIL_SPANS] + want = [[live_results.veil_id(spans, *r), [s_[0] for s_ in spans if live_results.veil_id([s_], *r)]] for r in VEIL_ROWS] got = node_veil(js_path) if got != want: - return [f"veilId {got} != veil_id {want}"] - if len({w for w in want}) < 4: + return [f"veilId, veilIds {got} != veil_id {want}"] + if len({w[0] for w in want}) < 4 or not any(len(w[1]) > 1 for w in want): return [f"the rows exercise too few answers: {want}"] return [] @@ -598,11 +601,14 @@ def tagged(sample_dir): "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) check("8. node is on the PATH (ubuntu-latest has it)", shutil.which("node") is not None) bad = veil_js_problems(CORE_JS) - check("8. core.js veilId() tags each family-page row as veil_id() does (under node)", not bad, bad) + check("8. core.js veilId() tags each family-page row as veil_id() does, and veilIds() with every result " + "veil_id() finds one result at a time (under node)", not bad, bad) with open(CORE_JS) as f: core = f.read() for a, z, desc in (("if (s[1] && words[s[1]]) { return s[0]; }", "", "veilId() without its gene rule"), - ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule")): + ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule"), + (".map(function (s) { return s[0]; });\n }", ".map(function (s) { return s[0]; }).slice(0, 1);\n }", + "veilIds() keeping only the first result")): planted = os.path.join(work, "core-planted.js") with open(planted, "w") as f: f.write(core.replace(a, z)) From 40c41109ae6653c3276944cb058d7aff21e27a23 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:38:48 +0100 Subject: [PATCH 05/53] fix(live): close position-only family rows with a gene-only result, and open the map alone A closed result known by its gene alone (two changes in one gene) could sit at any de novo candidate or painted stretch, which name no gene, so those rows stay closed with it. 'Show the map' now opens the drawn map only, not the child's results. Bead: pgp-iqon.24 --- assets/live/family-page.js | 22 ++++++---- docs/live-site.md | 2 +- tests/test_live_demo.py | 88 +++++++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 1253b4d59..99b5b3237 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -80,7 +80,7 @@ // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; - var kids = {}, opened = {}, openedAll = {}, last = null; + var kids = {}, opened = {}, openedAll = {}, mapShown = {}, last = null; function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { (f.state.children || []).forEach(function (c) { @@ -115,11 +115,16 @@ }); } // The closed results a row of CHILD belongs to: their ids, [UNKNOWN] when this page cannot tell, or []. - function tagOf(child, text, chrom, pos, end) { + // COORDS: the row has a position and no gene name (a de novo candidate, a stretch). A closed result known by its + // gene alone (two changes in one gene, with no position) could sit at that position, and only the gene would + // tell: such a row stays closed with every result of that kind too. + function tagOf(child, text, chrom, pos, end, coords) { var k = kids[child]; if (!k || k.status !== 'known') { return [UNKNOWN]; } if (!k.spans) { return k.acts.length ? [UNKNOWN] : []; } // results without the tags: cannot tell - return U.veilIds(k.spans, text, chrom, pos, end); + var ids = U.veilIds(k.spans, text, chrom, pos, end); + if (coords) { k.spans.forEach(function (s) { if (!s[2] && ids.indexOf(s[0]) < 0) { ids.push(s[0]); } }); } + return ids; } // The results of TAGS still closed for CHILD: a row stays closed while one of them is. function stillClosed(child, tags) { @@ -367,7 +372,7 @@ // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : []; + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; }); var rows = split.open; var lengths = f.chrom_lengths || []; @@ -458,10 +463,11 @@ // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); - var shut = stillClosed(c.child, all); - if (shut.length) { + var shut = stillClosed(c.child, all), mk = openKey(c.child, 'map'); + // "Show the map" opens the map only: each result keeps its own closed rows until it is opened. + if (shut.length && !mapShown[mk]) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + - 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[openKey(c.child, t)] = true; } }); redraw(); })); + 'it could show which copy the child got there.', 'Show the map', function () { mapShown[mk] = true; redraw(); })); } else { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } @@ -471,7 +477,7 @@ var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'De novo candidates')); if (dn.length) { - var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos); }); + var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos, null, true); }); add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], diff --git a/docs/live-site.md b/docs/live-site.md index 4634555ab..a06b94f3a 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 2fa60be1a..36e0f5559 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -60,7 +60,10 @@ count until "Show the result" is clicked there, a row that names both planted results stays closed until both are opened (one per click), each row then comes back, and a reload closes them again; with the son's results.json - unavailable every row of his that could name one stays closed; on his page + unavailable, or answered as unreadable, every row of his that could name one + stays closed; with a closed result known by its gene alone his de novo rows + (a position, no gene) stay closed with it; his map as an image stays closed, + and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out of their drawings until it is opened; DEMO-001's page mid-replay and at the end leads with the report (a short run band above @@ -74,7 +77,8 @@ without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil -and core.js whose veilId() tags nothing; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +core.js whose veilId() tags nothing, family-page.js without the gene-only fallback and a "Show the map" that opens +the results too; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -812,6 +816,52 @@ def family_unknown(b, base): return out +def son_tables(b): + """[rows of each table] in the son's fold of the family page.""" + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + return json.loads(rows) if rows else None + + +def gene_only_problems(b, base): + """[problems] of the family page when the son also has a closed result known by its gene alone: a de novo row + shown before it is opened, a pair in another gene closed, or the rows not back once each result is opened.""" + family_page(b, base) + out = [] + t = son_tables(b) + shown = b.eval(FAMILY_SHOWN_JS) or "" + # Open: the stretches' table and the two pairs in other genes; closed: both de novo rows. + if t != [80, 2] or "48211904" in shown: + out.append(f"before any click: tables {t}") + for _ in range(3): # the gene-only result, then the two planted ones (the pair in KCNH2) + b.eval("var x = document.querySelector('details[data-child=\"DEMO-003\"] .veil button'); x && x.click(); true") + time.sleep(0.4) + t = son_tables(b) + if t != [80, 2, 3]: + out.append(f"after opening each result: tables {t}") + return out + + +def map_problems(b, base): + """[problems] of the family page whose son has his map as an image: the image shown while a result is closed, + or 'Show the map' opening a result's rows.""" + family_page(b, base) + out = [] + img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") + if img: + out.append("the map shown while a result is closed") + b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil'), " + "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; v && v.querySelector('button').click(); true") + time.sleep(0.4) + img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") + shown = b.eval(FAMILY_SHOWN_JS) or "" + if not img: + out.append("'Show the map' did not show the map") + if "KCNH2" in shown or PAIR_B in shown: + out.append("'Show the map' opened the closed pair too") + return out + + def family_veil_checks(b, base, out): """On the family page a child's rows that name a closed result stay closed as on the child's page, until that result is opened there; a page that cannot read the child's results keeps every row of his that could closed.""" @@ -872,6 +922,40 @@ def write(p, text): bad = family_unknown(b, base) check("browser: with the son's results answered as unreadable (a note, no findings) the family page keeps every " "row of his that could name a result to act on closed", not bad, bad) + + # A closed result known by its gene alone: a row with only a position (a de novo candidate) could be it. + res = json.loads(keep[rj]) + res["veil"]["spans"].append(["two-hits-g0999", "G0999", None, None, None]) + write(rj, json.dumps(res)) + write(fj, keep[fj]) # the fixture's family: two de novo rows, three pairs (one of them in KCNH2) + bad = gene_only_problems(b, base) + check("browser: with a closed result known by its gene alone, the son's de novo rows (a position, no gene) stay " + "closed until it is opened, and his pairs in other genes stay open", not bad, bad) + a = "if (coords) {" + if a not in keep[fp]: + check("control: family-page.js without the gene-only fallback (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "if (false) {")) + control("family-page.js without the gene-only fallback", not gene_only_problems(b, base)) + write(fp, keep[fp]) + write(rj, keep[rj]) + + # The drawing as an image (no stretches in family.json): "Show the map" opens the map and no result. + fam = json.loads(keep[fj]) + fam["segments"] = {} + fam["state"]["children"][0]["segments"] = fam["state"]["pedigree"] + write(fj, json.dumps(fam)) + bad = map_problems(b, base) + check("browser: the son's map as an image stays closed while a result is, and 'Show the map' opens the map " + "and leaves each result's rows closed", not bad, bad) + a = "function () { mapShown[mk] = true; redraw(); }" + if a not in keep[fp]: + check("control: a 'Show the map' that opens the results too (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "function () { mapShown[mk] = true; shut.forEach(function (t) { " + "opened[openKey(c.child, t)] = true; }); redraw(); }")) + control("a 'Show the map' that opens the results too", not map_problems(b, base)) + write(fp, keep[fp]) finally: keep.pop("planted", None) for p, text in keep.items(): From 996de9d7dc485876302fd370a50acb43456ac650 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:42:29 +0100 Subject: [PATCH 06/53] fix(live): close a child's side-by-side rows and the grandparents of the painting with a closed result The side by side showed a child's CPIC row (RYR1, CACNA1S) for a gene whose result is closed on the child's page, and the painting drew a closed stretch hatched at its own place, which pointed at the result. A side-by-side row that names a child's closed result is now left out until opened there (every medicines and HLA row while the child's results cannot be read), and while a grandparent stretch lies over a closed result every stretch of that child is drawn from its parent alone. --- CHANGELOG.md | 2 +- assets/live/family-page.js | 59 +++++++++++++++++------ assets/live/live.css | 1 - docs/live-site.md | 2 +- tests/test_live_demo.py | 75 +++++++++++++++++++++++++---- tests/test_owngenome_live_family.py | 6 +++ 6 files changed, 118 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83143e76b..69de5401e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a stretch painted from a grandparent over one, stay behind "Show the result" until that result is opened there; a reload closes them again. While the page cannot read the child's results, every such row of that child stays closed. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. +- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a side-by-side row whose value in the child's column names one (a CPIC gene such as RYR1), stay behind "Show the result" until that result is opened there; a reload closes them again. While a stretch painted from a grandparent lies over a closed result, the child's painting shows every stretch from its parent alone. While the page cannot read the child's results, every such row of that child stays closed (in the side by side, every medicines and HLA row). On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. - Step 49 (clonal haematopoiesis) now prepares the CHIP gene models and writes its two tables with the Rust `og` binary: `og chip prepare`, `og chip report` and `og chip check-table` replace the commands of `bin/chip_report.py`, which keeps only the detection limit and the column names the reports import. On Ensembl 116's full GFF3, prepare takes 1.7 s and 6 MB instead of 15 s and 20 MB. The two tables, `chip.gff3` and `chip.bed` are the same, byte for byte. What changes: a missing input file or a position that is not a whole number stops with an `ERROR:` line and exit 1, not a Python traceback, and each output is written whole or not at all. Step 49 now needs `og` (`scripts/setup.sh --og`). - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 99b5b3237..2bbef5a13 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -8,7 +8,8 @@ // the drawing through createElementNS; images from this server only. // A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json // gives the closed results (veil.spans), and a row of the child that names one (a de novo or compound-het -// candidate, a stretch painted from a grandparent) stays closed until that result is opened on purpose here. +// candidate, a side-by-side row) stays closed until that result is opened on purpose here; while a stretch +// painted from a grandparent lies over one, the child's painting shows each stretch's parent only. (function () { 'use strict'; var U = window.PGPUI; @@ -80,6 +81,7 @@ // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; + var GENE_GROUPS = { Medicines: true, HLA: true }; // the side-by-side groups whose rows name a gene var kids = {}, opened = {}, openedAll = {}, mapShown = {}, last = null; function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { @@ -370,11 +372,18 @@ var SIDE = { paternal: 'father', maternal: 'mother' }; function painting(f, child, all) { // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a - // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. + // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only that + // stretch would point at the closed result (a hatch, a gap in the table), so while one is closed every stretch + // of this child is drawn from its parent alone, and no grandparent, basis or closed mark is shown. var split = veiled(child, all, function (r) { return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; }); - var rows = split.open; + var shut = []; + split.closed.forEach(function (t) { t.forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); + var rows = !shut.length ? all : all.map(function (r) { + return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, + discordant: r.discordant, ancestor: '.', basis: 'parent' }; + }); var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -393,14 +402,6 @@ ['paternal', 'maternal'].forEach(function (side) { var track = el('span', { 'class': 'paint-track ' + side }); track.style.setProperty('--len', (100 * cl[1] / longest).toFixed(2) + '%'); - all.filter(function (r) { return r.chrom === chrom && r.side === side && rows.indexOf(r) < 0; }).forEach(function (r) { - var a = parseInt(r.start, 10), b = parseInt(r.end, 10); - if (!isFinite(a) || !isFinite(b) || b < a) { return; } - var seg = el('span', { 'class': 'paint-seg closed', title: 'A closed stretch' }); - seg.style.setProperty('--a', (100 * a / cl[1]).toFixed(3) + '%'); - seg.style.setProperty('--w', Math.max(0.4, 100 * (b - a) / cl[1]).toFixed(3) + '%'); - add(track, seg); - }); rows.filter(function (r) { return r.chrom === chrom && r.side === side; }).forEach(function (r) { var a = parseInt(r.start, 10), b = parseInt(r.end, 10); if (!isFinite(a) || !isFinite(b) || b < a) { return; } @@ -425,10 +426,18 @@ who + (colours[who].basis === 'other copy' ? ' (the copy not from the sequenced parent)' : '')); add(leg, it); }); - add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).' + - (split.closed.length ? ' Hatched: a closed stretch.' : ''))); + add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); add(fig, leg); - add(fig, veilBox(child, split.closed, 'stretch', 'stretches')); + if (shut.length) { + // One box for the whole painting: a count of closed stretches would say how many sit over the result. + add(fig, shut[0] === UNKNOWN + ? U.veil('The grandparents of ' + child + '’s stretches stay hidden: this page cannot read ' + child + '’s results yet, ' + + 'so it cannot tell which stretch lies over a result to act on in adult life. Each stretch shows its parent only.', + 'Show the grandparents', function () { openedAll[child] = true; redraw(); }) + : U.veil('The grandparents of ' + child + '’s stretches stay hidden while a result to act on in adult life is ' + + 'closed: they could show which copy the child got there. Each stretch shows its parent only.', null, + function () { opened[openKey(child, shut[0])] = true; redraw(); })); + } var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -510,9 +519,25 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); + // A child's value that names a closed result (a CPIC gene that is also the gene of a result to act on, RYR1 or + // CACNA1S) closes its whole row, as the child's own page closes that row of its medicines card: a closed cell + // under the gene's name would say which gene it is. While this page cannot read a child's results, every row + // that names a gene (the medicines and HLA rows) stays closed; the haplogroups and blood groups name none. + var kidsHere = (f.state.children || []).map(function (c) { return c.child; }); + var closedBy = {}, shown = []; + sb.rows.forEach(function (r) { + var open = true; + sb.members.forEach(function (m, i) { + if (kidsHere.indexOf(m) < 0) { return; } + var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || ''))); + if (t[0] === UNKNOWN && !GENE_GROUPS[r.group]) { return; } + if (t.length) { (closedBy[m] = closedBy[m] || []).push(t); open = false; } + }); + if (open) { shown.push(r); } + }); // A value from a section older than its input (state "stale") says so in its own cell. var group = null, rows = [], old = 0; - sb.rows.forEach(function (r) { + shown.forEach(function (r) { rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { if (!(r.stale || [])[i]) { return v; } old++; @@ -521,6 +546,10 @@ group = r.group; }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); + sb.members.forEach(function (m) { + var box = closedBy[m] && veilBox(m, closedBy[m], 'row', 'rows'); + if (box) { box.setAttribute('data-veil-child', m); add(s, box); } + }); if (old) { add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + 'earlier run, and the latest run did not write it again; that member’s page says why.')); diff --git a/assets/live/live.css b/assets/live/live.css index 4adff6602..5dba9f176 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -607,7 +607,6 @@ ul.rel-pairs .chip { white-space: normal; } .anc1 { background: var(--anc1); } .anc2 { background: var(--anc2); } .anc3 { background: var(--anc3); } .anc4 { background: var(--anc4); } .anc5 { background: var(--anc5); } .paint-seg.other, .paint-swatch.other { background-image: repeating-linear-gradient(45deg, transparent 0 2px, rgba(0, 0, 0, .35) 2px 4px); } -.paint-seg.closed { background: repeating-linear-gradient(-45deg, rgba(143, 141, 134, .45) 0 2px, transparent 2px 6px); } .paint-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; font-size: 12.5px; margin-top: 8px; } .paint-key { display: inline-flex; gap: 6px; align-items: center; } .paint-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 3px; } diff --git a/docs/live-site.md b/docs/live-site.md index a06b94f3a..d2f6ecf85 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one stays closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. A stretch painted from a grandparent over one says which grandparent's copy the child got there: while such a stretch exists, every stretch of that child is drawn and listed from its parent alone, with no grandparent, so the painting does not point at where the result lies, and "Show the result" brings the grandparents back. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. A row whose value in a child's column names a closed result of that child (a CPIC gene such as RYR1 that is also the gene of a result to act on) is left out whole, for every member, until that result is opened here: a closed cell under the gene's name would say which gene it is. While the page cannot read a child's results, every medicines and HLA row stays closed; the haplogroups and blood groups name no gene. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 36e0f5559..db9b5f30b 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -59,9 +59,12 @@ name its gene, positions and that grandparent in no row, link, title or count until "Show the result" is clicked there, a row that names both planted results stays closed until both are opened (one per click), each - row then comes back, and a reload closes them again; with the son's results.json + row then comes back, and a reload closes them again; while a stretch painted + from a grandparent lies over one, every stretch of his is drawn and listed from + its parent alone, none marked; with a closed result of his in RYR1 the side by + side's RYR1 row stays closed until opened there; with the son's results.json unavailable, or answered as unreadable, every row of his that could name one - stays closed; with a closed result known by its gene alone his de novo rows + stays closed (in the side by side every medicines and HLA row); with a closed result known by its gene alone his de novo rows (a position, no gene) stay closed with it; his map as an image stays closed, and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out @@ -77,8 +80,8 @@ without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil -core.js whose veilId() tags nothing, family-page.js without the gene-only fallback and a "Show the map" that opens -the results too; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +core.js whose veilId() tags nothing, family-page.js without the gene-only fallback, a "Show the map" that opens +the results too, a closed painting that keeps each stretch's grandparent and a side by side that ignores the veil; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -768,10 +771,14 @@ def family_closed(b, base): rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") closed = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") - segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed').length") - # The stretches' table, the 2 de novo rows of the fixture and its 2 other pairs stay; one closed box each. - if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or segs != 1: - out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} closed stretches") + segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length") + marked = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed, " + "[data-painting=\"DEMO-003\"] .paint-seg.other').length") + # The 2 de novo rows of the fixture and its 2 other pairs stay, one closed box each; the painting closes whole + # (one box): every stretch drawn and listed from its parent, none marked, so none points at the closed result. + if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or json.loads(rows)[0] != segs or marked: + out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} stretches drawn, " + f"{marked} marked") return out @@ -813,6 +820,38 @@ def family_unknown(b, base): "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") if rows is None or len(json.loads(rows)) != 1: out.append(f"the son's de novo or compound-het rows open with his results unavailable: tables {rows}") + side = side_labels(b) + if any(g in side for g in SIDE_GENES) or "Mitochondrial" not in side: + out.append(f"side by side with the son's results unavailable: rows {sorted(side)[:8]}") + return out + + +SIDE_GENES = ("RYR1", "CACNA1S", "CYP2D6", "HLA-A") + + +def side_labels(b): + """The 'Gene or system' column of the family page's side by side.""" + got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " + "function (r) { return r.children[1] ? r.children[1].textContent : ''; }))") + return json.loads(got) if got else [] + + +def side_problems(b, base): + """[problems] of the family page's side by side with a closed result of the son's in RYR1: the RYR1 row (his + diplotype and phenotype under the gene's name) shown before it is opened there, the other rows closed with it, + or the row not back once opened.""" + family_page(b, base) + out = [] + labels = side_labels(b) + shown = b.eval("document.getElementById('ch-side-by-side').textContent") or "" + boxes = b.eval("document.querySelectorAll('#ch-side-by-side .veil[data-veil-child=\"DEMO-003\"]').length") + if "RYR1" in shown or "CACNA1S" not in labels or boxes != 1: + out.append(f"before any click: RYR1 {'shown' if 'RYR1' in shown else 'closed'}, {len(labels)} rows, {boxes} closed boxes") + b.eval("var x = document.querySelector('#ch-side-by-side .veil button'); x && x.click(); true") + time.sleep(0.4) + labels2 = side_labels(b) + if "RYR1" not in labels2 or len(labels2) != len(labels) + 1: + out.append(f"after 'Show the result': {len(labels2)} rows, RYR1 {'back' if 'RYR1' in labels2 else 'still closed'}") return out @@ -900,7 +939,9 @@ def write(p, text): for p, a, z, desc in ( (fp, "return tags.filter(function (t) {", "return [].filter(function (t) {", "family-page.js whose rows ignore the veil"), - (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): + (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing"), + (fp, "ancestor: '.', basis: 'parent' };", "ancestor: r.ancestor, basis: r.basis };", + "a closed painting that keeps each stretch's grandparent")): if a not in keep[p]: check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) continue @@ -940,6 +981,22 @@ def write(p, text): write(fp, keep[fp]) write(rj, keep[rj]) + # A closed result in a CPIC gene: the side by side lists that gene with the son's diplotype in his column. + res = json.loads(keep[rj]) + res["veil"]["spans"].append(["sf-ryr1-demo", "RYR1", "chr19", 38440000, 38440000]) + write(rj, json.dumps(res)) + bad = side_problems(b, base) + check("browser: with a closed result of the son's in RYR1, the side by side's RYR1 row (his diplotype and " + "phenotype) stays closed until it is opened there, and the other rows stay", not bad, bad) + a = "var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || '')));" + if a not in keep[fp]: + check("control: a side by side that ignores the veil (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "var t = [];")) + control("a side by side that ignores the veil", not side_problems(b, base)) + write(fp, keep[fp]) + write(rj, keep[rj]) + # The drawing as an image (no stretches in family.json): "Show the map" opens the map and no result. fam = json.loads(keep[fj]) fam["segments"] = {} diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 8cb7c8678..5965c1045 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -349,6 +349,12 @@ def browser_checks(port, work): couple and "1 gene(s) where both carry a finding, 1 X-linked" in couple and b.eval("!!document.querySelector('p[data-couple=\\'father_a mother_a\\'] a')"), couple) b.eval("document.querySelector('details[data-child=child_a]').open = true; true") + # child_a's results cannot be built here (its summary is cut short), so the page cannot tell whether a + # result to act on is closed: the map stays closed until "Show the map" is clicked. + shut = b.wait("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=child_a] .veil'), " + "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; " + "v && !document.querySelector('details[data-child=child_a] img') && (v.querySelector('button').click(), true)", 10) + check("browser: child_a's chromosome map stays closed while its results cannot be read", shut, shut) seg = b.wait("var i = document.querySelector('details[data-child=child_a] img'); " "i && i.complete && i.naturalWidth > 0 && i.getAttribute('src')", 10) check("browser: child_a's chromosome map loads", seg == "/family/family_a/child_a_segments.svg", seg) From 0bb490670b33a768a9b83e25125ff7a951e0000a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:48:27 +0100 Subject: [PATCH 07/53] fix(live): open a child's drawing only by its own button while the page cannot read the child's results 'Show the rows' of a table opened the child's map as an image, and the painting's grandparents, too. --- assets/live/family-page.js | 18 +++++++++----- tests/test_live_demo.py | 49 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 2bbef5a13..174352935 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -132,6 +132,11 @@ function stillClosed(child, tags) { return tags.filter(function (t) { return t === UNKNOWN ? !openedAll[child] : !opened[openKey(child, t)]; }); } + // The same for a whole drawing of CHILD (its painting, its map as an image): while this page cannot read the + // child's results, only the drawing's own button (KEY) opens it, never "Show the rows" of a table. + function drawingClosed(child, tags, key) { + return tags.filter(function (t) { return t === UNKNOWN ? !mapShown[openKey(child, key)] : !opened[openKey(child, t)]; }); + } // The rows of CHILD split by the veil: { open: rows, closed: [the still closed results of each closed row] }. function veiled(child, rows, tagFn) { var out = { open: [], closed: [] }; @@ -375,11 +380,12 @@ // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only that // stretch would point at the closed result (a hatch, a gap in the table), so while one is closed every stretch // of this child is drawn from its parent alone, and no grandparent, basis or closed mark is shown. - var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; + var shut = [], pk = openKey(child, 'paint'); + all.forEach(function (r) { + if (!r.ancestor || r.ancestor === '.') { return; } + drawingClosed(child, tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true), 'paint') + .forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); - var shut = []; - split.closed.forEach(function (t) { t.forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); var rows = !shut.length ? all : all.map(function (r) { return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, discordant: r.discordant, ancestor: '.', basis: 'parent' }; @@ -433,7 +439,7 @@ add(fig, shut[0] === UNKNOWN ? U.veil('The grandparents of ' + child + '’s stretches stay hidden: this page cannot read ' + child + '’s results yet, ' + 'so it cannot tell which stretch lies over a result to act on in adult life. Each stretch shows its parent only.', - 'Show the grandparents', function () { openedAll[child] = true; redraw(); }) + 'Show the grandparents', function () { mapShown[pk] = true; redraw(); }) : U.veil('The grandparents of ' + child + '’s stretches stay hidden while a result to act on in adult life is ' + 'closed: they could show which copy the child got there. Each stretch shows its parent only.', null, function () { opened[openKey(child, shut[0])] = true; redraw(); })); @@ -472,7 +478,7 @@ // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); - var shut = stillClosed(c.child, all), mk = openKey(c.child, 'map'); + var shut = drawingClosed(c.child, all, 'map'), mk = openKey(c.child, 'map'); // "Show the map" opens the map only: each result keeps its own closed rows until it is opened. if (shut.length && !mapShown[mk]) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index db9b5f30b..968eeaaa9 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -64,7 +64,9 @@ its parent alone, none marked; with a closed result of his in RYR1 the side by side's RYR1 row stays closed until opened there; with the son's results.json unavailable, or answered as unreadable, every row of his that could name one - stays closed (in the side by side every medicines and HLA row); with a closed result known by its gene alone his de novo rows + stays closed (in the side by side every medicines and HLA row), and "Show the + rows" opens neither his painting's grandparents nor his map as an image, which + open only by their own button; with a closed result known by its gene alone his de novo rows (a position, no gene) stay closed with it; his map as an image stays closed, and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out @@ -81,7 +83,8 @@ rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil core.js whose veilId() tags nothing, family-page.js without the gene-only fallback, a "Show the map" that opens -the results too, a closed painting that keeps each stretch's grandparent and a side by side that ignores the veil; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +the results too, a closed painting that keeps each stretch's grandparent, a side by side that ignores the veil and a +drawing that "Show the rows" opens; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -829,6 +832,33 @@ def family_unknown(b, base): SIDE_GENES = ("RYR1", "CACNA1S", "CYP2D6", "HLA-A") +def click_veil(b, words): + """Click the son's closed box whose button says WORDS; whether there was one.""" + return b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil " + "button'), function (x) { return x.textContent === " + json.dumps(words) + "; })[0]; " + "if (v) { v.click(); } !!v") + + +def unknown_drawing_problems(b, base, image): + """[problems] of the family page with the son's results unavailable: "Show the rows" of a table opening his + drawing (the map as an image, or the painting's grandparents), or the drawing's own button not opening it.""" + family_page(b, base) + out = [] + if not click_veil(b, "Show the rows"): + out.append("no 'Show the rows' box") + time.sleep(0.4) + img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" + seen = (lambda: b.eval(img)) if image else (lambda: GRANDPARENT in (b.eval(FAMILY_SHOWN_JS) or "")) + if seen(): + out.append("'Show the rows' opened the drawing too") + if not click_veil(b, "Show the map" if image else "Show the grandparents"): + out.append("no closed box of the drawing") + time.sleep(0.4) + if not seen(): + out.append("the drawing's own button did not open it") + return out + + def side_labels(b): """The 'Gene or system' column of the family page's side by side.""" got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " @@ -951,6 +981,7 @@ def write(p, text): os.replace(rj, rj + ".away") try: bad = family_unknown(b, base) + bad += unknown_drawing_problems(b, base, False) finally: os.replace(rj + ".away", rj) check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " @@ -1005,6 +1036,20 @@ def write(p, text): bad = map_problems(b, base) check("browser: the son's map as an image stays closed while a result is, and 'Show the map' opens the map " "and leaves each result's rows closed", not bad, bad) + os.replace(rj, rj + ".away") + try: + bad = unknown_drawing_problems(b, base, True) + check("browser: with the son's results unavailable, 'Show the rows' leaves his map as an image closed, and " + "'Show the map' opens it", not bad, bad) + a = "return t === UNKNOWN ? !mapShown[openKey(child, key)] :" + if a not in keep[fp]: + check("control: a drawing that 'Show the rows' opens (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "return t === UNKNOWN ? !mapShown[openKey(child, key)] && !openedAll[child] :")) + control("a drawing that 'Show the rows' opens", not unknown_drawing_problems(b, base, True)) + write(fp, keep[fp]) + finally: + os.replace(rj + ".away", rj) a = "function () { mapShown[mk] = true; redraw(); }" if a not in keep[fp]: check("control: a 'Show the map' that opens the results too (the planted line is in family-page.js)", False, a) From 0bc60efda244e5361c54fe411f46db9514e5e8c3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:11:38 +0100 Subject: [PATCH 08/53] feat(og): og cn, the copy-number engine in Rust, checked against the Python og cn bed, call, intervals, check-bam, marker-regions and marker-read do what bin/copy_number.py, bin/hba_copy_number.py, bin/lpa_kiv2.py and the marker half of bin/traits_report.py did. 74 parity cases compare them with the Python at 9885cc5; the goldens are the deliberate fixes: one SMN1 exon-7 rule, the LPA 4925G>A null allele judged on the raw MAPQ 0 share, and messages that name og cn bed. Bead: pgp-bs9u.24 --- rust/Cargo.lock | 11 + rust/Cargo.toml | 3 +- rust/crates/og-cn/Cargo.toml | 16 + rust/crates/og-cn/src/catalog.rs | 743 ++++++++++++++ rust/crates/og-cn/src/cli.rs | 703 +++++++++++++ rust/crates/og-cn/src/engine.rs | 658 ++++++++++++ rust/crates/og-cn/src/input.rs | 272 +++++ rust/crates/og-cn/src/layout.rs | 385 +++++++ rust/crates/og-cn/src/lib.rs | 164 +++ rust/crates/og-cn/src/markers.rs | 451 +++++++++ rust/crates/og-cn/src/smn.rs | 200 ++++ rust/og/Cargo.toml | 1 + rust/og/src/main.rs | 7 +- tests/parity/cases.tsv | 74 ++ tests/parity/gen/cn-oracle.sh | 13 + tests/parity/golden/cn/cyp-no-pileup/stderr | 1 + .../parity/golden/cn/hba-wrong-regions/stderr | 1 + .../golden/cn/intervals-wrong-table/stderr | 1 + .../cn/marker-lpa-share1996-kiv-present/n.tsv | 10 + .../marker-lpa-share1996-kiv-present/stdout | 2 + .../cn/smn-smn-present-smn-two-rows/m.tsv | 11 + .../cn/smn-smn-present-smn-two-rows/stdout | 1 + tests/parity/inputs/cn/bad.pileup | 1 + tests/parity/inputs/cn/ccl1.bedcov.tsv | 3 + tests/parity/inputs/cn/ccl1.q1.tsv | 3 + .../inputs/cn/cyp-conversion.bedcov.tsv | 10 + tests/parity/inputs/cn/cyp-conversion.pileup | 12 + tests/parity/inputs/cn/cyp-conversion.q1.tsv | 10 + .../parity/inputs/cn/cyp-deletion.bedcov.tsv | 10 + tests/parity/inputs/cn/cyp-deletion.pileup | 12 + tests/parity/inputs/cn/cyp-deletion.q1.tsv | 10 + tests/parity/inputs/cn/cyp-mapq0.q1.tsv | 10 + tests/parity/inputs/cn/cyp-normal.bedcov.tsv | 10 + tests/parity/inputs/cn/cyp-normal.pileup | 12 + tests/parity/inputs/cn/cyp-normal.q1.tsv | 10 + tests/parity/inputs/cn/empty.tsv | 0 tests/parity/inputs/cn/hba-between.bedcov.tsv | 4 + tests/parity/inputs/cn/hba-between.q1.tsv | 4 + tests/parity/inputs/cn/hba-broken.bedcov.tsv | 4 + tests/parity/inputs/cn/hba-broken.q1.tsv | 4 + .../parity/inputs/cn/hba-disagree.bedcov.tsv | 4 + tests/parity/inputs/cn/hba-disagree.q1.tsv | 4 + .../parity/inputs/cn/hba-lowflank.bedcov.tsv | 4 + tests/parity/inputs/cn/hba-lowflank.q1.tsv | 4 + tests/parity/inputs/cn/hba-mapq0.bedcov.tsv | 4 + tests/parity/inputs/cn/hba-mapq0.q1.tsv | 4 + tests/parity/inputs/cn/hba0.bedcov.tsv | 4 + tests/parity/inputs/cn/hba0.q1.tsv | 4 + tests/parity/inputs/cn/hba3.bedcov.tsv | 4 + tests/parity/inputs/cn/hba3.q1.tsv | 4 + tests/parity/inputs/cn/hba4.bedcov.tsv | 4 + tests/parity/inputs/cn/hba4.q1.tsv | 4 + tests/parity/inputs/cn/hba5.bedcov.tsv | 4 + tests/parity/inputs/cn/hba5.q1.tsv | 4 + tests/parity/inputs/cn/header-alt.sam | 5 + tests/parity/inputs/cn/header-clean.sam | 4 + tests/parity/inputs/cn/hp3.bedcov.tsv | 4 + tests/parity/inputs/cn/hp3.q1.tsv | 4 + tests/parity/inputs/cn/intervals.bedcov.tsv | 946 ++++++++++++++++++ tests/parity/inputs/cn/kiv-notseen.pileup | 7 + tests/parity/inputs/cn/kiv-present.pileup | 7 + tests/parity/inputs/cn/lce0.bedcov.tsv | 3 + tests/parity/inputs/cn/lce0.q1.tsv | 3 + tests/parity/inputs/cn/lpa-floor.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa-floor.q1.tsv | 29 + .../parity/inputs/cn/lpa-lowflank.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa-lowflank.q1.tsv | 29 + tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa-mapq0.q1.tsv | 29 + .../parity/inputs/cn/lpa-share1996.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa-share1996.q1.tsv | 29 + tests/parity/inputs/cn/lpa-table.tsv | 5 + tests/parity/inputs/cn/lpa12.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa12.q1.tsv | 29 + tests/parity/inputs/cn/lpa29.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa29.q1.tsv | 29 + tests/parity/inputs/cn/lpa47.bedcov.tsv | 29 + tests/parity/inputs/cn/lpa47.q1.tsv | 29 + tests/parity/inputs/cn/oracle.sh | 55 + tests/parity/inputs/cn/rhd0.bedcov.tsv | 13 + tests/parity/inputs/cn/rhd1.bedcov.tsv | 13 + tests/parity/inputs/cn/smn-lowq.tsv | 2 + tests/parity/inputs/cn/smn-notseen.pileup | 1 + tests/parity/inputs/cn/smn-present.pileup | 3 + tests/parity/inputs/cn/smn-two-rows.tsv | 3 + tests/parity/inputs/cn/smn1.tsv | 2 + tests/parity/inputs/cn/smn2.tsv | 2 + tests/parity/inputs/cn/stdin | 2 + tests/parity/oracle.ref | 1 + 89 files changed, 5402 insertions(+), 2 deletions(-) create mode 100644 rust/crates/og-cn/Cargo.toml create mode 100644 rust/crates/og-cn/src/catalog.rs create mode 100644 rust/crates/og-cn/src/cli.rs create mode 100644 rust/crates/og-cn/src/engine.rs create mode 100644 rust/crates/og-cn/src/input.rs create mode 100644 rust/crates/og-cn/src/layout.rs create mode 100644 rust/crates/og-cn/src/lib.rs create mode 100644 rust/crates/og-cn/src/markers.rs create mode 100644 rust/crates/og-cn/src/smn.rs create mode 100644 tests/parity/gen/cn-oracle.sh create mode 100644 tests/parity/golden/cn/cyp-no-pileup/stderr create mode 100644 tests/parity/golden/cn/hba-wrong-regions/stderr create mode 100644 tests/parity/golden/cn/intervals-wrong-table/stderr create mode 100644 tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv create mode 100644 tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout create mode 100644 tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv create mode 100644 tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout create mode 100644 tests/parity/inputs/cn/bad.pileup create mode 100644 tests/parity/inputs/cn/ccl1.bedcov.tsv create mode 100644 tests/parity/inputs/cn/ccl1.q1.tsv create mode 100644 tests/parity/inputs/cn/cyp-conversion.bedcov.tsv create mode 100644 tests/parity/inputs/cn/cyp-conversion.pileup create mode 100644 tests/parity/inputs/cn/cyp-conversion.q1.tsv create mode 100644 tests/parity/inputs/cn/cyp-deletion.bedcov.tsv create mode 100644 tests/parity/inputs/cn/cyp-deletion.pileup create mode 100644 tests/parity/inputs/cn/cyp-deletion.q1.tsv create mode 100644 tests/parity/inputs/cn/cyp-mapq0.q1.tsv create mode 100644 tests/parity/inputs/cn/cyp-normal.bedcov.tsv create mode 100644 tests/parity/inputs/cn/cyp-normal.pileup create mode 100644 tests/parity/inputs/cn/cyp-normal.q1.tsv create mode 100644 tests/parity/inputs/cn/empty.tsv create mode 100644 tests/parity/inputs/cn/hba-between.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba-between.q1.tsv create mode 100644 tests/parity/inputs/cn/hba-broken.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba-broken.q1.tsv create mode 100644 tests/parity/inputs/cn/hba-disagree.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba-disagree.q1.tsv create mode 100644 tests/parity/inputs/cn/hba-lowflank.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba-lowflank.q1.tsv create mode 100644 tests/parity/inputs/cn/hba-mapq0.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba-mapq0.q1.tsv create mode 100644 tests/parity/inputs/cn/hba0.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba0.q1.tsv create mode 100644 tests/parity/inputs/cn/hba3.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba3.q1.tsv create mode 100644 tests/parity/inputs/cn/hba4.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba4.q1.tsv create mode 100644 tests/parity/inputs/cn/hba5.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hba5.q1.tsv create mode 100644 tests/parity/inputs/cn/header-alt.sam create mode 100644 tests/parity/inputs/cn/header-clean.sam create mode 100644 tests/parity/inputs/cn/hp3.bedcov.tsv create mode 100644 tests/parity/inputs/cn/hp3.q1.tsv create mode 100644 tests/parity/inputs/cn/intervals.bedcov.tsv create mode 100644 tests/parity/inputs/cn/kiv-notseen.pileup create mode 100644 tests/parity/inputs/cn/kiv-present.pileup create mode 100644 tests/parity/inputs/cn/lce0.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lce0.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa-floor.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa-floor.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa-lowflank.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa-mapq0.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa-share1996.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa-share1996.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa-table.tsv create mode 100644 tests/parity/inputs/cn/lpa12.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa12.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa29.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa29.q1.tsv create mode 100644 tests/parity/inputs/cn/lpa47.bedcov.tsv create mode 100644 tests/parity/inputs/cn/lpa47.q1.tsv create mode 100644 tests/parity/inputs/cn/oracle.sh create mode 100644 tests/parity/inputs/cn/rhd0.bedcov.tsv create mode 100644 tests/parity/inputs/cn/rhd1.bedcov.tsv create mode 100644 tests/parity/inputs/cn/smn-lowq.tsv create mode 100644 tests/parity/inputs/cn/smn-notseen.pileup create mode 100644 tests/parity/inputs/cn/smn-present.pileup create mode 100644 tests/parity/inputs/cn/smn-two-rows.tsv create mode 100644 tests/parity/inputs/cn/smn1.tsv create mode 100644 tests/parity/inputs/cn/smn2.tsv create mode 100644 tests/parity/inputs/cn/stdin diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 11c5e36b1..7bd8be4c5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -554,6 +554,7 @@ version = "0.1.0" dependencies = [ "og-bam", "og-chip", + "og-cn", "og-dev", "og-filters", "og-intervals", @@ -586,6 +587,16 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-cn" +version = "0.1.0" +dependencies = [ + "indexmap", + "og-filters", + "og-io", + "pycompat", +] + [[package]] name = "og-dev" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c9c55d60f..36fb4f0db 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "crates/og-cn", "og"] [workspace.package] version = "0.1.0" @@ -22,6 +22,7 @@ og-nf = { path = "crates/og-nf" } og-runlog = { path = "crates/og-runlog" } og-steps = { path = "crates/og-steps" } og-chip = { path = "crates/og-chip" } +og-cn = { path = "crates/og-cn" } indexmap = "2" flate2 = "1" memchr = "2" diff --git a/rust/crates/og-cn/Cargo.toml b/rust/crates/og-cn/Cargo.toml new file mode 100644 index 000000000..c82cdc159 --- /dev/null +++ b/rust/crates/og-cn/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "og-cn" +description = "Copy number from read depth: og cn bed, call, intervals, check-bam, marker-regions, marker-read" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +indexmap.workspace = true +og-filters.workspace = true +og-io.workspace = true +pycompat.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-cn/src/catalog.rs b/rust/crates/og-cn/src/catalog.rs new file mode 100644 index 000000000..a9564a447 --- /dev/null +++ b/rust/crates/og-cn/src/catalog.rs @@ -0,0 +1,743 @@ +//! `bin/cn_loci.tsv`: one row per locus, read and checked once per process. +//! +//! The checks and their messages are those of `copy_number.py` +//! (`parse_row`, `load_catalog`), plus one it lacked: a `more=` reading +//! needs a numbered reading to count past (the Python failed on `max()` of +//! an empty sequence when a call reached it, naming no row). + +use crate::{Error, py_float, py_int}; +use indexmap::IndexMap; +use og_filters::py_repr; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +/// The catalog's columns, in order. +pub const COLUMNS: [&str; 22] = [ + "locus", + "kind", + "chrom", + "targets", + "controls", + "details", + "target_depth", + "control_depth", + "scale", + "offset", + "bedcov", + "mapq0_min", + "mapq0_max", + "min_control_depth", + "max_control_ratio", + "call", + "interval", + "readings", + "input", + "owner", + "cross_check", + "pmids", +]; + +/// The findings a `call psv` row must have a reading for. +pub const PSV_FINDINGS: [&str; 4] = ["none", "deletion", "conversion", "pseudogene_deletion"]; + +/// What a row counts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Gene, + Interval, + Organelle, +} + +/// Where a row's depths come from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Input { + Bedcov, + CnvpytorLevel, +} + +/// How the target segments combine into T. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetMode { + /// Added up: each segment is one reference copy. + Sum, + /// Length-weighted mean. + Pooled, +} + +/// How the control segments combine into C. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ControlMode { + /// Plain mean of the segments. + Flanks, + /// Length-weighted mean. + Pooled, +} + +/// The `call` column. +#[derive(Debug, Clone, PartialEq)] +pub enum CallRule { + /// The nearest whole number, when the estimate lies within `max_off` of it. + Round { max_off: f64 }, + /// The number of cut points the estimate reaches. + Cut(Vec), + /// low, typical or high by the 95% interval against two bounds. + Class { low: f64, high: f64 }, + /// The module total as `Round`, then the PSV sites. + Psv { max_off: f64 }, +} + +/// One `NAME:start-end[:label]` segment, 0-based half-open. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Segment { + pub name: String, + pub start: i64, + pub end: i64, + pub label: String, +} + +/// One paralogous sequence variant site of a `call psv` row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PsvSite { + pub name: String, + /// 0-based position on the gene. + pub gene_pos: i64, + /// 0-based position on the pseudogene. + pub pseudo_pos: i64, + /// The gene's bases there (`b"AC"` for In2G). + pub gene: Vec, + /// The pseudogene's bases there. + pub pseudo: Vec, + pub text: String, +} + +/// The depth-ratio settings of a `bedcov` row. +#[derive(Debug, Clone, PartialEq)] +pub struct Depth { + pub target_mode: TargetMode, + pub control_mode: ControlMode, + pub scale: f64, + pub offset: f64, + /// The `bedcov` passes, as written (`.` for the plain pass). + pub passes: Vec, + pub mapq0_min: Option, + pub mapq0_max: Option, + pub min_control: f64, + pub max_ratio: Option, + /// A 95% interval from the per-segment spread (`interval cv`). + pub ci_cv: bool, +} + +/// One catalog row. +#[derive(Debug, Clone, PartialEq)] +pub struct Locus { + pub id: String, + pub kind: Kind, + pub chrom: String, + pub input: Input, + pub targets: Vec, + pub controls: Vec, + pub details: Vec, + pub call: CallRule, + /// `KEY=text` readings, in the row's order. + pub readings: IndexMap, + pub psv: Vec, + /// `target_depth` as written (a `call psv` row needs `sum` whatever its input). + pub target_depth: String, + /// Present for a `bedcov` row. + pub depth: Option, +} + +impl Locus { + /// The bedcov settings; every row `og cn` counts has them. + pub fn depth(&self) -> &Depth { + self.depth + .as_ref() + .expect("a bedcov row has its depth settings") + } + + /// `[(chrom, start, end, name)]` of the row's segments, sorted by + /// position (controls, targets, details; stable). + pub fn regions(&self) -> Vec<(&str, i64, i64, &str)> { + let mut out: Vec<_> = self + .controls + .iter() + .chain(&self.targets) + .chain(&self.details) + .map(|s| (self.chrom.as_str(), s.start, s.end, s.name.as_str())) + .collect(); + out.sort_by_key(|r| (r.1, r.2)); + out + } + + /// The row's reading for a count: its own line, else `more=` past the + /// highest numbered line, else empty. + pub fn reading_count(&self, n: i64) -> String { + if let Some(t) = self.readings.get(&n.to_string()) { + return t.clone(); + } + match (self.readings.get("more"), self.max_numbered()) { + (Some(more), Some(max)) if n > max => more.replace("{n}", &n.to_string()), + _ => String::new(), + } + } + + /// The row's reading for a word key (a class or a finding), or empty. + pub fn reading_word(&self, key: &str) -> String { + self.readings.get(key).cloned().unwrap_or_default() + } + + fn max_numbered(&self) -> Option { + self.readings + .keys() + .filter(|k| !k.is_empty() && k.bytes().all(|c| c.is_ascii_digit())) + .filter_map(|k| k.parse().ok()) + .max() + } + + /// An interval row `og cn intervals` calls: kind interval, bedcov, one + /// plain pass (LCE3BC, with a MAPQ >= 1 pass, stays a gene-dosage row). + pub fn is_interval_row(&self) -> bool { + self.kind == Kind::Interval && self.input == Input::Bedcov && self.depth().passes == ["."] + } +} + +/// The catalog, rows in file order. +#[derive(Debug, Clone)] +pub struct Catalog { + pub path: String, + pub loci: IndexMap, +} + +impl Catalog { + /// The row named `id`, when it is a bedcov row. + pub fn bedcov(&self, id: &str) -> Option<&Locus> { + self.loci.get(id).filter(|l| l.input == Input::Bedcov) + } + + /// Reads and checks the catalog at `path`. + pub fn load(path: &str) -> Result { + let lines = crate::read_lines_strict(path)?; + Catalog::parse(path, lines.iter().map(String::as_str)) + } + + /// Checks the catalog's lines (each without its `\n`); `path` names it + /// in messages. + pub fn parse<'a>(path: &str, lines: impl Iterator) -> Result { + let mut rows = lines.filter(|l| !crate::strip(l).is_empty() && !l.starts_with('#')); + let header_ok = rows + .next() + .is_some_and(|h| h.split('\t').eq(COLUMNS.iter().copied())); + let mut loci = IndexMap::new(); + let mut seen: HashSet = HashSet::new(); + let mut any = false; + for line in rows { + if !header_ok { + break; + } + any = true; + let cells: Vec<&str> = line.split('\t').collect(); + if cells.len() != COLUMNS.len() { + return Err(Error(format!( + "{path}: row {} has the wrong number of columns", + py_repr(cells[0]) + ))); + } + if loci.contains_key(cells[0]) { + return Err(Error(format!("{path}: locus {} is listed twice", cells[0]))); + } + let row = parse_row(&cells)?; + for s in row.targets.iter().chain(&row.controls).chain(&row.details) { + if !seen.insert(s.name.clone()) { + return Err(Error(format!( + "{path}: segment name {} is used by two loci", + s.name + ))); + } + } + loci.insert(row.id.clone(), row); + } + if !header_ok || !any { + return Err(Error(format!( + "{path}: the header must be {}", + COLUMNS.join(" ") + ))); + } + Ok(Catalog { + path: path.to_owned(), + loci, + }) + } +} + +/// The catalog beside the running binary: the first folder above it that +/// holds `cn_loci.tsv` or `bin/cn_loci.tsv` (`bin/.og/og-linux-*` in a +/// checkout, `/pgp-bin/.og/...` in a container, `rust/target/*/og`). +pub fn default_path() -> Option { + let exe = std::env::current_exe().ok()?; + exe.ancestors().skip(1).find_map(|d| { + [d.join("cn_loci.tsv"), d.join("bin").join("cn_loci.tsv")] + .into_iter() + .find(|p| Path::is_file(p)) + }) +} + +fn cell<'a>(cells: &[&'a str], col: &str) -> &'a str { + cells[COLUMNS.iter().position(|c| *c == col).expect("a column")] +} + +/// A number column: `.` gives `default`; anything not a finite number is +/// refused. +fn num(cells: &[&str], col: &str, default: Option) -> Result, Error> { + let v = cell(cells, col); + if v == "." { + return Ok(default); + } + match pycompat::num::parse_float(v) { + Ok(x) if x.is_finite() => Ok(Some(x)), + _ => Err(Error(format!( + "{}: {col} is {}, not a number or '.'", + cells[0], + py_repr(v) + ))), + } +} + +fn segments(cells: &[&str], col: &str) -> Result, Error> { + let v = cell(cells, col); + if v == "." { + return Ok(Vec::new()); + } + let loc = cells[0]; + let mut out = Vec::new(); + for part in v.split(';') { + let mut bits = part.splitn(3, ':'); + let name = bits.next().unwrap_or(""); + let span = bits.next(); + let label = bits.next().unwrap_or(""); + let bad = || { + Error(format!( + "{loc}: {col} segment {} is not NAME:start-end[:label]", + py_repr(part) + )) + }; + let span: Vec<&str> = span.ok_or_else(bad)?.split('-').collect(); + let [s, e] = span[..] else { return Err(bad()) }; + let (start, end) = match (py_int(s), py_int(e)) { + (Ok(s), Ok(e)) => (s, e), + _ => return Err(bad()), + }; + if name.is_empty() || end <= start || start < 0 { + return Err(Error(format!( + "{loc}: {col} segment {} has no name or an empty span", + py_repr(part) + ))); + } + out.push(Segment { + name: name.to_owned(), + start, + end, + label: label.to_owned(), + }); + } + Ok(out) +} + +fn readings(cells: &[&str]) -> Result, Error> { + let v = cell(cells, "readings"); + let mut out = IndexMap::new(); + if v == "." { + return Ok(out); + } + for part in v.split('|') { + match part.split_once('=') { + Some((k, t)) if !k.is_empty() && !t.is_empty() => { + out.insert(k.to_owned(), t.to_owned()); + } + _ => { + return Err(Error(format!( + "{}: reading {} is not KEY=text", + cells[0], + py_repr(part) + ))); + } + } + } + Ok(out) +} + +fn call(cells: &[&str]) -> Result { + let raw = cell(cells, "call"); + let bad = || { + Error(format!( + "{}: call {} is not round:D, cut:A,B..., class:LOW,HIGH or psv:D", + cells[0], + py_repr(raw) + )) + }; + let (mode, args) = raw.split_once(':').ok_or_else(bad)?; + let vals: Vec = args + .split(',') + .map(|x| py_float(x).ok().filter(|v| v.is_finite())) + .collect::>() + .unwrap_or_default(); + if vals.is_empty() || vals.windows(2).any(|w| w[0] > w[1]) { + return Err(bad()); + } + match (mode, &vals[..]) { + ("round", [d]) => Ok(CallRule::Round { max_off: *d }), + ("psv", [d]) => Ok(CallRule::Psv { max_off: *d }), + ("class", [lo, hi]) => Ok(CallRule::Class { + low: *lo, + high: *hi, + }), + ("cut", _) => Ok(CallRule::Cut(vals)), + _ => Err(bad()), + } +} + +fn psv_sites(loc: &str, details: &[Segment]) -> Result, Error> { + let mut out = Vec::new(); + for seg in details { + let label = &seg.label; + let bits: Vec<&str> = label.splitn(3, ' ').collect(); + let parsed = py_int(bits[0]) + .ok() + .and_then(|pos| bits.get(1).map(|b| (pos, b))); + let Some((pos, alleles)) = parsed else { + return Err(Error(format!( + "{loc}: PSV {} label {} is not 'PSEUDO_POS GENE>PSEUDO text'", + seg.name, + py_repr(label) + ))); + }; + let (gene, pseudo) = alleles.split_once('>').unwrap_or((alleles, "")); + let acgt = |s: &str| s.bytes().all(|c| b"ACGT".contains(&c)); + let shared = gene.bytes().any(|c| pseudo.as_bytes().contains(&c)); + if seg.end - seg.start != 1 + || pos < 0 + || gene.is_empty() + || pseudo.is_empty() + || !acgt(gene) + || !acgt(pseudo) + || shared + || bits.len() < 3 + { + return Err(Error(format!( + "{loc}: PSV {} must be one base with distinct gene and pseudogene bases and a text (got {}-{}, {})", + seg.name, + seg.start, + seg.end, + py_repr(label) + ))); + } + out.push(PsvSite { + name: seg.name.clone(), + gene_pos: seg.start, + pseudo_pos: pos, + gene: unique(gene), + pseudo: unique(pseudo), + text: bits[2].to_owned(), + }); + } + Ok(out) +} + +/// The distinct bases of `s`, in order (the Python kept them as a set). +fn unique(s: &str) -> Vec { + let mut out: Vec = Vec::new(); + for c in s.bytes() { + if !out.contains(&c) { + out.push(c); + } + } + out +} + +/// Checks one row (its 22 cells) as `copy_number.parse_row` does, in the +/// same order, so the first problem is the one the Python named. +pub fn parse_row(cells: &[&str]) -> Result { + let loc = cells[0]; + let kind = match cell(cells, "kind") { + "gene" => Kind::Gene, + "interval" => Kind::Interval, + "organelle" => Kind::Organelle, + k => { + return Err(Error(format!( + "{loc}: kind {} (one of gene, interval, organelle)", + py_repr(k) + ))); + } + }; + let input = match cell(cells, "input") { + "bedcov" => Input::Bedcov, + "cnvpytor-level" => Input::CnvpytorLevel, + i => { + return Err(Error(format!( + "{loc}: input {} (one of bedcov, cnvpytor-level)", + py_repr(i) + ))); + } + }; + let chrom = cell(cells, "chrom"); + if !is_chrom(chrom) { + return Err(Error(format!( + "{loc}: chrom {} (GRCh38 names, chr1 to chrM)", + py_repr(chrom) + ))); + } + let targets = segments(cells, "targets")?; + let controls = segments(cells, "controls")?; + let details = segments(cells, "details")?; + let call = call(cells)?; + let readings = readings(cells)?; + let is_psv = matches!(call, CallRule::Psv { .. }); + let psv = if is_psv { + psv_sites(loc, &details)? + } else { + Vec::new() + }; + let target_depth = cell(cells, "target_depth"); + if is_psv + && (psv.is_empty() + || target_depth != "sum" + || !PSV_FINDINGS.iter().all(|f| readings.contains_key(*f))) + { + return Err(Error(format!( + "{loc}: call psv needs PSV sites in details, target_depth sum (gene and pseudogene) and a reading for each of {}", + PSV_FINDINGS.join(", ") + ))); + } + if targets.is_empty() { + return Err(Error(format!("{loc}: no target segment"))); + } + let depth = if input == Input::Bedcov { + Some(depth(cells, &controls, &call)?) + } else { + None + }; + let mut names = HashSet::new(); + if !targets + .iter() + .chain(&controls) + .chain(&details) + .all(|s| names.insert(s.name.as_str())) + { + return Err(Error(format!("{loc}: a segment name is used twice"))); + } + let row = Locus { + id: loc.to_owned(), + kind, + chrom: chrom.to_owned(), + input, + targets, + controls, + details, + call, + readings, + psv, + target_depth: target_depth.to_owned(), + depth, + }; + if row.readings.contains_key("more") && row.max_numbered().is_none() { + return Err(Error(format!( + "{loc}: reading more= needs a numbered reading (0=, 1=, ...) to count past" + ))); + } + Ok(row) +} + +fn depth(cells: &[&str], controls: &[Segment], call: &CallRule) -> Result { + let loc = cells[0]; + if controls.is_empty() { + return Err(Error(format!("{loc}: a bedcov row needs control segments"))); + } + let target_mode = match cell(cells, "target_depth") { + "sum" => Some(TargetMode::Sum), + "pooled" => Some(TargetMode::Pooled), + _ => None, + }; + let control_mode = match cell(cells, "control_depth") { + "flanks" => Some(ControlMode::Flanks), + "pooled" => Some(ControlMode::Pooled), + _ => None, + }; + let (Some(target_mode), Some(control_mode)) = (target_mode, control_mode) else { + return Err(Error(format!( + "{loc}: target_depth sum|pooled and control_depth flanks|pooled" + ))); + }; + let scale = num(cells, "scale", None)?; + let offset = num(cells, "offset", Some(0.0))?.unwrap_or(0.0); + let scale = match scale { + Some(s) if s > 0.0 => s, + _ => return Err(Error(format!("{loc}: scale must be a positive number"))), + }; + let min_control = num(cells, "min_control_depth", Some(0.0))?.unwrap_or(0.0); + let max_ratio = num(cells, "max_control_ratio", None)?; + let mapq0_min = num(cells, "mapq0_min", None)?; + let mapq0_max = num(cells, "mapq0_max", None)?; + let passes: Vec = cell(cells, "bedcov") + .split(',') + .map(str::to_owned) + .collect(); + if (mapq0_min.is_some() || mapq0_max.is_some()) && passes.len() != 2 { + return Err(Error(format!( + "{loc}: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)" + ))); + } + let interval = cell(cells, "interval"); + if matches!(call, CallRule::Class { .. }) && interval != "cv" { + return Err(Error(format!("{loc}: call class needs interval cv"))); + } + if interval != "cv" && interval != "." { + return Err(Error(format!( + "{loc}: interval {} (cv or '.')", + py_repr(interval) + ))); + } + Ok(Depth { + target_mode, + control_mode, + scale, + offset, + passes, + mapq0_min, + mapq0_max, + min_control, + max_ratio, + ci_cv: interval == "cv", + }) +} + +/// A GRCh38 primary chromosome name: chr1 to chr22, chrX, chrY, chrM. +fn is_chrom(c: &str) -> bool { + match c.strip_prefix("chr") { + Some("X" | "Y" | "M") => true, + Some(n) => n + .parse::() + .is_ok_and(|v| (1..=22).contains(&v) && n == v.to_string()), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(over: &[(&str, &str)]) -> Vec { + let mut base: Vec<(&str, String)> = COLUMNS + .iter() + .map(|c| { + let v = match *c { + "locus" => "G", + "kind" => "gene", + "chrom" => "chr1", + "targets" => "G_A:1000-2000", + "controls" => "G_L:0-500;G_R:3000-3500", + "target_depth" => "sum", + "control_depth" => "flanks", + "scale" => "2", + "offset" => "0", + "bedcov" => ".", + "min_control_depth" => "8", + "call" => "round:0.3", + "input" => "bedcov", + _ => ".", + }; + (*c, v.to_owned()) + }) + .collect(); + for (k, v) in over { + base.iter_mut().find(|(c, _)| c == k).unwrap().1 = (*v).to_owned(); + } + base.into_iter().map(|(_, v)| v).collect() + } + + fn parse(over: &[(&str, &str)]) -> Result { + let r = row(over); + let cells: Vec<&str> = r.iter().map(String::as_str).collect(); + parse_row(&cells) + } + + #[test] + fn a_plain_row_loads() { + let l = parse(&[]).unwrap(); + assert_eq!(l.depth().scale, 2.0); + assert_eq!(l.regions()[0], ("chr1", 0, 500, "G_L")); + } + + #[test] + fn broken_rows_are_refused_with_the_python_text() { + let cases: &[(&[(&str, &str)], &str)] = &[ + ( + &[("kind", "cnv")], + "G: kind 'cnv' (one of gene, interval, organelle)", + ), + ( + &[("chrom", "chrx")], + "G: chrom 'chrx' (GRCh38 names, chr1 to chrM)", + ), + ( + &[("targets", "G_A:5-5")], + "G: targets segment 'G_A:5-5' has no name or an empty span", + ), + ( + &[("targets", "G_A:5")], + "G: targets segment 'G_A:5' is not NAME:start-end[:label]", + ), + ( + &[("call", "class:30,45")], + "G: call class needs interval cv", + ), + ( + &[("call", "cut:1.5,0.5")], + "G: call 'cut:1.5,0.5' is not round:D, cut:A,B..., class:LOW,HIGH or psv:D", + ), + ( + &[("controls", "G_A:0-500")], + "G: a segment name is used twice", + ), + (&[("scale", "0")], "G: scale must be a positive number"), + ( + &[("scale", "nan")], + "G: scale is 'nan', not a number or '.'", + ), + ( + &[("mapq0_min", "0.02")], + "G: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)", + ), + (&[("readings", "x")], "G: reading 'x' is not KEY=text"), + ]; + for (over, want) in cases { + assert_eq!(parse(over).unwrap_err().0, *want, "{over:?}"); + } + } + + #[test] + fn more_without_a_numbered_reading_is_a_catalog_error() { + let e = parse(&[("readings", "more={n} copies")]).unwrap_err(); + assert!(e.0.contains("more= needs a numbered reading"), "{e}"); + let l = parse(&[("readings", "2=two|more={n} copies")]).unwrap(); + assert_eq!(l.reading_count(5), "5 copies"); + assert_eq!(l.reading_count(2), "two"); + assert_eq!(l.reading_count(1), ""); + } + + #[test] + fn chrom_names() { + for c in ["chr1", "chr22", "chrX", "chrY", "chrM"] { + assert!(is_chrom(c), "{c}"); + } + for c in ["chr0", "chr23", "chr01", "chrx", "1", "chrUn"] { + assert!(!is_chrom(c), "{c}"); + } + } + + #[test] + fn the_repository_catalog_loads() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin/cn_loci.tsv"); + let cat = Catalog::load(path).unwrap(); + assert!(cat.bedcov("HBA").is_some() && cat.bedcov("LPA_KIV2").is_some()); + assert_eq!(cat.loci["CYP21A2"].psv.len(), 6); + } +} diff --git a/rust/crates/og-cn/src/cli.rs b/rust/crates/og-cn/src/cli.rs new file mode 100644 index 000000000..e00ef21a0 --- /dev/null +++ b/rust/crates/og-cn/src/cli.rs @@ -0,0 +1,703 @@ +//! `og cn `: +//! +//! ```text +//! og cn bed [--catalog TSV] [--locus L ...] [--kind interval] [--psv] +//! og cn call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] [--pileup P] +//! [--sex male|female|unknown] [--table OUT.tsv] [--bam-header H] +//! [--marker LPA_KIV2_4925 --marker-pileup P --marker-out OUT.tsv] +//! og cn intervals [--catalog TSV] --bedcov T --sex male|female|unknown --out OUT.tsv +//! og cn intervals [--catalog TSV] --failed --out OUT.tsv +//! og cn check-bam --header H +//! og cn marker-regions NAME +//! og cn marker-read NAME --pileup P [--copy-table T] --out OUT.tsv [--catalog TSV] +//! ``` +//! +//! The commands of `copy_number.py` with the same options (`--json`, which +//! no step passed, is gone), `traits_report.py --marker-regions` and +//! `--marker`, and two additions: `call --sex`, so a gene row on chrX or +//! chrY is read against the sample's count there (the Python never passed a +//! sex, so such a row could only read `sex_unknown`), and `call --marker`, +//! which reads the KIV-2 4925G>A marker beside the count in the same run and +//! judges the aligner on the count's own MAPQ 0 share. `marker-read +//! LPA_KIV2_4925 --copy-table` reads a table written earlier, where the +//! share is printed at 3 decimals, and judges that printed share. +//! +//! Exit codes: `call` 0 whatever the status, 2 on unreadable input, 3 on a +//! broken control, 4 on a BAM with ALT contigs; `check-bam` 0 or 4 (2 when +//! the header cannot be read); `bed` and `intervals` 0 or 2; `marker-read` +//! 0 whatever the reading (an unreadable pileup reads 'not assessed'), 2 +//! when the table cannot be written. Usage errors exit 2. + +use crate::catalog::{self, Catalog, Input, Locus}; +use crate::engine::{self, AssessError, Block, Means, Reason, Sex, Status}; +use crate::layout::{self, INTERVAL_COLUMNS}; +use crate::markers::{self, LPA_KIV2_4925, Marker, SMN1_27134}; +use crate::{Error, input, smn}; +use og_filters::py_repr; +use std::collections::HashMap; +use std::io::Write; + +const USAGE: &str = "usage: og cn bed [--catalog TSV] [--locus L ...] [--kind interval] [--psv] + og cn call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] [--pileup P] + [--sex male|female|unknown] [--table OUT] [--bam-header H] + [--marker LPA_KIV2_4925 --marker-pileup P --marker-out OUT] + og cn intervals [--catalog TSV] (--bedcov T --sex male|female|unknown | --failed) --out OUT + og cn check-bam --header H + og cn marker-regions NAME + og cn marker-read NAME --pileup P [--copy-table T] --out OUT [--catalog TSV] +"; + +/// A parsed command line: options with a value (`--locus` may repeat), +/// flags, positionals. +#[derive(Debug, Default)] +struct Args { + opts: HashMap<&'static str, Vec>, + flags: Vec<&'static str>, + pos: Vec, +} + +impl Args { + fn get(&self, name: &str) -> Option<&str> { + self.opts + .get(name) + .and_then(|v| v.last()) + .map(String::as_str) + } + + fn all(&self, name: &str) -> &[String] { + self.opts.get(name).map_or(&[], Vec::as_slice) + } + + fn flag(&self, name: &str) -> bool { + self.flags.contains(&name) + } +} + +/// Parses `args` by the options a verb takes, as argparse does: `--name +/// VALUE` or `--name=VALUE`, flags without a value, positionals in order. +fn parse( + args: &[String], + options: &[&'static str], + flags: &[&'static str], + positionals: usize, + required: &[&str], +) -> Result { + let mut out = Args::default(); + let mut extra = Vec::new(); + let mut i = 0; + while i < args.len() { + let a = &args[i]; + i += 1; + if a == "-h" || a == "--help" { + return Err(String::new()); + } + if a == "-" || !a.starts_with('-') { + if out.pos.len() < positionals { + out.pos.push(a.clone()); + } else { + extra.push(a.clone()); + } + continue; + } + let (name, inline) = match a.split_once('=') { + Some((n, v)) => (n, Some(v.to_owned())), + None => (a.as_str(), None), + }; + if let Some(&f) = flags.iter().find(|f| **f == name) { + if inline.is_some() { + return Err(format!( + "argument {f}: ignored explicit argument {}", + py_repr(inline.as_deref().unwrap_or("")) + )); + } + out.flags.push(f); + continue; + } + let Some(&known) = options.iter().find(|o| **o == name) else { + extra.push(a.clone()); + continue; + }; + let value = match inline { + Some(v) => v, + None => match args.get(i) { + Some(v) if v == "-" || !v.starts_with('-') => { + i += 1; + v.clone() + } + _ => return Err(format!("argument {known}: expected one argument")), + }, + }; + out.opts.entry(known).or_default().push(value); + } + if out.pos.len() < positionals { + return Err("the following arguments are required: NAME".into()); + } + let missing: Vec<&str> = required + .iter() + .copied() + .filter(|r| !out.opts.contains_key(r)) + .collect(); + if !missing.is_empty() { + return Err(format!( + "the following arguments are required: {}", + missing.join(", ") + )); + } + if !extra.is_empty() { + return Err(format!("unrecognized arguments: {}", extra.join(" "))); + } + Ok(out) +} + +fn usage(verb: &str, msg: &str, err: &mut dyn Write) -> u8 { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og cn {verb}: error: {msg}"); + 2 +} + +/// `og cn [args]`: the exit code. +pub fn main(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + return usage("", "the following arguments are required: verb", err); + }; + let rest = &args[1..]; + let spec: (&[&'static str], &[&'static str], usize, &[&str]) = match verb.as_str() { + "bed" => (&["--catalog", "--locus", "--kind"], &["--psv"], 0, &[]), + "call" => ( + &[ + "--catalog", + "--locus", + "--bedcov", + "--bedcov-mapq1", + "--table", + "--bam-header", + "--pileup", + "--sex", + "--marker", + "--marker-pileup", + "--marker-out", + ], + &[], + 0, + &["--locus", "--bedcov"], + ), + "intervals" => ( + &["--catalog", "--bedcov", "--sex", "--out"], + &["--failed"], + 0, + &["--out"], + ), + "check-bam" => (&["--header"], &[], 0, &["--header"]), + "marker-regions" => (&[], &[], 1, &[]), + "marker-read" => ( + &["--pileup", "--copy-table", "--out", "--catalog"], + &[], + 1, + &["--pileup", "--out"], + ), + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => return usage("", &format!("unknown cn verb {}", py_repr(other)), err), + }; + let a = match parse(rest, spec.0, spec.1, spec.2, spec.3) { + Ok(a) => a, + Err(m) if m.is_empty() => { + let _ = write!(out, "{USAGE}"); + return 0; + } + Err(m) => return usage(verb, &m, err), + }; + if let Some(m) = choice_error(&a) { + return usage(verb, &m, err); + } + let code = match verb.as_str() { + "bed" => bed(&a, out), + "call" => call(&a, out, err), + "intervals" => intervals(&a, out, err), + "check-bam" => check_bam(&a, err), + "marker-regions" => { + let m = markers::marker(&a.pos[0]).expect("checked"); + for r in markers::regions(m) { + let _ = writeln!(out, "{r}"); + } + Ok(0) + } + _ => marker_read(&a, out, err), + }; + let _ = out.flush(); + match code { + Ok(c) => c, + Err((c, e)) => { + let _ = writeln!(err, "ERROR: {e}"); + c + } + } +} + +/// The argparse `choices` checks. +fn choice_error(a: &Args) -> Option { + let bad = |name: &str, v: &str, choices: &[&str]| { + Some(format!( + "argument {name}: invalid choice: {} (choose from {})", + py_repr(v), + choices + .iter() + .map(|c| py_repr(c)) + .collect::>() + .join(", ") + )) + }; + if let Some(k) = a.get("--kind").filter(|k| *k != "interval") { + return bad("--kind", k, &["interval"]); + } + if let Some(s) = a.get("--sex").filter(|s| Sex::parse(s).is_none()) { + return bad("--sex", s, &["male", "female", "unknown"]); + } + if let Some(m) = a.get("--marker").filter(|m| *m != LPA_KIV2_4925.key) { + return bad("--marker", m, &[LPA_KIV2_4925.key]); + } + if let Some(n) = a.pos.first().filter(|n| markers::marker(n).is_none()) { + return bad("NAME", n, &[LPA_KIV2_4925.key, SMN1_27134.key]); + } + None +} + +type Fail = (u8, Error); + +fn fail2(e: Error) -> Fail { + (2, e) +} + +fn load(a: &Args) -> Result { + match a.get("--catalog") { + Some(p) => Catalog::load(p), + None => { + let p = catalog::default_path() + .ok_or_else(|| Error("no bin/cn_loci.tsv beside og; give --catalog".to_owned()))?; + Catalog::load(&p.to_string_lossy()) + } + } +} + +fn bed(a: &Args, out: &mut dyn Write) -> Result { + let cat = load(a).map_err(fail2)?; + let loci = a.all("--locus"); + if a.flag("--psv") { + if loci.is_empty() || a.get("--kind").is_some() { + return Err(fail2(Error( + "bed --psv needs --locus (and no --kind)".into(), + ))); + } + for loc in loci { + let Some(r) = cat.loci.get(loc).filter(|r| !r.psv.is_empty()) else { + return Err(fail2(Error(format!( + "no psv locus {} in the catalog", + py_repr(loc) + )))); + }; + let mut pos: Vec<(i64, String)> = r + .psv + .iter() + .map(|p| (p.gene_pos, p.name.clone())) + .chain( + r.psv + .iter() + .map(|p| (p.pseudo_pos, format!("{}_pseudo", p.name))), + ) + .collect(); + pos.sort(); + for (s, n) in pos { + let _ = writeln!(out, "{}\t{s}\t{}\t{n}", r.chrom, s + 1); + } + } + return Ok(0); + } + let names: Vec = if a.get("--kind").is_some() { + if !loci.is_empty() { + return Err(fail2(Error( + "bed: --kind interval or --locus, not both".into(), + ))); + } + cat.loci + .values() + .filter(|r| r.input == Input::Bedcov && r.is_interval_row()) + .map(|r| r.id.clone()) + .collect() + } else if loci.is_empty() { + cat.loci + .values() + .filter(|r| r.input == Input::Bedcov) + .map(|r| r.id.clone()) + .collect() + } else { + loci.to_vec() + }; + for loc in &names { + let Some(r) = cat.bedcov(loc) else { + return Err(fail2(Error(format!( + "no bedcov locus {} in the catalog", + py_repr(loc) + )))); + }; + for (c, s, e, n) in r.regions() { + let _ = writeln!(out, "{c}\t{s}\t{e}\t{n}"); + } + } + Ok(0) +} + +fn check_bam(a: &Args, err: &mut dyn Write) -> Result { + match input::check_bam(a.get("--header").expect("required")).map_err(fail2)? { + Some(refusal) => { + let _ = writeln!(err, "{refusal}"); + Ok(4) + } + None => Ok(0), + } +} + +/// The count of one locus, and with `--marker`, the KIV-2 4925G>A reading. +fn call(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + if let Some(h) = a.get("--bam-header") + && let Some(refusal) = input::check_bam(h).map_err(fail2)? + { + let _ = writeln!(err, "{refusal}"); + return Ok(4); + } + let marker = match ( + a.get("--marker"), + a.get("--marker-pileup"), + a.get("--marker-out"), + ) { + (None, None, None) => None, + (Some(_), Some(p), Some(o)) if a.get("--locus") == Some("LPA_KIV2") => Some((p, o)), + _ => { + return Err(fail2(Error( + "--marker LPA_KIV2_4925 goes with --locus LPA_KIV2, --marker-pileup and --marker-out".into(), + ))); + } + }; + let sex = a.get("--sex").and_then(Sex::parse).unwrap_or_default(); + let cat = load(a).map_err(fail2)?; + let locus = a.get("--locus").expect("required"); + let row = cat.bedcov(locus).ok_or_else(|| { + fail2(Error(format!( + "no bedcov locus {} in the catalog", + py_repr(locus) + ))) + })?; + let block = count(a, row, sex).map_err(|e| match e { + Count::Broken(s) => (3, Error(s)), + Count::Input(e) => (2, e), + })?; + let (rows, line) = layout::table(row, &block); + if let Some(t) = a.get("--table") { + write_table(t, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + } + let _ = writeln!(out, "{line}"); + if let Some((pileup, path)) = marker { + let kiv2 = rows + .iter() + .find(|(k, _)| k == "kiv2_copies") + .map_or(".", |(_, v)| v.as_str()); + let why = match (block.mapq0_share, row.depth().mapq0_min) { + (Some(s), Some(min)) => markers::mapq0_why_not(s, min), + _ => None, + }; + let counts = marker_counts(pileup, &LPA_KIV2_4925, "KIV-2 4925G>A is not assessed", err); + let rows = markers::null_allele(counts, kiv2, why.as_deref()); + write_table(path, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let _ = writeln!(out, "LPA {}", rows[8].1); + } + Ok(0) +} + +enum Count { + Broken(String), + Input(Error), +} + +impl From for Count { + fn from(e: Error) -> Count { + Count::Input(e) + } +} + +fn count(a: &Args, row: &Locus, sex: Sex) -> Result { + let bedcov = a.get("--bedcov").expect("required"); + let m = input::read_bedcov(bedcov, row)?; + let dp = row.depth(); + let q1_path = a.get("--bedcov-mapq1"); + if (dp.mapq0_min.is_some() || dp.mapq0_max.is_some()) && q1_path.is_none() { + return Err(Error(format!( + "{} has a MAPQ 0 guard: --bedcov-mapq1 is needed", + row.id + )) + .into()); + } + let q1 = q1_path.map(|p| input::read_bedcov(p, row)).transpose()?; + if let (Some(q), Some(p)) = (&q1, q1_path) + && m.is_empty() != q.is_empty() + { + let which = if m.is_empty() { bedcov } else { p }; + return Err(Error(format!( + "{which} is empty but the other table is not: both must cover the regions, or both be empty (no reads on that contig)" + )) + .into()); + } + let block = engine::assess(row, &m, q1.as_ref(), sex).map_err(|e| match e { + AssessError::BrokenControl(s) => Count::Broken(s), + AssessError::NeedsMapq1(s) => Count::Input(Error(s)), + })?; + if row.psv.is_empty() { + return Ok(block); + } + let Some(pileup) = a.get("--pileup") else { + return Err(Error(format!( + "{} is a psv row: --pileup (samtools mpileup over `og cn bed --psv`) is needed", + row.id + )) + .into()); + }; + let counts = if m.is_empty() { + HashMap::new() + } else { + input::read_psv_pileup(pileup, row)? + }; + Ok(engine::psv_assess(row, block, &counts)) +} + +fn intervals(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + let cat = load(a).map_err(fail2)?; + let rows: Vec<&Locus> = cat.loci.values().filter(|r| r.is_interval_row()).collect(); + let lines: Vec> = if a.flag("--failed") { + rows.iter() + .map(|r| { + let mut b = Block::empty(); + b.reason = Some(Reason::CheckFailed); + layout::interval_line(r, &b) + }) + .collect() + } else { + let (Some(bedcov), Some(sex)) = (a.get("--bedcov"), a.get("--sex").and_then(Sex::parse)) + else { + return Err(fail2(Error( + "intervals needs --bedcov and --sex (or --failed)".into(), + ))); + }; + call_intervals( + &rows, + &input::read_bedcov_all(bedcov).map_err(fail2)?, + sex, + err, + ) + .map_err(fail2)? + }; + let path = a.get("--out").expect("required"); + let mut body = INTERVAL_COLUMNS.join("\t"); + body.push('\n'); + for l in &lines { + body.push_str(&l.join("\t")); + body.push('\n'); + } + write_atomic(path, body.as_bytes()).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let mut n: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for l in &lines { + *n.entry(l[5].as_str()).or_default() += 1; + } + let parts: Vec = n.iter().map(|(k, v)| format!("{v} {k}")).collect(); + let _ = writeln!(out, "interval rows: {} ({})", lines.len(), parts.join(", ")); + Ok(0) +} + +/// Every interval row from one bedcov table; a broken control makes that +/// row `not assessed` (broken_control), never the whole table. +fn call_intervals( + rows: &[&Locus], + table: &[input::BedcovLine], + sex: Sex, + err: &mut dyn Write, +) -> Result>, Error> { + let mut by_name: HashMap<&str, &input::BedcovLine> = HashMap::new(); + for l in table { + if by_name.insert(&l.name, l).is_some() { + return Err(Error(format!( + "segment {} appears twice in the bedcov table", + l.name + ))); + } + } + let mut out = Vec::new(); + for row in rows { + let want = row.regions(); + if want.iter().any(|w| !by_name.contains_key(w.3)) { + return Err(Error(format!( + "the bedcov table lacks some of {}'s segments: its regions differ from `og cn bed --kind interval`", + row.id + ))); + } + let mut m = Means::new(); + for (c, s, e, n) in want { + let l = by_name[n]; + if (l.chrom.as_str(), l.start, l.end) != (c, s, e) { + return Err(Error(format!( + "segment {n} of {} has other coordinates in the bedcov table", + row.id + ))); + } + m.insert(n.to_owned(), l.sum / (e - s) as f64); + } + let block = match engine::assess(row, &m, None, sex) { + Ok(b) => b, + Err(e) => { + let _ = writeln!(err, "WARNING: {e}"); + let mut b = engine::assess(row, &Means::new(), None, sex) + .map_err(|e| Error(e.to_string()))?; + if b.status == Status::NotAssessed { + b.reason = Some(Reason::BrokenControl); + } + b + } + }; + out.push(layout::interval_line(row, &block)); + } + Ok(out) +} + +/// Reads a marker's pileup; an unreadable one is a warning and no counts. +fn marker_counts( + pileup: &str, + m: &Marker, + what: &str, + err: &mut dyn Write, +) -> Option { + match markers::read_pileup(pileup, m) { + Ok(c) => Some(c), + Err(e) => { + let _ = writeln!(err, "WARNING: {e}; {what}"); + None + } + } +} + +fn marker_read(a: &Args, out: &mut dyn Write, err: &mut dyn Write) -> Result { + let m = markers::marker(&a.pos[0]).expect("checked"); + let pileup = a.get("--pileup").expect("required"); + let path = a.get("--out").expect("required"); + let copy_table = a.get("--copy-table"); + let (rows, line) = if m.key == LPA_KIV2_4925.key { + // Step 46's table written earlier: the count, and the MAPQ 0 share + // as it was printed (3 decimals). + let t: HashMap = crate::read_lines(copy_table.unwrap_or("")) + .map(|ls| { + ls.iter() + .filter_map(|l| l.split_once('\t')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect() + }) + .unwrap_or_default(); + let kiv2 = t.get("kiv2_copies").map_or(".", String::as_str); + let why = match t.get("mapq0_share").map(|s| pycompat::num::parse_float(s)) { + Some(Ok(share)) => { + let cat = load(a).map_err(fail2)?; + let floor = cat + .bedcov("LPA_KIV2") + .and_then(|r| r.depth().mapq0_min) + .ok_or_else(|| { + fail2(Error( + "no LPA_KIV2 row with a MAPQ 0 floor in the catalog".into(), + )) + })?; + markers::mapq0_why_not(share, floor) + } + _ => None, + }; + let counts = marker_counts(pileup, m, "KIV-2 4925G>A is not assessed", err); + let rows = markers::null_allele(counts, kiv2, why.as_deref()); + let line = format!("LPA {}", rows[8].1); + (rows, line) + } else { + let counts = marker_counts(pileup, m, "the marker is not assessed", err); + let read = match smn::read(copy_table, m.chrom, m.copies) { + Ok(r) => r, + Err(e) => { + let _ = writeln!(err, "WARNING: {e}; the SMN copy numbers are not read"); + smn::SmnRead { + pscn: None, + text: "not read (the SMN table could not be read)".into(), + agcn: None, + } + } + }; + let rows = markers::smn1_marker(counts, &read); + let line = rows[9].1.clone(); + (rows, line) + }; + write_table(path, &rows).map_err(|e| (2, Error(format!("cannot write: {e}"))))?; + let _ = writeln!(out, "{line}"); + Ok(0) +} + +/// Writes a `metricvalue` table whole, through a temporary file. +fn write_table(path: &str, rows: &[(String, String)]) -> Result<(), String> { + let mut body = String::from("metric\tvalue\n"); + for (k, v) in rows { + body.push_str(k); + body.push('\t'); + body.push_str(v); + body.push('\n'); + } + write_atomic(path, body.as_bytes()) +} + +fn write_atomic(path: &str, body: &[u8]) -> Result<(), String> { + let run = || -> std::io::Result<()> { + let mut f = og_io::AtomicFile::create(path)?; + f.write_all(body)?; + f.commit() + }; + run().map_err(|e| og_filters::os_error(&e, path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + let (mut o, mut e) = (Vec::new(), Vec::new()); + let c = main(&args, &mut o, &mut e); + ( + c, + String::from_utf8(o).unwrap(), + String::from_utf8(e).unwrap(), + ) + } + + const CAT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin/cn_loci.tsv"); + + #[test] + fn marker_regions_lists_every_copy() { + let (c, o, _) = run(&["marker-regions", "SMN1_27134"]); + assert_eq!( + (c, o.as_str()), + (0, "chr5:70952074-70952074\nchr5:70076654-70076654\n") + ); + assert_eq!(run(&["marker-regions", "NOPE"]).0, 2); + } + + #[test] + fn bed_psv_and_usage() { + let (c, o, _) = run(&["bed", "--catalog", CAT, "--psv", "--locus", "CYP21A2"]); + assert_eq!(c, 0); + assert_eq!(o.lines().count(), 12); + let (c, _, e) = run(&["bed", "--catalog", CAT, "--psv", "--locus", "HBA"]); + assert_eq!( + (c, e.as_str()), + (2, "ERROR: no psv locus 'HBA' in the catalog\n") + ); + assert_eq!(run(&["call", "--locus", "HBA"]).0, 2); + } +} diff --git a/rust/crates/og-cn/src/engine.rs b/rust/crates/og-cn/src/engine.rs new file mode 100644 index 000000000..4e49373d1 --- /dev/null +++ b/rust/crates/og-cn/src/engine.rs @@ -0,0 +1,658 @@ +//! The depth-ratio engine: the guards in their order, the estimate, its 95% +//! interval and the call (`copy_number.assess`), and the PSV sites of a +//! `call psv` row (`copy_number.psv_assess`). The arithmetic is the +//! Python's step for step, with CPython 3.12's compensated `sum()` for +//! every interpreter (Parascopy's Python 3.11 summed plainly; the two agree +//! at every printed precision). + +use crate::catalog::{CallRule, ControlMode, Locus, Segment, TargetMode}; +use crate::round_i; +use pycompat::num::sum as py_sum; +use std::collections::HashMap; +use std::fmt; + +/// Mean depth per segment name. +pub type Means = HashMap; + +/// z of a two-sided 95% interval. +pub const Z95: f64 = 1.96; +/// Reads a PSV site needs (gene and pseudogene bases pooled) before its +/// copies are read. +pub const PSV_MIN_READS: u64 = 30; +/// A site reads as converted when its copies estimate lies this far under +/// the others' count (or under the usual count) or more. At 30x, about 70 +/// reads at a site, sampling alone gives an estimate of 2 copies a standard +/// deviation of about 0.25; HG002's six sites read 1.62 to 2.82 (35x), so +/// 0.6 is a screen setting, not a measured error rate. +pub const PSV_MARGIN: f64 = 0.6; + +/// The sample's sex, which sets the copies of chrX and chrY. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Sex { + Male, + Female, + #[default] + Unknown, +} + +impl Sex { + /// `male`, `female` or `unknown`. + pub fn parse(s: &str) -> Option { + match s { + "male" => Some(Sex::Male), + "female" => Some(Sex::Female), + "unknown" => Some(Sex::Unknown), + _ => None, + } + } +} + +/// What a block says. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Called, + NotAssessed, + /// A chrY row on a female (reason `no_y`). + NotApplicable, +} + +impl fmt::Display for Status { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Status::Called => "called", + Status::NotAssessed => "not assessed", + Status::NotApplicable => "not applicable", + }) + } +} + +/// Why a block is not called: the guard that stopped it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reason { + NoData, + LowControlDepth, + ControlsDisagree, + LowMapq0Share, + HighMapq0Share, + BelowFloor, + BetweenCounts, + SexUnknown, + NoY, + BrokenControl, + CheckFailed, + PsvLowDepth, +} + +impl fmt::Display for Reason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Reason::NoData => "no_data", + Reason::LowControlDepth => "low_control_depth", + Reason::ControlsDisagree => "controls_disagree", + Reason::LowMapq0Share => "low_mapq0_share", + Reason::HighMapq0Share => "high_mapq0_share", + Reason::BelowFloor => "below_floor", + Reason::BetweenCounts => "between_counts", + Reason::SexUnknown => "sex_unknown", + Reason::NoY => "no_y", + Reason::BrokenControl => "broken_control", + Reason::CheckFailed => "check_failed", + Reason::PsvLowDepth => "psv_low_depth", + }) + } +} + +/// A `call class` result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Class { + Low, + Typical, + High, +} + +impl Class { + pub fn as_str(self) -> &'static str { + match self { + Class::Low => "low", + Class::Typical => "typical", + Class::High => "high", + } + } +} + +/// What the PSV sites of a `call psv` row show. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Finding { + None, + Deletion, + Conversion, + PseudogeneDeletion, +} + +impl Finding { + pub fn as_str(self) -> &'static str { + match self { + Finding::None => "none", + Finding::Deletion => "deletion", + Finding::Conversion => "conversion", + Finding::PseudogeneDeletion => "pseudogene_deletion", + } + } +} + +/// One PSV site as read. +#[derive(Debug, Clone, PartialEq)] +pub struct SiteRead { + pub name: String, + pub gene_reads: u64, + pub pseudo_reads: u64, + pub other_reads: u64, + pub copies: Option, + pub estimate: Option, + pub ratio: Option, +} + +/// A locus's result. +#[derive(Debug, Clone, PartialEq)] +pub struct Block { + pub status: Status, + pub reason: Option, + pub estimate: Option, + pub total: Option, + pub copies: Option, + pub ci95: Option<(f64, f64)>, + pub class: Option, + pub target_depth: Option, + pub control_depth: Option, + /// Mean depth over every segment read (every read). + pub depths: Means, + /// The share of the target depth from reads of MAPQ 0; `None` without a + /// MAPQ >= 1 pass, or when a guard stopped before the depths. + pub mapq0_share: Option, + pub reading: String, + /// The copies with no change, when the sample's count of the chromosome + /// is known. + pub expected: Option, + /// A `call psv` row: the gene and pseudogene copies together. + pub module_total: Option, + /// A `call psv` row: its sites, in catalog order (after `psv_assess`). + pub sites: Vec, + pub finding: Option, + pub conversion_sites: Vec, +} + +impl Block { + /// A block that says nothing yet (not assessed, no reason). + pub fn empty() -> Block { + Block { + status: Status::NotAssessed, + reason: None, + estimate: None, + total: None, + copies: None, + ci95: None, + class: None, + target_depth: None, + control_depth: None, + depths: Means::new(), + mapq0_share: None, + reading: String::new(), + expected: None, + module_total: None, + sites: Vec::new(), + finding: None, + conversion_sites: Vec::new(), + } + } + + fn stop(mut self, reason: Reason) -> Block { + self.reason = Some(reason); + self + } +} + +/// Why `assess` gave no block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AssessError { + /// A control segment has no read while the other controls are covered: + /// the row's coordinates are wrong, or the sample lacks that stretch. + BrokenControl(String), + /// A row with a MAPQ 0 guard was given no MAPQ >= 1 depths. + NeedsMapq1(String), +} + +impl fmt::Display for AssessError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + AssessError::BrokenControl(s) | AssessError::NeedsMapq1(s) => f.write_str(s), + } + } +} + +fn combine_targets(mode: TargetMode, segs: &[Segment], m: &dyn Fn(&str) -> f64) -> f64 { + match mode { + TargetMode::Sum => py_sum(segs.iter().map(|s| m(&s.name))), + TargetMode::Pooled => pooled(segs, m), + } +} + +fn combine_controls(mode: ControlMode, segs: &[&Segment], m: &dyn Fn(&str) -> f64) -> f64 { + match mode { + ControlMode::Flanks => py_sum(segs.iter().map(|s| m(&s.name))) / segs.len() as f64, + ControlMode::Pooled => { + let owned: Vec = segs.iter().map(|s| (*s).clone()).collect(); + pooled(&owned, m) + } + } +} + +/// Length-weighted mean: `sum(m * len) / sum(len)`. +fn pooled(segs: &[Segment], m: &dyn Fn(&str) -> f64) -> f64 { + let num = py_sum(segs.iter().map(|s| m(&s.name) * (s.end - s.start) as f64)); + let den: i64 = segs.iter().map(|s| s.end - s.start).sum(); + num / den as f64 +} + +/// Coefficient of variation: sample standard deviation over the mean; 0 +/// with fewer than two values or a mean of 0 or less. +fn cv(values: &[f64]) -> f64 { + let mu = py_sum(values.iter().copied()) / values.len() as f64; + if mu <= 0.0 || values.len() < 2 { + return 0.0; + } + (py_sum(values.iter().map(|v| (v - mu).powi(2))) / (values.len() - 1) as f64).sqrt() / mu +} + +/// The copies of the row's chromosome a sample has: 2, or 1 on a male's +/// chrX or chrY; `None` where it cannot be said (a chrY row on a female or +/// on a sample of no known sex, a chrX row without a sex). +pub fn ploidy(row: &Locus, sex: Sex) -> Option { + let x = row.chrom == "chrX"; + if !x && row.chrom != "chrY" { + return Some(2); + } + match sex { + Sex::Male => Some(1), + Sex::Female if x => Some(2), + _ => None, + } +} + +/// The locus's block for mean depths `m` (every read) and `q1` (reads of +/// MAPQ >= 1, or `None` for a row without a MAPQ 0 guard). +pub fn assess(row: &Locus, m: &Means, q1: Option<&Means>, sex: Sex) -> Result { + let dp = row.depth(); + if q1.is_none() && (dp.mapq0_min.is_some() || dp.mapq0_max.is_some()) { + return Err(AssessError::NeedsMapq1(format!( + "{} has a MAPQ 0 guard: the MAPQ >= 1 depths are needed", + row.id + ))); + } + let mut b = Block::empty(); + b.depths = m.clone(); + let units = if row.target_depth == "sum" { + row.targets.len() as i64 + } else { + 1 + }; + let pl = ploidy(row, sex); + b.expected = pl.map(|p| p * units); + if row.chrom == "chrY" && sex == Sex::Female { + b.status = Status::NotApplicable; + return Ok(b.stop(Reason::NoY)); + } + let Some(pl) = pl else { + return Ok(b.stop(Reason::SexUnknown)); + }; + let (tg, ct) = (&row.targets, &row.controls); + if m.is_empty() || tg.iter().chain(ct).any(|s| !m.contains_key(&s.name)) { + return Ok(b.stop(Reason::NoData)); + } + let get = |n: &str| m[n]; + for s in ct { + let rest: Vec<&Segment> = ct.iter().filter(|c| c.name != s.name).collect(); + if m[&s.name] == 0.0 && !rest.is_empty() { + let other = combine_controls(dp.control_mode, &rest, &get); + if other >= dp.min_control && dp.min_control > 0.0 { + return Err(AssessError::BrokenControl(format!( + "BROKEN_CONTROL: control segment {} of {} has no read while the other controls have {}x; check its coordinates in bin/cn_loci.tsv (or the sample lacks that stretch)", + s.name, + row.id, + crate::fixed(other, 1) + ))); + } + } + } + let all_ct: Vec<&Segment> = ct.iter().collect(); + let t = combine_targets(dp.target_mode, tg, &get); + let c = combine_controls(dp.control_mode, &all_ct, &get); + let share = q1.map(|q| { + let tq = combine_targets(dp.target_mode, tg, &|n: &str| { + q.get(n).copied().unwrap_or(0.0) + }); + if t > 0.0 { 1.0 - tq / t } else { 0.0 } + }); + b.target_depth = Some(t); + b.control_depth = Some(c); + b.mapq0_share = share; + let low = match dp.control_mode { + ControlMode::Flanks => ct.iter().map(|s| m[&s.name]).fold(f64::INFINITY, f64::min), + ControlMode::Pooled => c, + }; + if low < dp.min_control || c <= 0.0 { + return Ok(b.stop(Reason::LowControlDepth)); + } + if dp.control_mode == ControlMode::Flanks + && ct.len() > 1 + && let Some(max_ratio) = dp.max_ratio + { + let vals: Vec = ct.iter().map(|s| m[&s.name]).collect(); + let lo = vals.iter().copied().fold(f64::INFINITY, f64::min); + let hi = vals.iter().copied().fold(f64::NEG_INFINITY, f64::max); + if lo <= 0.0 || hi / lo > max_ratio { + return Ok(b.stop(Reason::ControlsDisagree)); + } + } + let total = dp.scale * pl as f64 / 2.0 * t / c; + let est = total - dp.offset; + b.total = Some(total); + b.estimate = Some(est); + // With no copy left there is no read to judge the aligner by (a row with + // an offset has a floor of its own below, after this guard). + let no_copy = dp.offset == 0.0 && round_i(est) == 0; + if let (Some(min), Some(s)) = (dp.mapq0_min, share) + && !no_copy + && s < min + { + return Ok(b.stop(Reason::LowMapq0Share)); + } + if let (Some(max), Some(s)) = (dp.mapq0_max, share) + && s > max + { + return Ok(b.stop(Reason::HighMapq0Share)); + } + if dp.offset > 0.0 && est <= 0.0 { + return Ok(b.stop(Reason::BelowFloor)); + } + if dp.ci_cv { + let tv: Vec = tg.iter().map(|s| m[&s.name]).collect(); + let cv_c: Vec = ct.iter().map(|s| m[&s.name]).collect(); + let rel = (cv(&tv).powi(2) / tg.len() as f64 + cv(&cv_c).powi(2) / ct.len() as f64).sqrt(); + b.ci95 = Some(( + (total * (-Z95 * rel).exp() - dp.offset).max(0.0), + total * (Z95 * rel).exp() - dp.offset, + )); + } + match &row.call { + CallRule::Psv { max_off } | CallRule::Round { max_off } => { + let n = round_i(est); + if (est - n as f64).abs() > *max_off { + return Ok(b.stop(Reason::BetweenCounts)); + } + if matches!(row.call, CallRule::Psv { .. }) { + b.module_total = Some(n); + } else { + b.copies = Some(n); + b.reading = row.reading_count(n); + } + } + CallRule::Cut(cuts) => { + let n = cut_copies(est, cuts); + b.copies = Some(n); + b.reading = row.reading_count(n); + } + CallRule::Class { low, high } => { + let (lo, hi) = b.ci95.expect("a class row has interval cv"); + let class = if hi < *low { + Class::Low + } else if lo > *high { + Class::High + } else { + Class::Typical + }; + b.class = Some(class); + b.reading = row.reading_word(class.as_str()); + } + } + b.status = Status::Called; + Ok(b) +} + +/// The number of cut points a value reaches (`cut:0.5,1.5` gives 0, 1 or 2). +pub fn cut_copies(value: f64, cuts: &[f64]) -> i64 { + cuts.iter().filter(|c| value >= **c).count() as i64 +} + +/// Gene-base, pseudogene-base and other reads at one PSV site. +pub type SiteCounts = (u64, u64, u64); + +/// A `call psv` row's block (from [`assess`]) with its sites read. With the +/// module total called, each site of [`PSV_MIN_READS`] reads or more gets +/// the copies carrying the gene's base there (total x gene reads / gene and +/// pseudogene reads) and its ratio to the median of the sites. A site is +/// low when its estimate is more than [`PSV_MARGIN`] under the larger of +/// the median's count and the usual. The CYP21A2 copies are the least count +/// over the low sites, or the median's count when none is low. Finding: +/// `deletion` when a site is low and the module total under the usual; +/// `conversion` at the low sites when the module total is kept; +/// `pseudogene_deletion` when the module total is under the usual with no +/// low site; else `none`. +pub fn psv_assess(row: &Locus, mut b: Block, counts: &HashMap) -> Block { + b.sites = row + .psv + .iter() + .map(|p| { + let (g, ps, o) = counts.get(&p.name).copied().unwrap_or((0, 0, 0)); + SiteRead { + name: p.name.clone(), + gene_reads: g, + pseudo_reads: ps, + other_reads: o, + copies: None, + estimate: None, + ratio: None, + } + }) + .collect(); + b.finding = None; + b.conversion_sites = Vec::new(); + let (Status::Called, Some(module_total)) = (b.status, b.module_total) else { + b.status = Status::NotAssessed; + return b; + }; + let total = b.total.expect("a called block has a total"); + let expected = b.expected.expect("a called block has an expected count"); + let usual = expected.div_euclid(row.targets.len() as i64); + let mut read = Vec::new(); + for (i, s) in b.sites.iter_mut().enumerate() { + let n = s.gene_reads + s.pseudo_reads; + if n >= PSV_MIN_READS { + let e = total * s.gene_reads as f64 / n as f64; + s.estimate = Some(e); + s.copies = Some(round_i(e)); + read.push(i); + } + } + if read.is_empty() { + b.status = Status::NotAssessed; + b.reason = Some(Reason::PsvLowDepth); + b.copies = None; + return b; + } + let mut est: Vec = read.iter().filter_map(|&i| b.sites[i].estimate).collect(); + est.sort_by(f64::total_cmp); + let mid = est.len() / 2; + let median = if est.len() % 2 == 1 { + est[mid] + } else { + (est[mid - 1] + est[mid]) / 2.0 + }; + for &i in &read { + let e = b.sites[i].estimate.expect("read"); + b.sites[i].ratio = (median > 0.0).then(|| e / median); + } + let floor = round_i(median).max(usual) as f64; + let conv: Vec<&SiteRead> = b + .sites + .iter() + .filter(|s| s.estimate.is_some_and(|e| e < floor - PSV_MARGIN)) + .collect(); + // Only a site past the margin lowers the count, so one noisy site never + // gives fewer copies than the finding says. + let copies = conv + .iter() + .filter_map(|s| s.copies) + .min() + .unwrap_or_else(|| round_i(median)); + let conv_names: Vec = conv.iter().map(|s| s.name.clone()).collect(); + let finding = if !conv_names.is_empty() && module_total < expected { + Finding::Deletion + } else if !conv_names.is_empty() { + b.conversion_sites = conv_names.clone(); + Finding::Conversion + } else if module_total < expected { + Finding::PseudogeneDeletion + } else { + Finding::None + }; + b.copies = Some(copies); + b.finding = Some(finding); + let sites = conv_names + .iter() + .map(|n| { + let text = row + .psv + .iter() + .find(|p| &p.name == n) + .map_or("", |p| p.text.as_str()); + format!("{n} ({text})") + }) + .collect::>() + .join(", "); + b.reading = row + .reading_word(finding.as_str()) + .replace("{sites}", &sites); + b +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::Catalog; + + fn catalog() -> Catalog { + Catalog::load(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../bin/cn_loci.tsv" + )) + .unwrap() + } + + /// Depths with every segment of `row` at `x`, `over` overriding some. + fn means(row: &Locus, x: f64, over: &[(&str, f64)]) -> Means { + let mut m: Means = row + .targets + .iter() + .chain(&row.controls) + .map(|s| (s.name.clone(), x)) + .collect(); + for (k, v) in over { + m.insert((*k).to_owned(), *v); + } + m + } + + fn scaled(m: &Means, f: f64) -> Means { + m.iter().map(|(k, v)| (k.clone(), v * f)).collect() + } + + #[test] + fn hba_counts_and_guards() { + let cat = catalog(); + let hba = &cat.loci["HBA"]; + for (h2, h1, want) in [ + (30.0, 30.0, Some(4)), + (15.0, 30.0, Some(3)), + (15.0, 15.0, Some(2)), + ] { + let m = means(hba, 30.0, &[("HBA2", h2), ("HBA1", h1)]); + let b = assess(hba, &m, Some(&scaled(&m, 0.75)), Sex::Unknown).unwrap(); + assert_eq!((b.status, b.copies), (Status::Called, want)); + } + let m = means(hba, 30.0, &[("HBA2", 22.5)]); + let b = assess(hba, &m, Some(&scaled(&m, 0.75)), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::BetweenCounts)); + let m = means(hba, 30.0, &[]); + let b = assess(hba, &m, Some(&scaled(&m, 0.99)), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::LowMapq0Share)); + let b = assess(hba, &Means::new(), Some(&Means::new()), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::NoData)); + } + + #[test] + fn a_control_with_no_read_is_a_broken_control() { + let cat = catalog(); + let hba = &cat.loci["HBA"]; + let m = means(hba, 30.0, &[("flank_left", 0.0)]); + let e = assess(hba, &m, Some(&m), Sex::Unknown).unwrap_err(); + assert!( + e.to_string() + .starts_with("BROKEN_CONTROL: control segment flank_left of HBA"), + "{e}" + ); + } + + #[test] + fn ties_round_to_even() { + // An HBA estimate of exactly 2.5 lies 0.5 from both counts: + // between_counts whatever the rounding; with round:0.5 it would call + // 2 (half to even), never 3. + assert_eq!(round_i(2.5), 2); + assert_eq!(round_i(3.5), 4); + assert_eq!(round_i(0.5), 0); + } + + #[test] + fn chrx_and_chry_follow_the_sex() { + let cat = catalog(); + let az = cat + .loci + .values() + .find(|l| l.chrom == "chrY" && l.is_interval_row()) + .unwrap(); + let m = means(az, 15.0, &[]); + let b = assess(az, &m, None, Sex::Female).unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotApplicable, Some(Reason::NoY)) + ); + let b = assess(az, &m, None, Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::SexUnknown)); + let b = assess(az, &m, None, Sex::Male).unwrap(); + let units = if az.target_depth == "sum" { + az.targets.len() as i64 + } else { + 1 + }; + assert_eq!(b.expected, Some(units)); + } + + #[test] + fn lpa_class_and_interval() { + let cat = catalog(); + let lpa = &cat.loci["LPA_KIV2"]; + // N + 2 = 14 x M / D: M = 30 / 14 x (N + 2) at D = 30. + let m = means(lpa, 30.0, &[]); + let kiv: Vec<(String, f64)> = lpa + .targets + .iter() + .map(|s| (s.name.clone(), 30.0 / 14.0 * 22.0)) + .collect(); + let mut m2 = m.clone(); + m2.extend(kiv); + let b = assess(lpa, &m2, Some(&scaled(&m2, 0.1)), Sex::Unknown).unwrap(); + assert_eq!(b.class, Some(Class::Low)); + let (lo, hi) = b.ci95.unwrap(); + assert!((b.estimate.unwrap() - 20.0).abs() < 1e-9 && lo <= 20.0 && hi >= 20.0); + } +} diff --git a/rust/crates/og-cn/src/input.rs b/rust/crates/og-cn/src/input.rs new file mode 100644 index 000000000..12d56a4a5 --- /dev/null +++ b/rust/crates/og-cn/src/input.rs @@ -0,0 +1,272 @@ +//! The tables `og cn` reads: `samtools bedcov` sums, `samtools mpileup` +//! bases columns and a BAM's header. + +use crate::catalog::Locus; +use crate::engine::{Means, SiteCounts}; +use crate::{Error, each_line, py_float, py_int, strip}; +use og_filters::py_repr; +use std::collections::{HashMap, HashSet}; + +/// One `samtools bedcov` line: chrom, start, end, name, depth sum. +#[derive(Debug, Clone, PartialEq)] +pub struct BedcovLine { + pub chrom: String, + pub start: i64, + pub end: i64, + pub name: String, + pub sum: f64, +} + +/// Every line of a `samtools bedcov` table (blank and `#` lines skipped). +pub fn read_bedcov_all(path: &str) -> Result, Error> { + let mut out = Vec::new(); + each_line(path, |line| { + if strip(line).is_empty() || line.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + return Err(Error(format!( + "{path}: expected chrom, start, end, name, depth sum; got {}", + py_repr(strip(line)) + ))); + } + out.push(BedcovLine { + chrom: c[0].to_owned(), + start: py_int(c[1])?, + end: py_int(c[2])?, + name: c[3].to_owned(), + sum: py_float(c[4])?, + }); + Ok(()) + })?; + Ok(out) +} + +/// `{name: mean depth}` from `samtools bedcov` over `row`'s segments, which +/// must all be there and nothing else; empty for an empty table (no reads +/// on that contig). +pub fn read_bedcov(path: &str, row: &Locus) -> Result { + let mut means = Means::new(); + let mut seen: HashSet<(String, i64, i64, String)> = HashSet::new(); + each_line(path, |line| { + if strip(line).is_empty() || line.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + return Err(Error(format!( + "{path}: expected chrom, start, end, name, depth sum; got {}", + py_repr(strip(line)) + ))); + } + let (chrom, start, end, name) = (c[0], py_int(c[1])?, py_int(c[2])?, c[3]); + if end <= start { + return Err(Error(format!( + "{path}: region {name} {chrom}:{start}-{end} is empty or reversed" + ))); + } + if !seen.insert((chrom.to_owned(), start, end, name.to_owned())) { + return Err(Error(format!( + "{path}: region {name} {chrom}:{start}-{end} appears twice" + ))); + } + means.insert(name.to_owned(), py_float(c[4])? / (end - start) as f64); + Ok(()) + })?; + let want: HashSet<(String, i64, i64, String)> = row + .regions() + .into_iter() + .map(|(c, s, e, n)| (c.to_owned(), s, e, n.to_owned())) + .collect(); + if !seen.is_empty() && seen != want { + return Err(Error(format!( + "{path}: its regions differ from the ones `og cn bed --locus {}` prints", + row.id + ))); + } + Ok(means) +} + +/// The A, C, G and T reads of one `samtools mpileup` bases column. Read +/// starts (`^` and its MAPQ character), read ends, indels (`+N` and `-N` +/// with their N bases), deletions and skips are not bases; `.` and `,` are +/// the reference base when there is one. +pub fn count_bases(bases: &str, reference: &str) -> [u64; 4] { + let mut n = [0u64; 4]; + let refb = match reference.chars().next() { + Some(c) if reference.chars().count() == 1 => c.to_ascii_uppercase(), + _ => 'N', + }; + let chars: Vec = bases.chars().collect(); + let mut i = 0; + while i < chars.len() { + let ch = chars[i]; + if ch == '^' { + i += 2; + continue; + } + if ch == '+' || ch == '-' { + let mut j = i + 1; + while j < chars.len() && chars[j].is_ascii_digit() { + j += 1; + } + let len: usize = chars[i + 1..j] + .iter() + .collect::() + .parse() + .unwrap_or(0); + i = j + len; + continue; + } + let b = if ch == '.' || ch == ',' { + refb + } else { + ch.to_ascii_uppercase() + }; + if let Some(k) = base_index(b) { + n[k] += 1; + } + i += 1; + } + n +} + +fn base_index(b: char) -> Option { + "ACGT".find(b) +} + +/// Streams a `samtools mpileup` table (chrom, pos, ref, depth, bases, ...) +/// and hands each line on `chrom` at a position in `want` to `each` with +/// its base counts. A line that is not an mpileup line is an error. +pub fn each_pileup_site( + path: &str, + chrom: &str, + want: &dyn Fn(i64) -> bool, + mut each: impl FnMut(i64, [u64; 4]), +) -> Result<(), Error> { + each_line(path, |line| { + if strip(line).is_empty() { + return Ok(()); + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 || c[1].is_empty() || !c[1].bytes().all(|b| b.is_ascii_digit()) { + let shown: String = strip(line).chars().take(80).collect(); + return Err(Error(format!( + "{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); got {}", + py_repr(&shown) + ))); + } + let pos = py_int(c[1])?; + if c[0] == chrom && want(pos) { + each(pos, count_bases(c[4], c[2])); + } + Ok(()) + }) +} + +/// `{site: (gene-base reads, pseudogene-base reads, other reads)}` of a +/// `call psv` row from `samtools mpileup` over `og cn bed --psv`, each +/// site's two positions pooled: a read that carries the gene's base counts +/// for the gene wherever the aligner put it. A position missing from the +/// pileup has no read. +pub fn read_psv_pileup(path: &str, row: &Locus) -> Result, Error> { + let mut at: HashMap = HashMap::new(); + for (i, s) in row.psv.iter().enumerate() { + at.insert(s.gene_pos + 1, i); + at.insert(s.pseudo_pos + 1, i); + } + let mut counts = vec![[0u64; 4]; row.psv.len()]; + each_pileup_site(path, &row.chrom, &|p| at.contains_key(&p), |pos, n| { + let k = at[&pos]; + for (a, b) in counts[k].iter_mut().zip(n) { + *a += b; + } + })?; + Ok(row + .psv + .iter() + .zip(counts) + .map(|(s, n)| { + let pick = |bases: &[u8]| -> u64 { + bases + .iter() + .filter_map(|b| base_index(*b as char)) + .map(|k| n[k]) + .sum() + }; + let (g, p) = (pick(&s.gene), pick(&s.pseudo)); + (s.name.clone(), (g, p, n.iter().sum::() - g - p)) + }) + .collect()) +} + +/// The ALT or HLA contigs named by `samtools view -H` lines, or by contig +/// names one per line: a name ending in `_alt` or starting with `HLA-` +/// (the rule of `scripts/validate-setup.sh`). +pub fn alt_contig(line: &str) -> Option<&str> { + let name = if let Some(rest) = line.strip_prefix('@') { + if !rest.starts_with("SQ") { + return None; + } + line.split('\t') + .find_map(|f| f.strip_prefix("SN:")) + .unwrap_or("") + } else { + strip(line.split('\t').next().unwrap_or("")) + }; + (name.ends_with("_alt") || name.starts_with("HLA-")).then_some(name) +} + +/// The refusal line for a BAM header at `path` (`-` reads stdin) that names +/// an ALT or HLA contig, or `None`. +pub fn check_bam(path: &str) -> Result, Error> { + let mut found = 0usize; + let mut first = String::new(); + let mut each = |line: &str| { + if let Some(name) = alt_contig(line) { + if found == 0 { + first = name.to_owned(); + } + found += 1; + } + Ok(()) + }; + if path == "-" { + crate::each_line_of(std::io::stdin().lock(), "", &mut each)?; + } else { + each_line(path, &mut each)?; + } + Ok((found > 0).then(|| { + format!( + "REFUSED: the BAM was aligned to a reference with {found} ALT or HLA contigs (first: {first}). Reads that fit a primary locus and its ALT copy equally get MAPQ 0 or land on the ALT copy, so depth-based copy number reads deletions that are not there. Realign to the default no-ALT reference (docs/realignment.md)." + ) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bases_skip_starts_ends_and_indels() { + // ^] read start with its MAPQ; C . , c count; $ ends; +2AG and -1t + // are indels; * < > N are not bases. + assert_eq!(count_bases("^]C.,c$+2AGg-1tG*<>N", "C"), [0, 4, 2, 0]); + assert_eq!(count_bases(".,", "N"), [0, 0, 0, 0]); + } + + #[test] + fn alt_and_hla_contigs() { + assert_eq!( + alt_contig("@SQ\tSN:chr1_KI270762v1_alt\tLN:1"), + Some("chr1_KI270762v1_alt") + ); + assert_eq!( + alt_contig("HLA-A*01:01:01:01\t3503"), + Some("HLA-A*01:01:01:01") + ); + assert_eq!(alt_contig("@SQ\tSN:chr1\tLN:248956422"), None); + assert_eq!(alt_contig("@HD\tVN:1.6\tSN:x_alt"), None); + } +} diff --git a/rust/crates/og-cn/src/layout.rs b/rust/crates/og-cn/src/layout.rs new file mode 100644 index 000000000..cfc683669 --- /dev/null +++ b/rust/crates/og-cn/src/layout.rs @@ -0,0 +1,385 @@ +//! Each step's `metricvalue` table and stdout line for a block, and +//! the line of the interval table. The HBA and LPA_KIV2 layouts are those +//! `bin/hba_copy_number.py` and `bin/lpa_kiv2.py` wrote; a `call psv` row +//! gets the PSV layout and every other row the generic one. +//! +//! Each layout matches on the block's status and reason exhaustively. The +//! HBA layout wrote `status called` with no count for a reason it did not +//! name (a catalog edit could give `high_mapq0_share` or `below_floor`), and +//! the LPA layout failed on one; both now say `not assessed: `. + +use crate::catalog::Locus; +use crate::engine::{Block, Reason, Status}; +use crate::{d, f2, fixed}; + +/// The rows of a step table and its one stdout line. +pub type Table = (Vec<(String, String)>, String); + +/// How a row's table is laid out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Layout { + Generic, + Hba, + LpaKiv2, + Psv, +} + +impl Layout { + /// The layout of `row`. + pub fn of(row: &Locus) -> Layout { + if !row.psv.is_empty() { + Layout::Psv + } else { + match row.id.as_str() { + "HBA" => Layout::Hba, + "LPA_KIV2" => Layout::LpaKiv2, + _ => Layout::Generic, + } + } + } +} + +/// The step table of `row` for `b`. +pub fn table(row: &Locus, b: &Block) -> Table { + match Layout::of(row) { + Layout::Psv => psv(row, b), + Layout::Hba => hba(row, b), + Layout::LpaKiv2 => lpa(row, b), + Layout::Generic => generic(row, b), + } +} + +fn kv(rows: Vec<(&str, String)>) -> Vec<(String, String)> { + rows.into_iter().map(|(k, v)| (k.to_owned(), v)).collect() +} + +fn reason(b: &Block) -> String { + b.reason.map_or_else(|| ".".to_owned(), |r| r.to_string()) +} + +fn or_dot(s: &str) -> String { + if s.is_empty() { ".".into() } else { s.into() } +} + +fn generic(row: &Locus, b: &Block) -> Table { + let rows = kv(vec![ + ("status", b.status.to_string()), + ("copies", d(b.copies)), + ("estimate", f2(b.estimate)), + ("target_depth", f2(b.target_depth)), + ("control_depth", f2(b.control_depth)), + ("reason", reason(b)), + ("reading", or_dot(&b.reading)), + ]); + let why = b + .reason + .map_or_else(|| b.reading.clone(), |r| r.to_string()); + (rows, format!("{}: {} ({why})", row.id, b.status)) +} + +/// `""` for a reason a layout has no words for. +fn bare(b: &Block) -> String { + format!("not assessed: {}", reason(b)) +} + +fn hba(row: &Locus, b: &Block) -> Table { + const ORDER: [&str; 9] = [ + "status", + "alpha_globin_genes", + "estimate", + "hba2_depth", + "hba1_depth", + "flank_left_depth", + "flank_right_depth", + "mapq0_share", + "message", + ]; + if b.reason == Some(Reason::NoData) { + let msg = "not assessed: the BAM has no chr16 reads, so HBA1 and HBA2 have none"; + let rows = ORDER + .iter() + .map(|k| { + let v = match *k { + "status" => "not assessed", + "message" => msg, + _ => ".", + }; + ((*k).to_owned(), v.to_owned()) + }) + .collect(); + return (rows, format!("HBA1/HBA2: {msg}")); + } + let depth = |n: &str| b.depths.get(n).copied(); + let r2 = |x: Option| { + x.map_or_else( + || ".".to_owned(), + |v| pycompat::fmt::repr(pycompat::fmt::round(v, 2).unwrap_or(v)), + ) + }; + let (left, right) = (depth("flank_left"), depth("flank_right")); + let one = |x: Option| x.map_or_else(|| ".".to_owned(), |v| fixed(v, 1)); + let share = b.mapq0_share; + let est = b.estimate; + let (status, genes, message) = match (b.status, b.reason) { + (Status::Called, _) => ("called", d(b.copies), b.reading.clone()), + (_, Some(Reason::LowControlDepth)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: flank depth under {}x ({}x and {}x), too low to compare", + pycompat::fmt::general(row.depth().min_control, 6), + one(left), + one(right) + ), + ), + (_, Some(Reason::ControlsDisagree)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: the two flanks disagree ({}x and {}x); a deletion larger than --SEA, or uneven coverage, would skew the count", + one(left), + one(right) + ), + ), + (_, Some(Reason::LowMapq0Share)) => ( + "not assessed", + ".".into(), + format!( + "not assessed: reads of mapping quality 0 make up {}% of the depth over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM", + one(share.map(|s| 100.0 * s)) + ), + ), + (_, Some(Reason::BetweenCounts)) => { + let e = est.expect("an estimate between counts"); + let n = e.trunc() as i64; + ( + "not assessed", + ".".into(), + format!( + "not assessed: the estimate {} lies between {n} and {} alpha-globin genes", + fixed(e, 2), + n + 1 + ), + ) + } + _ => ("not assessed", ".".into(), bare(b)), + }; + let values = [ + status.to_owned(), + genes, + f2(est), + r2(depth("HBA2")), + r2(depth("HBA1")), + r2(left), + r2(right), + share.map_or_else(|| ".".to_owned(), |s| fixed(s, 3)), + message, + ]; + let line = format!( + "HBA1/HBA2: {} (estimate {}; HBA2 {}x, HBA1 {}x, flanks {}x and {}x; MAPQ 0 share over the genes {})", + values[8], values[2], values[3], values[4], values[5], values[6], values[7] + ); + let rows = ORDER + .iter() + .zip(values) + .map(|(k, v)| ((*k).to_owned(), v)) + .collect(); + (rows, line) +} + +/// The advice line of step 46's table. +pub const LPA_ADVICE: &str = "A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 European Atherosclerosis Society consensus advises measuring it at least once in every adult."; + +fn lpa(row: &Locus, b: &Block) -> Table { + let mut status = "not assessed".to_owned(); + let (mut copies, mut lo, mut hi, mut class) = ( + ".".to_owned(), + ".".to_owned(), + ".".to_owned(), + ".".to_owned(), + ); + let (mut kd, mut fd, mut sh) = (".".to_owned(), ".".to_owned(), ".".to_owned()); + let message; + if b.reason == Some(Reason::NoData) { + message = "not assessed: the BAM has no chr6, so LPA has no reads".to_owned(); + } else { + let k = b.target_depth; + let f = b.control_depth; + let one = |x: Option| x.map_or_else(|| ".".to_owned(), |v| fixed(v, 1)); + kd = f2(k); + fd = f2(f); + sh = b + .mapq0_share + .map_or_else(|| ".".to_owned(), |s| fixed(s, 3)); + message = match (b.status, b.reason) { + (Status::Called, _) => match (b.estimate, b.ci95) { + (Some(est), Some((l, h))) => { + status = "estimated".into(); + copies = fixed(est, 1); + lo = fixed(l, 1); + hi = fixed(h, 1); + class = b.class.map_or(".", |c| c.as_str()).to_owned(); + format!( + "about {} KIV-2 repeats in total (95% interval {lo} to {hi}): {}", + fixed(est, 0), + b.reading + ) + } + _ => bare(b), + }, + (_, Some(Reason::LowControlDepth)) => { + status = "too little depth".into(); + format!( + "too little depth: {}x over the single-copy LPA exons, under the {}x the count needs", + one(f), + pycompat::fmt::general(row.depth().min_control, 6) + ) + } + (_, Some(Reason::LowMapq0Share)) => format!( + "not assessed: reads of mapping quality 0 make up {}% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM", + one(b.mapq0_share.map(|s| 100.0 * s)) + ), + (_, Some(Reason::BelowFloor)) => format!( + "not assessed: the KIV-2 exons have {}x against {}x over the single-copy exons, too little for any repeat count (a deletion or a mapping problem)", + one(k), + one(f) + ), + _ => bare(b), + }; + } + let line = format!( + "LPA KIV-2: {message} (KIV-2 exons {kd}x, single-copy exons {fd}x, MAPQ 0 share {sh})" + ); + let rows = kv(vec![ + ("status", status), + ("kiv2_copies", copies), + ("ci95_low", lo), + ("ci95_high", hi), + ("class", class), + ("kiv2_depth", kd), + ("flank_depth", fd), + ("mapq0_share", sh), + ("message", message), + ("advice", LPA_ADVICE.to_owned()), + ]); + (rows, line) +} + +/// `.` for a called PSV row, else `not assessed: ` and why; a low MAPQ 0 +/// share is spelled out, because the BAM, not the genome, stopped the call. +fn psv_message(row: &Locus, b: &Block) -> String { + if b.status == Status::Called { + return ".".into(); + } + if b.reason == Some(Reason::LowMapq0Share) { + let targets: Vec<&str> = row + .targets + .iter() + .map(|s| s.name.strip_suffix("_BODY").unwrap_or(&s.name)) + .collect(); + let share = b.mapq0_share.unwrap_or(f64::NAN); + let min = row.depth().mapq0_min.unwrap_or(f64::NAN); + return format!( + "not assessed: the BAM has few or no multi-mapping reads here (reads of mapping quality 0 make up {}% of the depth over {}, under {}%), a possible sign of an aligner that leaves out the reads that fit both copies (as novoalign does by default); the module total may then come out low and read as a deletion; realign with minimap2 (step 02) or BWA-MEM", + fixed(100.0 * share, 1), + targets.join(" and "), + pycompat::fmt::general(100.0 * min, 6) + ); + } + format!("not assessed: {}", reason(b)) +} + +fn psv(row: &Locus, b: &Block) -> Table { + let mut rows = kv(vec![ + ("status", b.status.to_string()), + ("cyp21a2_copies", d(b.copies)), + ("module_total", d(b.module_total)), + ("expected_total", d(b.expected)), + ("estimate", f2(b.estimate)), + ("finding", b.finding.map_or(".", |f| f.as_str()).to_owned()), + ("conversion_sites", or_dot(&b.conversion_sites.join(","))), + ]); + for s in &b.sites { + rows.push(( + format!("psv_{}", s.name), + format!( + "{} gene, {} pseudogene, {} other reads; copies {} ({}); ratio {}", + s.gene_reads, + s.pseudo_reads, + s.other_reads, + d(s.copies), + f2(s.estimate), + f2(s.ratio) + ), + )); + } + rows.extend(kv(vec![ + ("target_depth", f2(b.target_depth)), + ("control_depth", f2(b.control_depth)), + ("reason", reason(b)), + ("reading", or_dot(&b.reading)), + ( + "mapq0_share", + b.mapq0_share + .map_or_else(|| ".".to_owned(), |s| fixed(s, 3)), + ), + ("message", psv_message(row, b)), + ])); + let why = match (b.reason, b.finding) { + (Some(r), _) => r.to_string(), + (None, Some(f)) => f.as_str().to_owned(), + (None, None) => "None".to_owned(), + }; + let line = format!( + "{}: {} ({why}; {} gene copies, module total {} of {})", + row.id, + b.status, + d(b.copies), + d(b.module_total), + d(b.expected) + ); + (rows, line) +} + +/// The interval table's columns. +pub const INTERVAL_COLUMNS: [&str; 11] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", + "reason", +]; + +/// One line of the interval table. +pub fn interval_line(row: &Locus, b: &Block) -> Vec { + let tg = &row.targets; + let change = match (b.copies, b.expected) { + (Some(c), Some(e)) if c < e => "deletion", + (Some(c), Some(e)) if c > e => "duplication", + (Some(_), Some(_)) => "no change", + _ => ".", + }; + let name = if tg[0].label.is_empty() { + row.id.clone() + } else { + tg[0].label.clone() + }; + vec![ + row.id.clone(), + name, + row.chrom.clone(), + tg.iter() + .map(|s| s.start) + .min() + .expect("a target") + .to_string(), + tg.iter() + .map(|s| s.end) + .max() + .expect("a target") + .to_string(), + b.status.to_string(), + d(b.copies), + d(b.expected), + change.to_owned(), + f2(b.estimate), + reason(b), + ] +} diff --git a/rust/crates/og-cn/src/lib.rs b/rust/crates/og-cn/src/lib.rs new file mode 100644 index 000000000..8c122d4c3 --- /dev/null +++ b/rust/crates/og-cn/src/lib.rs @@ -0,0 +1,164 @@ +//! Copy number from short-read depth (replaces `bin/hba_copy_number.py`, +//! `bin/lpa_kiv2.py`, the commands of `bin/copy_number.py` and the +//! copy-aware markers of `bin/traits_report.py`). +//! +//! One method, one catalog row per locus (`bin/cn_loci.tsv`, +//! `docs/cn-loci.md`): the depth over the row's target segments against the +//! depth over single-copy control segments, as `samtools bedcov` sums it, +//! +//! ```text +//! estimate = scale x T / C - offset +//! ``` +//! +//! guarded by a minimum control depth, flank agreement and the share of the +//! target depth from reads of mapping quality 0, then called by the row's +//! `call` rule. +//! +//! - [`catalog`]: the catalog, read and checked once per process. +//! - [`engine`]: the guards and the call ([`engine::assess`]), and the PSV +//! sites of a `call psv` row ([`engine::psv_assess`]). +//! - [`input`]: `samtools bedcov` and `samtools mpileup` tables, BAM headers. +//! - [`layout`]: each step's `metricvalue` table and the interval table. +//! - [`markers`]: the copy-aware markers (LPA KIV-2 4925G>A, SMN1 +//! g.27134T>G) read from the pileup at every GRCh38 copy of the site. +//! - [`smn`]: the one rule that reads the SMN1 psCN at exon 7 from step 35's +//! SMN table. +//! - [`cli`]: `og cn bed|call|intervals|check-bam|marker-regions|marker-read`. +//! +//! Each command writes what the Python wrote, byte for byte (`tests/parity`), +//! except where it fixes it on purpose; the module that does says where. + +pub mod catalog; +pub mod cli; +pub mod engine; +pub mod input; +pub mod layout; +pub mod markers; +pub mod smn; + +use std::fmt; +use std::io; + +/// A problem the command reports as `ERROR: ` with its exit code. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub String); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Error {} + +impl Error { + /// An I/O error on `path`, worded as Python's `OSError`. + pub fn io(e: &io::Error, path: &str) -> Error { + if e.kind() == io::ErrorKind::InvalidData { + Error(format!("{path}: {e}")) + } else { + Error(og_filters::os_error(e, path)) + } + } +} + +/// `int(s)` with Python's message when it is not one. +pub(crate) fn py_int(s: &str) -> Result { + pycompat::num::parse_int(s) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| { + Error(format!( + "invalid literal for int() with base 10: {}", + og_filters::py_repr(s) + )) + }) +} + +/// `float(s)` with Python's message when it is not one. +pub(crate) fn py_float(s: &str) -> Result { + pycompat::num::parse_float(s).map_err(|_| { + Error(format!( + "could not convert string to float: {}", + og_filters::py_repr(s) + )) + }) +} + +/// `f"{x:.{prec}f}"`. +pub(crate) fn fixed(x: f64, prec: usize) -> String { + pycompat::fmt::fixed(x, prec) +} + +/// `"." if x is None else f"{x:.2f}"`. +pub(crate) fn f2(x: Option) -> String { + x.map_or_else(|| ".".to_owned(), |v| fixed(v, 2)) +} + +/// `"." if x is None else str(x)`. +pub(crate) fn d(x: Option) -> String { + x.map_or_else(|| ".".to_owned(), |v| v.to_string()) +} + +/// `round(x)` of a finite float, as an integer (half to even). +pub(crate) fn round_i(x: f64) -> i64 { + x.round_ties_even() as i64 +} + +/// The text lines of a file as Python's `open(path, errors="replace")` +/// reads them (universal newlines), each without its `\n`. +pub(crate) fn read_lines(path: &str) -> Result, Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + lines_of(io::BufReader::new(file), path) +} + +/// The lines of the file at `path`, as Python's `open(path)` reads them: +/// universal newlines, strict UTF-8. +pub(crate) fn read_lines_strict(path: &str) -> Result, Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + og_io::Lines::new(io::BufReader::new(file), og_io::Utf8::Strict) + .map(|l| l.map_err(|e| Error::io(&e, path))) + .collect() +} + +/// The lines of `r`, as [`read_lines`]. +pub(crate) fn lines_of(r: impl io::BufRead, path: &str) -> Result, Error> { + let mut out = Vec::new(); + each_line_of(r, path, |l| { + out.push(l.to_owned()); + Ok(()) + })?; + Ok(out) +} + +/// Streams the lines of `r` (as [`read_lines`] reads them) through `each`, +/// one at a time, so a long pileup or header is never held whole. +pub(crate) fn each_line_of( + r: impl io::BufRead, + path: &str, + mut each: impl FnMut(&str) -> Result<(), Error>, +) -> Result<(), Error> { + let mut lines = og_io::Lines::new(r, og_io::Utf8::Replace); + let mut line = String::new(); + while lines + .read_line(&mut line) + .map_err(|e| Error::io(&e, path))? + { + each(line.strip_suffix('\n').unwrap_or(&line))?; + } + Ok(()) +} + +/// [`each_line_of`] over the file at `path`. +pub(crate) fn each_line( + path: &str, + each: impl FnMut(&str) -> Result<(), Error>, +) -> Result<(), Error> { + let file = std::fs::File::open(path).map_err(|e| Error::io(&e, path))?; + each_line_of(io::BufReader::new(file), path, each) +} + +/// Python's `str.strip()`. +pub(crate) fn strip(s: &str) -> &str { + pycompat::text::strip(s, None) +} diff --git a/rust/crates/og-cn/src/markers.rs b/rust/crates/og-cn/src/markers.rs new file mode 100644 index 000000000..851a27c56 --- /dev/null +++ b/rust/crates/og-cn/src/markers.rs @@ -0,0 +1,451 @@ +//! Copy-aware markers: two well-sourced markers sit in sequence the genome +//! holds more than once, so a gVCF genotype at one position cannot read +//! them. Each is read from the BAM by the step that counts its copies: the +//! reads at every GRCh38 copy of the site are added up, and the share that +//! carries the marker's base is a fraction of copies, not a diploid +//! genotype. +//! +//! - `LPA_KIV2_4925`: the LPA KIV-2 4925G>A splice variant (Coassin et al. +//! 2017, PMID 28444229), read by step 46 beside the KIV-2 count. +//! - `SMN1_27134`: SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID +//! 23788250), read by step 35 beside the SMN1 psCN. +//! +//! The values are those `bin/traits_report.py` held (MARKERS), each copy +//! checked against the 21-base context and the dbSNP record by +//! `marker_errors` (a unit test here, and `tests/test_traits_report.py` +//! compares the records with what collect_summary reads). + +use crate::input::each_pileup_site; +use crate::{Error, fixed}; + +/// One copy-aware marker. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Marker { + pub key: &'static str, + pub name: &'static str, + pub chrom: &'static str, + /// The plus-strand base read at each copy, and the marker's base. + pub reference: u8, + pub alt: u8, + /// dbSNP b157's record of one copy: rsID, chrom, POS, REF, ALT. + pub dbsnp: (&'static str, &'static str, i64, &'static str, &'static str), + /// Every GRCh38 copy of the site, 1-based. + pub copies: &'static [i64], + /// The GRCh38 sequence of 10 bases on each side of every copy. + pub context: &'static str, + /// Fewer reads over all copies together: too little depth. + pub min_depth: u64, + /// Reads with the marker's base for 'present' ... + pub min_alt: u64, + /// ... and a share of the reads at least this. + pub min_fraction: f64, + /// This many reads with the marker's base or fewer: 'not seen'. + pub max_absent: u64, +} + +/// The LPA KIV-2 4925G>A splice variant: the last base of the second KIV-2 +/// exon, G>A on LPA's minus strand, so C>T on the plus strand; one copy at +/// the end of each of the six KIV-2 units of GRCh38 and one at the +/// near-identical last exon of KIV-1. dbSNP names the first KIV-2 copy. +pub const LPA_KIV2_4925: Marker = Marker { + key: "LPA_KIV2_4925", + name: "LPA KIV-2 4925G>A", + chrom: "chr6", + reference: b'C', + alt: b'T', + dbsnp: ("rs1484919544", "chr6", 160_640_667, "C", "T"), + copies: &[ + 160_646_214, + 160_640_667, + 160_635_123, + 160_629_574, + 160_624_030, + 160_618_484, + 160_612_932, + ], + context: "GACTCCTTACCTTGTTCGGAA", + min_depth: 100, + min_alt: 5, + min_fraction: 0.008, + max_absent: 2, +}; + +/// SMN1 g.27134T>G (NM_000344.4:c.*3+80T>G, intron 7, plus strand); SMN2 +/// holds the same 21 bases 875,420 bases upstream. +pub const SMN1_27134: Marker = Marker { + key: "SMN1_27134", + name: "SMN1 g.27134T>G", + chrom: "chr5", + reference: b'T', + alt: b'G', + dbsnp: ("rs143838139", "chr5", 70_952_074, "T", "G"), + copies: &[70_952_074, 70_076_654], + context: "TTTTTGAACATTTAAAAAGTT", + min_depth: 10, + min_alt: 3, + min_fraction: 0.05, + max_absent: 1, +}; + +/// Every marker, by key. +pub const MARKERS: [Marker; 2] = [LPA_KIV2_4925, SMN1_27134]; + +/// The marker named `key`. +pub fn marker(key: &str) -> Option<&'static Marker> { + MARKERS.iter().find(|m| m.key == key) +} + +/// A marker's reading. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Present, + NotSeen, + Uncertain, + TooLittleDepth, + NotAssessed, +} + +impl Status { + pub fn as_str(self) -> &'static str { + match self { + Status::Present => "present", + Status::NotSeen => "not seen", + Status::Uncertain => "uncertain", + Status::TooLittleDepth => "too little depth", + Status::NotAssessed => "not assessed", + } + } +} + +/// Reads with the reference base, with the marker's base, and others. +pub type Counts = (u64, u64, u64); + +/// A marker's reading and the numbers behind it, as the table prints them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reading { + pub status: Status, + pub alt_reads: String, + pub depth: String, + /// `alt / depth` at 4 decimals, or `.`. + pub fraction: String, +} + +/// The copy sites as samtools regions, `chrom:pos-pos`, in the marker's order. +pub fn regions(m: &Marker) -> Vec { + m.copies + .iter() + .map(|p| format!("{}:{p}-{p}", m.chrom)) + .collect() +} + +/// The reads over every copy of the marker from `samtools mpileup` lines +/// at its sites, without a reference: a read that carries the marker's base +/// counts wherever the aligner put it. +pub fn read_pileup(path: &str, m: &Marker) -> Result { + let mut n = [0u64; 4]; + each_pileup_site(path, m.chrom, &|p| m.copies.contains(&p), |_, c| { + for (a, b) in n.iter_mut().zip(c) { + *a += b; + } + })?; + let at = |b: u8| n["ACGT".find(b as char).expect("a base")]; + let (r, a) = (at(m.reference), at(m.alt)); + Ok((r, a, n.iter().sum::() - r - a)) +} + +/// The marker's reading from its counts (`None`: the pileup could not be +/// read). The depth counts the reads with the reference or the marker's base. +pub fn assess(m: &Marker, counts: Option) -> Reading { + let Some((r, a, _)) = counts else { + return Reading { + status: Status::NotAssessed, + alt_reads: ".".into(), + depth: ".".into(), + fraction: ".".into(), + }; + }; + let depth = r + a; + let frac = (depth > 0).then(|| a as f64 / depth as f64); + let status = if depth < m.min_depth { + Status::TooLittleDepth + } else if a >= m.min_alt && frac.is_some_and(|f| f >= m.min_fraction) { + Status::Present + } else if a <= m.max_absent { + Status::NotSeen + } else { + Status::Uncertain + }; + Reading { + status, + alt_reads: a.to_string(), + depth: depth.to_string(), + fraction: frac.map_or_else(|| ".".into(), |f| fixed(f, 4)), + } +} + +/// The printed fraction times `copies`, at one decimal (the reading of a +/// present or uncertain marker; `.` otherwise). +fn copies_of(a: &Reading, copies: Option) -> String { + match (a.status, copies, pycompat::num::parse_float(&a.fraction)) { + (Status::Present | Status::Uncertain, Some(c), Ok(f)) => fixed(f * c, 1), + _ => ".".into(), + } +} + +const NULL_LIMIT: &str = "A short-read fraction cannot say which chromosome or which repeat carries it, so it cannot say whether it sits on the small or the large isoform."; + +/// The KIV-2 4925G>A table: the reading beside step 46's count. +/// `kiv2_copies` is the count as its table prints it (`.` without one); +/// `why_not` is the reason the reads cannot be trusted (the aligner left +/// out the reads of MAPQ 0), which makes it not assessed whatever the counts. +pub fn null_allele( + counts: Option, + kiv2_copies: &str, + why_not: Option<&str>, +) -> Vec<(String, String)> { + let m = &LPA_KIV2_4925; + let a = assess(m, if why_not.is_some() { None } else { counts }); + let total = pycompat::num::parse_float(kiv2_copies) + .ok() + .map(|n| n + 2.0); + let copies = copies_of(&a, total); + let name = format!( + "{} (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017)", + m.name + ); + let st = a.status.as_str(); + let msg = if a.status == Status::NotAssessed { + format!( + "{name}: not assessed, {}", + why_not.unwrap_or("the reads at its seven GRCh38 copies could not be read.") + ) + } else { + let mut s = format!( + "{name}: {st}, {} of {} reads at its seven GRCh38 copies carry it", + a.alt_reads, a.depth + ); + if copies != "." { + let t = total.expect("copies need the count"); + s += &format!(" (about {copies} of the {} copies counted)", fixed(t, 0)); + } + if a.status == Status::TooLittleDepth { + s += &format!(", under the {} reads a reading needs", m.min_depth); + } + s.push('.'); + match a.status { + Status::Present => { + s += " Carriers had a lower Lp(a) than non-carriers with the same number of repeats, most of all with few repeats. "; + s += NULL_LIMIT; + } + Status::NotSeen => { + s += " Not seen in the reads is not proof of absence: one carrying repeat among many gives few reads, and a short-read fraction never says which chromosome or repeat holds a variant." + } + _ => { + s.push(' '); + s += NULL_LIMIT; + } + } + s + }; + rows(&[ + ("marker", m.name.to_owned()), + ("rsid", format!("{} (first KIV-2 copy)", m.dbsnp.0)), + ("status", st.to_owned()), + ("alt_reads", a.alt_reads), + ("depth", a.depth), + ("fraction", a.fraction), + ("copies", copies), + ("kiv2_copies", kiv2_copies.to_owned()), + ("message", msg), + ]) +} + +/// The reason a KIV-2 4925G>A reading cannot be trusted when reads of +/// MAPQ 0 make up less than `floor` of the KIV-2 depth: the reads that fit +/// several copies are the ones a carrying repeat gives. +pub fn mapq0_why_not(share: f64, floor: f64) -> Option { + (share < floor).then(|| { + format!( + "reads of mapping quality 0 make up {}% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives.", + fixed(100.0 * share, 1) + ) + }) +} + +/// SMN1's marker never proves 2+0, and its absence never rules it out. +pub const SMN1_NOT_PROOF: &str = + "This marker does not prove 2+0, and its absence does not rule 2+0 out."; + +/// Step 35's SMN1 marker table: the marker's reading beside the SMN1 psCN +/// of the SMN table (`smn`, read by [`crate::smn::read`]). +pub fn smn1_marker(counts: Option, smn: &crate::smn::SmnRead) -> Vec<(String, String)> { + let m = &SMN1_27134; + let a = assess(m, counts); + let copies = copies_of(&a, smn.agcn.map(|v| v as f64)); + let st = a.status.as_str(); + let head = if a.status == Status::NotAssessed { + format!( + "{} ({}): not assessed, the reads at the marker could not be read.", + m.name, m.dbsnp.0 + ) + } else { + let mut s = format!( + "{} ({}): {st}, {} of {} reads carry G at the SMN1 and SMN2 copies of the site together", + m.name, m.dbsnp.0, a.alt_reads, a.depth + ); + if copies != "." { + s += &format!( + " (about {copies} of the {} SMN1 and SMN2 copies)", + smn.agcn.expect("copies need the agCN") + ); + } + if a.status == Status::TooLittleDepth { + s += &format!(", under the {} reads a reading needs", m.min_depth); + } + s + ". The reads cannot tell whether a G sits on SMN1 or SMN2." + }; + let body = match smn.pscn { + Some(2) => match a.status { + Status::Present => " With two SMN1 copies, this marker goes with a higher chance that both sit on one chromosome (2+0, a silent carrier): in Luo 2014 it raised the residual carrier risk of a two-copy result (in African Americans from about 1 in 121 to 1 in 34); the figures depend on ancestry.".to_owned(), + Status::NotSeen => " With two SMN1 copies and no marker, the chance of 2+0 is lower than for a two-copy result alone (in Luo 2014, in African Americans, about 1 in 396 against 1 in 121), not zero; the figures depend on ancestry.".to_owned(), + _ => " With two SMN1 copies the marker would refine the chance of 2+0; here it gives no reading.".to_owned(), + }, + Some(ps) => format!( + " SMN1 has {ps} cop{} here; the marker refines the carrier risk of a two-copy result only.", + if ps == 1 { "y" } else { "ies" } + ), + None => format!( + " SMN1 copy number: {}; the marker refines the carrier risk of a two-copy result only.", + smn.text + ), + }; + rows(&[ + ("marker", m.name.to_owned()), + ("rsid", m.dbsnp.0.to_owned()), + ("status", st.to_owned()), + ("alt_reads", a.alt_reads), + ("depth", a.depth), + ("fraction", a.fraction), + ("copies", copies), + ("smn1_pscn", smn.text.clone()), + ("agcn", crate::d(smn.agcn)), + ("message", format!("{head}{body} {SMN1_NOT_PROOF}")), + ]) +} + +fn rows(r: &[(&str, String)]) -> Vec<(String, String)> { + r.iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() +} + +/// Problems of one marker (empty when it is consistent): the dbSNP record +/// is one of its copies with its ref and alt, on the plus strand and not +/// swapped; the context's middle base is the ref; no copy is listed twice. +pub fn marker_errors(m: &Marker) -> Vec { + let (rs, chrom, pos, r, a) = m.dbsnp; + let comp = |b: &str| match b { + "A" => "T", + "C" => "G", + "G" => "C", + "T" => "A", + _ => "", + }; + let (mr, ma) = ( + (m.reference as char).to_string(), + (m.alt as char).to_string(), + ); + let mut errs = Vec::new(); + if chrom != m.chrom || !m.copies.contains(&pos) { + errs.push(format!( + "{}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies", + m.key + )); + } + if (mr.as_str(), ma.as_str()) == (a, r) { + errs.push(format!( + "{}: ref and alt are swapped (marker {mr}>{ma}, dbSNP {r}>{a})", + m.key + )); + } else if (mr.as_str(), ma.as_str()) == (comp(r), comp(a)) { + errs.push(format!( + "{}: the alleles are the minus-strand bases (marker {mr}>{ma}, dbSNP {r}>{a}): write the plus-strand bases", + m.key + )); + } else if (mr.as_str(), ma.as_str()) != (r, a) { + errs.push(format!("{}: marker {mr}>{ma}, dbSNP {r}>{a}", m.key)); + } + let ctx = m.context.as_bytes(); + if ctx.len() % 2 != 1 || ctx[ctx.len() / 2] != m.reference { + errs.push(format!( + "{}: the middle base of the context is not the ref {mr}", + m.key + )); + } + let mut seen = std::collections::HashSet::new(); + if !m.copies.iter().all(|c| seen.insert(c)) { + errs.push(format!("{}: a copy is listed twice", m.key)); + } + errs +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn markers_are_consistent() { + for m in &MARKERS { + assert!(marker_errors(m).is_empty(), "{:?}", marker_errors(m)); + } + // A planted swap, a minus-strand pair and a copy listed twice are caught. + let mut bad = SMN1_27134; + bad.reference = b'G'; + bad.alt = b'T'; + assert!(marker_errors(&bad)[0].contains("swapped")); + bad.reference = b'A'; + bad.alt = b'C'; + assert!( + marker_errors(&bad) + .iter() + .any(|e| e.contains("minus-strand")) + ); + bad = LPA_KIV2_4925; + bad.copies = &[160_640_667, 160_640_667]; + assert!( + marker_errors(&bad) + .iter() + .any(|e| e.contains("listed twice")) + ); + } + + #[test] + fn statuses() { + let m = &LPA_KIV2_4925; + assert_eq!(assess(m, Some((95, 5, 0))).status, Status::Present); + assert_eq!(assess(m, Some((98, 2, 0))).status, Status::NotSeen); + assert_eq!(assess(m, Some((996, 4, 0))).status, Status::Uncertain); + assert_eq!(assess(m, Some((50, 5, 0))).status, Status::TooLittleDepth); + assert_eq!(assess(m, None).status, Status::NotAssessed); + assert_eq!(assess(m, Some((95, 5, 0))).fraction, "0.0500"); + } + + #[test] + fn null_allele_copies_and_why_not() { + let t = null_allele(Some((190, 10, 0)), "20.0", None); + assert_eq!(t[6], ("copies".into(), "1.1".into())); + assert!( + t[8].1.contains("(about 1.1 of the 22 copies counted)"), + "{}", + t[8].1 + ); + let why = mapq0_why_not(0.1996, 0.2).unwrap(); + assert!( + why.starts_with("reads of mapping quality 0 make up 20.0%"), + "{why}" + ); + let t = null_allele(Some((190, 10, 0)), "20.0", Some(&why)); + assert_eq!(t[2].1, "not assessed"); + assert!(mapq0_why_not(0.2, 0.2).is_none()); + } +} diff --git a/rust/crates/og-cn/src/smn.rs b/rust/crates/og-cn/src/smn.rs new file mode 100644 index 000000000..fba00a73b --- /dev/null +++ b/rust/crates/og-cn/src/smn.rs @@ -0,0 +1,200 @@ +//! The one rule that reads SMN1's copy number at exon 7 from step 35's SMN +//! table (Parascopy's `res.samples.bed`, as `chrom start end locus +//! agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual +//! homologous_regions`). +//! +//! Rows in file order; each row covers its own region and its homologous +//! regions. The first row with a region that holds exon 7 gives the psCN of +//! that region, and the first row with a region that holds a copy of the +//! SMN1 g.27134T>G site gives the agCN. A copy number counts only with +//! filter PASS, quality 20 or more and a whole number. This is the rule of +//! `paraphase_report.parascopy_smn1`; the SMN1 marker table took the last +//! matching row before, so a table with two rows over exon 7 made the +//! marker and the Paraphase comparison read different rows. + +use crate::Error; + +/// SMN1 exon 7 (c.840, chr5:70,951,946). +pub const SMN1_EXON7: (&str, i64) = ("chr5", 70_951_946); + +/// What the SMN table says at SMN1 exon 7 and at the marker site. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SmnRead { + /// SMN1 psCN at exon 7, when resolved. + pub pscn: Option, + /// How the psCN reads, for the table. + pub text: String, + /// The aggregate SMN1 + SMN2 copies at the marker site, when resolved. + pub agcn: Option, +} + +/// One value of a `psCN`/`agCN` column (comma separated, one per region). +fn value(row: &[(&str, &str)], kind: &str, idx: usize) -> (Option, String) { + let get = |k: &str| { + row.iter() + .rev() + .find(|(h, _)| *h == k) + .map(|(_, v)| *v) + .unwrap_or("") + }; + let vals: Vec<&str> = get(kind).split(',').collect(); + let quals: Vec<&str> = get(&format!("{kind}_qual")).split(',').collect(); + let filt = match get(&format!("{kind}_filter")) { + "" => ".", + f => f, + }; + let v = vals.get(idx).copied().unwrap_or("?"); + let q = quals.get(idx).copied().unwrap_or("*"); + let ok = filt == "PASS" + && pycompat::num::parse_float(q).is_ok_and(|q| q >= 20.0) + && !v.is_empty() + && v.bytes().all(|c| c.is_ascii_digit()); + let n = if ok { v.parse().ok() } else { None }; + let text = format!( + "{v} (quality {q}, filter {filt}){}", + if n.is_some() { "" } else { ", not resolved" } + ); + (n, text) +} + +/// The row's region and its homologous regions, 1-based closed. +fn regions(row: &[(&str, &str)]) -> Option> { + let get = |k: &str| row.iter().rev().find(|(h, _)| *h == k).map(|(_, v)| *v); + let start = pycompat::num::parse_int(get("start")?).ok()?; + let end = pycompat::num::parse_int(get("end")?).ok()?; + let mut out = vec![( + get("chrom")?.to_owned(), + i64::try_from(start).ok()? + 1, + i64::try_from(end).ok()?, + )]; + for h in get("homologous_regions").unwrap_or("").split(',') { + if let Some(r) = homologous(crate::strip(h)) { + out.push(r); + } + } + Some(out) +} + +/// `^([^:]+):(\d+)-(\d+)` at the start of `h`. +fn homologous(h: &str) -> Option<(String, i64, i64)> { + let (chrom, rest) = h.split_once(':')?; + if chrom.is_empty() { + return None; + } + let digits = |s: &str| s.bytes().take_while(u8::is_ascii_digit).count(); + let n = digits(rest); + let (s, rest) = rest.split_at(n); + let rest = rest.strip_prefix('-')?; + let e = &rest[..digits(rest)]; + if s.is_empty() || e.is_empty() { + return None; + } + Some((chrom.to_owned(), s.parse().ok()?, e.parse().ok()?)) +} + +/// Reads step 35's SMN table at `path` (`None`, or a path that does not +/// exist: no table). `marker_copies` are the marker's sites on `chrom`. +pub fn read(path: Option<&str>, chrom: &str, marker_copies: &[i64]) -> Result { + let mut out = SmnRead { + pscn: None, + text: "not read (no SMN table from step 35)".to_owned(), + agcn: None, + }; + let Some(path) = path.filter(|p| !p.is_empty() && std::path::Path::new(p).exists()) else { + return Ok(out); + }; + out.text = "not read (no row of the SMN table covers SMN1 exon 7)".to_owned(); + let (mut ps_done, mut ag_done) = (false, false); + let mut header: Option> = None; + crate::each_line(path, |line| { + let Some(h) = &header else { + header = Some(line.split('\t').map(str::to_owned).collect()); + return Ok(()); + }; + let row: Vec<(&str, &str)> = h.iter().map(String::as_str).zip(line.split('\t')).collect(); + let Some(regs) = regions(&row) else { + return Ok(()); + }; + if !ps_done + && let Some(i) = regs + .iter() + .position(|(c, s, e)| c == SMN1_EXON7.0 && *s <= SMN1_EXON7.1 && SMN1_EXON7.1 <= *e) + { + (out.pscn, out.text) = value(&row, "psCN", i); + ps_done = true; + } + if !ag_done + && regs + .iter() + .any(|(c, s, e)| c == chrom && marker_copies.iter().any(|p| s <= p && p <= e)) + { + out.agcn = value(&row, "agCN", 0).0; + ag_done = true; + } + Ok(()) + })?; + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HEAD: &str = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n"; + + fn table(rows: &[&str]) -> String { + let dir = + std::env::temp_dir().join(format!("og-cn-smn-{}-{}", std::process::id(), rows.len())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join(format!("t{}.tsv", rows.join("").len())); + std::fs::write(&p, format!("{HEAD}{}", rows.join(""))).unwrap(); + p.to_string_lossy().into_owned() + } + + const SITES: [i64; 2] = [70_952_074, 70_076_654]; + + #[test] + fn the_first_row_over_exon_7_wins() { + // Two rows over exon 7: the first is unresolved, the second says 2. + // The first row's reading stands, as the Paraphase comparison reads it. + let p = table(&[ + "chr5\t70951000\t70953000\tSMN1\tPASS\t4\t30\tLowQual\t1,3\t12,40\tchr5:70075000-70078000\n", + "chr5\t70951900\t70952000\tSMN1\tPASS\t4\t30\tPASS\t2,2\t40,40\tchr5:70076000-70076100\n", + ]); + let r = read(Some(&p), "chr5", &SITES).unwrap(); + assert_eq!(r.pscn, None); + assert_eq!(r.text, "1 (quality 12, filter LowQual), not resolved"); + assert_eq!(r.agcn, Some(4)); + } + + #[test] + fn a_resolved_row_and_the_placeholders() { + let p = table(&[ + "chr5\t70925000\t70955000\tSMN1\tPASS\t4\t30\tPASS\t2\t40\tchr5:70049000-70080000\n", + ]); + let r = read(Some(&p), "chr5", &SITES).unwrap(); + assert_eq!( + (r.pscn, r.text.as_str()), + (Some(2), "2 (quality 40, filter PASS)") + ); + // The homologous copy holds exon 7 too, but psCN has one value. + let p = table(&[ + "chr5\t70049000\t70050000\tSMN2\tPASS\t4\t30\tPASS\t2\t40\tchr5:70951000-70952000\n", + ]); + let r = read(Some(&p), "chr5", &SITES).unwrap(); + assert_eq!(r.text, "? (quality *, filter PASS), not resolved"); + } + + #[test] + fn no_table_and_no_row() { + assert_eq!( + read(None, "chr5", &SITES).unwrap().text, + "not read (no SMN table from step 35)" + ); + let p = table(&["chr1\t1\t2\tX\tPASS\t2\t30\tPASS\t2\t40\t.\n"]); + assert_eq!( + read(Some(&p), "chr5", &SITES).unwrap().text, + "not read (no row of the SMN table covers SMN1 exon 7)" + ); + } +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml index f05413c7e..c55743e3f 100644 --- a/rust/og/Cargo.toml +++ b/rust/og/Cargo.toml @@ -17,6 +17,7 @@ og-vcf.workspace = true og-steps.workspace = true og-filters.workspace = true og-chip.workspace = true +og-cn.workspace = true og-runlog.workspace = true [features] diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index acc0e4476..1412515b0 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -18,7 +18,7 @@ fn tree_hash() -> &'static str { &TREE_MARK["og-tree-hash:".len()..TREE_MARK.len() - 1] } -const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og dev parity-compare KIND ORACLE NEW\n og version\n og --tree-hash\n"; +const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og cn bed|call|intervals|check-bam|marker-regions|marker-read [args]\n og dev parity-compare KIND ORACLE NEW\n og version\n og --tree-hash\n"; /// argparse's way to refuse a command line: usage and the error on stderr, exit 2. fn usage_error(msg: &str) -> ExitCode { @@ -119,6 +119,11 @@ fn main() -> ExitCode { )), None => usage_error("the following arguments are required: chip tool"), }, + "cn" => ExitCode::from(og_cn::cli::main( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), "str" => match rest.first().map(String::as_str) { #[cfg(feature = "bam")] Some("motif") => ExitCode::from(og_bam::motif::cli( diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index d71c7f6ae..b457f9e48 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -118,3 +118,77 @@ chip check-bad-table chip/bad-table.tsv check-table bad-table.tsv - bytes PYTHON chip check-bad-header chip/bad-header.tsv check-table bad-header.tsv - bytes PYTHON_IMAGE chip check-short-row chip/short-row.tsv check-table short-row.tsv - bytes PYTHON_IMAGE chip check-no-genes chip/no-genes.tsv check-table no-genes.tsv - bytes PYTHON_IMAGE +cn bed-all gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv - kv PYTHON_IMAGE +cn bed-loci gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --locus HBA --locus RHD - kv PYTHON_IMAGE +cn bed-interval gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --kind interval - kv PYTHON_IMAGE +cn bed-psv gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --psv --locus CYP21A2 - kv PYTHON_IMAGE +cn bed-psv-not-psv gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --psv --locus HBA - kv PYTHON_IMAGE +cn bed-unknown gen:cn-oracle.sh,cn/oracle.sh bed --catalog cn_loci.tsv --locus HBA --locus NOPE - kv PYTHON_IMAGE +cn hba4 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba3 gen:cn-oracle.sh,cn/oracle.sh,cn/hba3.bedcov.tsv,cn/hba3.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba3.bedcov.tsv --bedcov-mapq1 hba3.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-between gen:cn-oracle.sh,cn/oracle.sh,cn/hba-between.bedcov.tsv,cn/hba-between.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-between.bedcov.tsv --bedcov-mapq1 hba-between.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-lowflank gen:cn-oracle.sh,cn/oracle.sh,cn/hba-lowflank.bedcov.tsv,cn/hba-lowflank.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-lowflank.bedcov.tsv --bedcov-mapq1 hba-lowflank.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-disagree gen:cn-oracle.sh,cn/oracle.sh,cn/hba-disagree.bedcov.tsv,cn/hba-disagree.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-disagree.bedcov.tsv --bedcov-mapq1 hba-disagree.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/hba-mapq0.bedcov.tsv,cn/hba-mapq0.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-mapq0.bedcov.tsv --bedcov-mapq1 hba-mapq0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba0 gen:cn-oracle.sh,cn/oracle.sh,cn/hba0.bedcov.tsv,cn/hba0.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba0.bedcov.tsv --bedcov-mapq1 hba0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba5 gen:cn-oracle.sh,cn/oracle.sh,cn/hba5.bedcov.tsv,cn/hba5.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba5.bedcov.tsv --bedcov-mapq1 hba5.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-broken gen:cn-oracle.sh,cn/oracle.sh,cn/hba-broken.bedcov.tsv,cn/hba-broken.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-broken.bedcov.tsv --bedcov-mapq1 hba-broken.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus HBA --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn hba-one-empty gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/empty.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 empty.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-no-mapq1 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-wrong-regions gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-missing-file gen:cn-oracle.sh,cn/oracle.sh call --catalog cn_loci.tsv --locus HBA --bedcov nope.tsv --bedcov-mapq1 nope.tsv --table t.tsv - kv PYTHON_IMAGE +cn hba-alt-header gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv,cn/header-alt.sam call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --bam-header header-alt.sam --table t.tsv - kv PYTHON_IMAGE +cn lpa12 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa12.bedcov.tsv,cn/lpa12.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa12.bedcov.tsv --bedcov-mapq1 lpa12.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa29 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa29.bedcov.tsv,cn/lpa29.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa29.bedcov.tsv --bedcov-mapq1 lpa29.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa47 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa47.bedcov.tsv,cn/lpa47.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa47.bedcov.tsv --bedcov-mapq1 lpa47.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-lowflank gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-lowflank.bedcov.tsv,cn/lpa-lowflank.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-lowflank.bedcov.tsv --bedcov-mapq1 lpa-lowflank.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-mapq0.bedcov.tsv,cn/lpa-mapq0.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-mapq0.bedcov.tsv --bedcov-mapq1 lpa-mapq0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-floor gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-floor.bedcov.tsv,cn/lpa-floor.q1.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-floor.bedcov.tsv --bedcov-mapq1 lpa-floor.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn marker-lpa29-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa29.bedcov.tsv,cn/lpa29.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa29.bedcov.tsv --bedcov-mapq1 lpa29.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa12-kiv-notseen gen:cn-oracle.sh,cn/oracle.sh,cn/lpa12.bedcov.tsv,cn/lpa12.q1.tsv,cn/kiv-notseen.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa12.bedcov.tsv --bedcov-mapq1 lpa12.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-notseen.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa47-bad gen:cn-oracle.sh,cn/oracle.sh,cn/lpa47.bedcov.tsv,cn/lpa47.q1.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa47.bedcov.tsv --bedcov-mapq1 lpa47.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup bad.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa-mapq0-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-mapq0.bedcov.tsv,cn/lpa-mapq0.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-mapq0.bedcov.tsv --bedcov-mapq1 lpa-mapq0.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-lpa-share1996-kiv-present gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-share1996.bedcov.tsv,cn/lpa-share1996.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-share1996.bedcov.tsv --bedcov-mapq1 lpa-share1996.q1.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup kiv-present.pileup --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn marker-empty-chr6 gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --table t.tsv --marker LPA_KIV2_4925 --marker-pileup empty.tsv --marker-out n.tsv t.tsv,n.tsv kv PYTHON_IMAGE +cn dosage-hp3 gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HP --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-ccl1 gen:cn-oracle.sh,cn/oracle.sh,cn/ccl1.bedcov.tsv,cn/ccl1.q1.tsv call --catalog cn_loci.tsv --locus CCL3L1 --bedcov ccl1.bedcov.tsv --bedcov-mapq1 ccl1.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-lce0 gen:cn-oracle.sh,cn/oracle.sh,cn/lce0.bedcov.tsv,cn/lce0.q1.tsv call --catalog cn_loci.tsv --locus LCE3BC --bedcov lce0.bedcov.tsv --bedcov-mapq1 lce0.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-rhd1 gen:cn-oracle.sh,cn/oracle.sh,cn/rhd1.bedcov.tsv call --catalog cn_loci.tsv --locus RHD --bedcov rhd1.bedcov.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn dosage-rhd0 gen:cn-oracle.sh,cn/oracle.sh,cn/rhd0.bedcov.tsv call --catalog cn_loci.tsv --locus RHD --bedcov rhd0.bedcov.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-normal gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv,cn/cyp-normal.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --pileup cyp-normal.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-deletion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-deletion.bedcov.tsv,cn/cyp-deletion.q1.tsv,cn/cyp-deletion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-deletion.bedcov.tsv --bedcov-mapq1 cyp-deletion.q1.tsv --pileup cyp-deletion.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-conversion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-conversion.bedcov.tsv,cn/cyp-conversion.q1.tsv,cn/cyp-conversion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-conversion.bedcov.tsv --bedcov-mapq1 cyp-conversion.q1.tsv --pileup cyp-conversion.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-mapq0 gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-mapq0.q1.tsv,cn/cyp-normal.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-mapq0.q1.tsv --pileup cyp-normal.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-empty gen:cn-oracle.sh,cn/oracle.sh,cn/empty.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov empty.tsv --bedcov-mapq1 empty.tsv --pileup bad.pileup --table t.tsv t.tsv kv PYTHON_IMAGE +cn cyp-no-pileup gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --table t.tsv - kv PYTHON_IMAGE +cn cyp-bad-pileup gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-normal.bedcov.tsv,cn/cyp-normal.q1.tsv,cn/bad.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-normal.bedcov.tsv --bedcov-mapq1 cyp-normal.q1.tsv --pileup bad.pileup --table t.tsv - kv PYTHON_IMAGE +cn intervals-male gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex male --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-female gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex female --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-unknown gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex unknown --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-failed gen:cn-oracle.sh,cn/oracle.sh intervals --catalog cn_loci.tsv --failed --out iv.tsv iv.tsv tsv PYTHON_IMAGE +cn intervals-no-sex gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --out iv.tsv - kv PYTHON_IMAGE +cn intervals-wrong-table gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov hba4.bedcov.tsv --sex male --out iv.tsv - kv PYTHON_IMAGE +cn check-bam-alt gen:cn-oracle.sh,cn/oracle.sh,cn/header-alt.sam check-bam --header header-alt.sam - kv PYTHON_IMAGE +cn check-bam-clean gen:cn-oracle.sh,cn/oracle.sh,cn/header-clean.sam check-bam --header header-clean.sam - kv PYTHON_IMAGE +cn check-bam-stdin gen:cn-oracle.sh,cn/oracle.sh,cn/stdin check-bam --header - - kv PYTHON_IMAGE +cn marker-regions-lpa gen:cn-oracle.sh,cn/oracle.sh marker-regions LPA_KIV2_4925 - kv PYTHON_IMAGE +cn marker-regions-smn gen:cn-oracle.sh,cn/oracle.sh marker-regions SMN1_27134 - kv PYTHON_IMAGE +cn smn-smn-present-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-notseen-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-notseen.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-notseen.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn1 gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn1.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn1.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn-lowq gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn-lowq.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn-lowq.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-smn-present-smn-two-rows gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn-two-rows.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn-two-rows.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-bad-smn2 gen:cn-oracle.sh,cn/oracle.sh,cn/bad.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup bad.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PYTHON_IMAGE +cn smn-no-table gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup marker-read SMN1_27134 --pileup smn-present.pileup --out m.tsv m.tsv kv PYTHON_IMAGE +cn lpa-share1996 gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-share1996.bedcov.tsv,cn/lpa-share1996.q1.tsv,cn/kiv-present.pileup call --catalog cn_loci.tsv --locus LPA_KIV2 --bedcov lpa-share1996.bedcov.tsv --bedcov-mapq1 lpa-share1996.q1.tsv --table t.tsv t.tsv kv PYTHON_IMAGE +cn lpa-marker-read-table gen:cn-oracle.sh,cn/oracle.sh,cn/lpa-table.tsv,cn/kiv-present.pileup marker-read LPA_KIV2_4925 --pileup kiv-present.pileup --copy-table lpa-table.tsv --out n.tsv --catalog cn_loci.tsv n.tsv kv PYTHON_IMAGE +cn lpa-marker-read-no-table gen:cn-oracle.sh,cn/oracle.sh,cn/kiv-present.pileup marker-read LPA_KIV2_4925 --pileup kiv-present.pileup --out n.tsv --catalog cn_loci.tsv n.tsv kv PYTHON_IMAGE +cn py311-hba4 gen:cn-oracle.sh,cn/oracle.sh,cn/hba4.bedcov.tsv,cn/hba4.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba4.bedcov.tsv --bedcov-mapq1 hba4.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-hba-between gen:cn-oracle.sh,cn/oracle.sh,cn/hba-between.bedcov.tsv,cn/hba-between.q1.tsv call --catalog cn_loci.tsv --locus HBA --bedcov hba-between.bedcov.tsv --bedcov-mapq1 hba-between.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-dosage-hp3 gen:cn-oracle.sh,cn/oracle.sh,cn/hp3.bedcov.tsv,cn/hp3.q1.tsv call --catalog cn_loci.tsv --locus HP --bedcov hp3.bedcov.tsv --bedcov-mapq1 hp3.q1.tsv --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-cyp-conversion gen:cn-oracle.sh,cn/oracle.sh,cn/cyp-conversion.bedcov.tsv,cn/cyp-conversion.q1.tsv,cn/cyp-conversion.pileup call --catalog cn_loci.tsv --locus CYP21A2 --bedcov cyp-conversion.bedcov.tsv --bedcov-mapq1 cyp-conversion.q1.tsv --pileup cyp-conversion.pileup --table t.tsv t.tsv kv PARASCOPY_IMAGE +cn py311-intervals-male gen:cn-oracle.sh,cn/oracle.sh,cn/intervals.bedcov.tsv intervals --catalog cn_loci.tsv --bedcov intervals.bedcov.tsv --sex male --out iv.tsv iv.tsv tsv PARASCOPY_IMAGE +cn py311-smn-present gen:cn-oracle.sh,cn/oracle.sh,cn/smn-present.pileup,cn/smn2.tsv marker-read SMN1_27134 --pileup smn-present.pileup --copy-table smn2.tsv --out m.tsv m.tsv kv PARASCOPY_IMAGE +cn py311-check-bam-alt gen:cn-oracle.sh,cn/oracle.sh,cn/header-alt.sam check-bam --header header-alt.sam - kv PARASCOPY_IMAGE diff --git a/tests/parity/gen/cn-oracle.sh b/tests/parity/gen/cn-oracle.sh new file mode 100644 index 000000000..92a487dda --- /dev/null +++ b/tests/parity/gen/cn-oracle.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# The Python that og cn replaced, as it was at the commit tests/parity/oracle.ref +# names for cn, for the cn oracle (inputs/cn/oracle.sh): copy_number.py imports +# hba_copy_number.py and lpa_kiv2.py, the markers lived in traits_report.py, +# and every case reads that commit's bin/cn_loci.tsv. +set -euo pipefail +root=$(cd "$(dirname "$0")/../../.." && pwd) +sha=$(awk -F'\t' '$1 == "cn" {print $2}' "${root}/tests/parity/oracle.ref") +[ -n "$sha" ] || { echo "cn-oracle.sh: no cn row in oracle.ref" >&2; exit 1; } +git -C "$root" cat-file -e "${sha}^{commit}" 2> /dev/null || git -C "$root" fetch -q --depth=1 origin "$sha" +for f in hba_copy_number.py lpa_kiv2.py traits_report.py cn_loci.tsv; do + git -C "$root" show "${sha}:bin/${f}" > "$f" +done diff --git a/tests/parity/golden/cn/cyp-no-pileup/stderr b/tests/parity/golden/cn/cyp-no-pileup/stderr new file mode 100644 index 000000000..fb715e0ff --- /dev/null +++ b/tests/parity/golden/cn/cyp-no-pileup/stderr @@ -0,0 +1 @@ +ERROR: CYP21A2 is a psv row: --pileup (samtools mpileup over `og cn bed --psv`) is needed diff --git a/tests/parity/golden/cn/hba-wrong-regions/stderr b/tests/parity/golden/cn/hba-wrong-regions/stderr new file mode 100644 index 000000000..ce426b040 --- /dev/null +++ b/tests/parity/golden/cn/hba-wrong-regions/stderr @@ -0,0 +1 @@ +ERROR: hp3.bedcov.tsv: its regions differ from the ones `og cn bed --locus HBA` prints diff --git a/tests/parity/golden/cn/intervals-wrong-table/stderr b/tests/parity/golden/cn/intervals-wrong-table/stderr new file mode 100644 index 000000000..491caeebd --- /dev/null +++ b/tests/parity/golden/cn/intervals-wrong-table/stderr @@ -0,0 +1 @@ +ERROR: the bedcov table lacks some of 1q21.1_TAR_syndrome_proximal_BP2-BP3's segments: its regions differ from `og cn bed --kind interval` diff --git a/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv new file mode 100644 index 000000000..933da31f0 --- /dev/null +++ b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/n.tsv @@ -0,0 +1,10 @@ +metric value +marker LPA KIV-2 4925G>A +rsid rs1484919544 (first KIV-2 copy) +status not assessed +alt_reads . +depth . +fraction . +copies . +kiv2_copies . +message LPA KIV-2 4925G>A (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017): not assessed, reads of mapping quality 0 make up 20.0% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives. diff --git a/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout new file mode 100644 index 000000000..354d95fca --- /dev/null +++ b/tests/parity/golden/cn/marker-lpa-share1996-kiv-present/stdout @@ -0,0 +1,2 @@ +LPA KIV-2: not assessed: reads of mapping quality 0 make up 20.0% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM (KIV-2 exons 68.57x, single-copy exons 30.00x, MAPQ 0 share 0.200) +LPA LPA KIV-2 4925G>A (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017): not assessed, reads of mapping quality 0 make up 20.0% of the KIV-2 depth, so the aligner left out the reads that fit several copies, the ones a carrying repeat gives. diff --git a/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv new file mode 100644 index 000000000..9e7eb9363 --- /dev/null +++ b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/m.tsv @@ -0,0 +1,11 @@ +metric value +marker SMN1 g.27134T>G +rsid rs143838139 +status present +alt_reads 5 +depth 60 +fraction 0.0833 +copies 0.3 +smn1_pscn 1 (quality 12, filter LowQual), not resolved +agcn 4 +message SMN1 g.27134T>G (rs143838139): present, 5 of 60 reads carry G at the SMN1 and SMN2 copies of the site together (about 0.3 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or SMN2. SMN1 copy number: 1 (quality 12, filter LowQual), not resolved; the marker refines the carrier risk of a two-copy result only. This marker does not prove 2+0, and its absence does not rule 2+0 out. diff --git a/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout new file mode 100644 index 000000000..35fcc0935 --- /dev/null +++ b/tests/parity/golden/cn/smn-smn-present-smn-two-rows/stdout @@ -0,0 +1 @@ +SMN1 g.27134T>G (rs143838139): present, 5 of 60 reads carry G at the SMN1 and SMN2 copies of the site together (about 0.3 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or SMN2. SMN1 copy number: 1 (quality 12, filter LowQual), not resolved; the marker refines the carrier risk of a two-copy result only. This marker does not prove 2+0, and its absence does not rule 2+0 out. diff --git a/tests/parity/inputs/cn/bad.pileup b/tests/parity/inputs/cn/bad.pileup new file mode 100644 index 000000000..1a14d3e27 --- /dev/null +++ b/tests/parity/inputs/cn/bad.pileup @@ -0,0 +1 @@ +# samtools mpileup failed: 4925G>A not assessed diff --git a/tests/parity/inputs/cn/ccl1.bedcov.tsv b/tests/parity/inputs/cn/ccl1.bedcov.tsv new file mode 100644 index 000000000..012420305 --- /dev/null +++ b/tests/parity/inputs/cn/ccl1.bedcov.tsv @@ -0,0 +1,3 @@ +chr17 36112000 36148000 CCL3L_FLANK_L 1080000 +chr17 36194868 36196758 CCL3L3 28350 +chr17 36460000 36500000 CCL3L_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/ccl1.q1.tsv b/tests/parity/inputs/cn/ccl1.q1.tsv new file mode 100644 index 000000000..9003d9ad9 --- /dev/null +++ b/tests/parity/inputs/cn/ccl1.q1.tsv @@ -0,0 +1,3 @@ +chr17 36112000 36148000 CCL3L_FLANK_L 540000 +chr17 36194868 36196758 CCL3L3 14175 +chr17 36460000 36500000 CCL3L_FLANK_R 600000 diff --git a/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv b/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-conversion.pileup b/tests/parity/inputs/cn/cyp-conversion.pileup new file mode 100644 index 000000000..bb2216f14 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 12 ^]Cc+2AGCgGgGgGgGg$ IIIIIIIIIIII +chr6 32006690 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007074 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007374 N 12 ^]Gg+2AGGgGgTtTtTt$ IIIIIIIIIIII +chr6 32007686 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32007800 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32039081 N 48 ^]Cc+2AGCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 48 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-conversion.q1.tsv b/tests/parity/inputs/cn/cyp-conversion.q1.tsv new file mode 100644 index 000000000..b0cb3dfc5 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-conversion.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 72603 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv b/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv new file mode 100644 index 000000000..0b4604b7b --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 40335 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-deletion.pileup b/tests/parity/inputs/cn/cyp-deletion.pileup new file mode 100644 index 000000000..ef5331e7a --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 9 ^]Cc+2AGCgGgGgG$ IIIIIIIII +chr6 32006690 N 9 ^]Tt+2AGTaAaAaA$ IIIIIIIII +chr6 32007074 N 9 ^]Tt+2AGTtTtAaA$ IIIIIIIII +chr6 32007374 N 9 ^]Gg+2AGGgGgTtT$ IIIIIIIII +chr6 32007686 N 9 ^]Cc+2AGCcCcTtT$ IIIIIIIII +chr6 32007800 N 9 ^]Cc+2AGCcCcTtT$ IIIIIIIII +chr6 32039081 N 36 ^]Cc+2AGCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 36 ^]Tt+2AGTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 36 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 36 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 36 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 36 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-deletion.q1.tsv b/tests/parity/inputs/cn/cyp-deletion.q1.tsv new file mode 100644 index 000000000..f2528c3ac --- /dev/null +++ b/tests/parity/inputs/cn/cyp-deletion.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 36302 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/cyp-mapq0.q1.tsv b/tests/parity/inputs/cn/cyp-mapq0.q1.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-mapq0.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-normal.bedcov.tsv b/tests/parity/inputs/cn/cyp-normal.bedcov.tsv new file mode 100644 index 000000000..ac62c8687 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.bedcov.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 900000 +chr6 32006220 32008909 CYP21A1P_BODY 80670 +chr6 32038950 32041644 CYP21A2_BODY 80820 +chr6 32039080 32039081 PSV_In2G 30 +chr6 32039425 32039426 PSV_I172N 30 +chr6 32039809 32039810 PSV_V237E 30 +chr6 32040109 32040110 PSV_V281L 30 +chr6 32040420 32040421 PSV_Q318X 30 +chr6 32040534 32040535 PSV_R356W 30 +chr6 32070000 32100000 RCCX_FLANK_R 900000 diff --git a/tests/parity/inputs/cn/cyp-normal.pileup b/tests/parity/inputs/cn/cyp-normal.pileup new file mode 100644 index 000000000..72b23db42 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.pileup @@ -0,0 +1,12 @@ +chr6 32006353 N 12 ^]Cc+2AGCcCcGgGgGg$ IIIIIIIIIIII +chr6 32006690 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007074 N 12 ^]Tt+2AGTtTtAaAaAa$ IIIIIIIIIIII +chr6 32007374 N 12 ^]Gg+2AGGgGgTtTtTt$ IIIIIIIIIIII +chr6 32007686 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32007800 N 12 ^]Cc+2AGCcCcTtTtTt$ IIIIIIIIIIII +chr6 32039081 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcGgGgGgGgGgGgGgGgGgGgGgGg$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039426 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32039810 N 48 ^]Tt+2AGTtTtTtTtTtTtTtTtTtTtTtAaAaAaAaAaAaAaAaAaAaAaAa$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040110 N 48 ^]Gg+2AGGgGgGgGgGgGgGgGgGgGgGgTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040421 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII +chr6 32040535 N 48 ^]Cc+2AGCcCcCcCcCcCcCcCcCcCcCcTtTtTtTtTtTtTtTtTtTtTtTt$ IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII diff --git a/tests/parity/inputs/cn/cyp-normal.q1.tsv b/tests/parity/inputs/cn/cyp-normal.q1.tsv new file mode 100644 index 000000000..b0cb3dfc5 --- /dev/null +++ b/tests/parity/inputs/cn/cyp-normal.q1.tsv @@ -0,0 +1,10 @@ +chr6 31945000 31975000 RCCX_FLANK_L 810000 +chr6 32006220 32008909 CYP21A1P_BODY 72603 +chr6 32038950 32041644 CYP21A2_BODY 72738 +chr6 32039080 32039081 PSV_In2G 27 +chr6 32039425 32039426 PSV_I172N 27 +chr6 32039809 32039810 PSV_V237E 27 +chr6 32040109 32040110 PSV_V281L 27 +chr6 32040420 32040421 PSV_Q318X 27 +chr6 32040534 32040535 PSV_R356W 27 +chr6 32070000 32100000 RCCX_FLANK_R 810000 diff --git a/tests/parity/inputs/cn/empty.tsv b/tests/parity/inputs/cn/empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/cn/hba-between.bedcov.tsv b/tests/parity/inputs/cn/hba-between.bedcov.tsv new file mode 100644 index 000000000..988dc3fe5 --- /dev/null +++ b/tests/parity/inputs/cn/hba-between.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-between.q1.tsv b/tests/parity/inputs/cn/hba-between.q1.tsv new file mode 100644 index 000000000..445c896d2 --- /dev/null +++ b/tests/parity/inputs/cn/hba-between.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 14091 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba-broken.bedcov.tsv b/tests/parity/inputs/cn/hba-broken.bedcov.tsv new file mode 100644 index 000000000..16000c2ed --- /dev/null +++ b/tests/parity/inputs/cn/hba-broken.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 0 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-broken.q1.tsv b/tests/parity/inputs/cn/hba-broken.q1.tsv new file mode 100644 index 000000000..e8c4e8b7b --- /dev/null +++ b/tests/parity/inputs/cn/hba-broken.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 0 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba-disagree.bedcov.tsv b/tests/parity/inputs/cn/hba-disagree.bedcov.tsv new file mode 100644 index 000000000..20b8343f3 --- /dev/null +++ b/tests/parity/inputs/cn/hba-disagree.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 600000 diff --git a/tests/parity/inputs/cn/hba-disagree.q1.tsv b/tests/parity/inputs/cn/hba-disagree.q1.tsv new file mode 100644 index 000000000..7b20f3051 --- /dev/null +++ b/tests/parity/inputs/cn/hba-disagree.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 450000 diff --git a/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv b/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv new file mode 100644 index 000000000..f55852795 --- /dev/null +++ b/tests/parity/inputs/cn/hba-lowflank.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 240000 +chr16 172875 173710 HBA2 4175 +chr16 176679 177522 HBA1 4215 +chr16 190000 230000 flank_right 240000 diff --git a/tests/parity/inputs/cn/hba-lowflank.q1.tsv b/tests/parity/inputs/cn/hba-lowflank.q1.tsv new file mode 100644 index 000000000..4e340cfde --- /dev/null +++ b/tests/parity/inputs/cn/hba-lowflank.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 180000 +chr16 172875 173710 HBA2 3131 +chr16 176679 177522 HBA1 3161 +chr16 190000 230000 flank_right 180000 diff --git a/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv b/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv new file mode 100644 index 000000000..c04168f01 --- /dev/null +++ b/tests/parity/inputs/cn/hba-mapq0.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba-mapq0.q1.tsv b/tests/parity/inputs/cn/hba-mapq0.q1.tsv new file mode 100644 index 000000000..b0950dddc --- /dev/null +++ b/tests/parity/inputs/cn/hba-mapq0.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1188000 +chr16 172875 173710 HBA2 24800 +chr16 176679 177522 HBA1 25037 +chr16 190000 230000 flank_right 1188000 diff --git a/tests/parity/inputs/cn/hba0.bedcov.tsv b/tests/parity/inputs/cn/hba0.bedcov.tsv new file mode 100644 index 000000000..e88ac074c --- /dev/null +++ b/tests/parity/inputs/cn/hba0.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 0 +chr16 176679 177522 HBA1 0 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba0.q1.tsv b/tests/parity/inputs/cn/hba0.q1.tsv new file mode 100644 index 000000000..e88ac074c --- /dev/null +++ b/tests/parity/inputs/cn/hba0.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 0 +chr16 176679 177522 HBA1 0 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba3.bedcov.tsv b/tests/parity/inputs/cn/hba3.bedcov.tsv new file mode 100644 index 000000000..3828c69a2 --- /dev/null +++ b/tests/parity/inputs/cn/hba3.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 12525 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba3.q1.tsv b/tests/parity/inputs/cn/hba3.q1.tsv new file mode 100644 index 000000000..4392cb0fd --- /dev/null +++ b/tests/parity/inputs/cn/hba3.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 9394 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba4.bedcov.tsv b/tests/parity/inputs/cn/hba4.bedcov.tsv new file mode 100644 index 000000000..c04168f01 --- /dev/null +++ b/tests/parity/inputs/cn/hba4.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 25050 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba4.q1.tsv b/tests/parity/inputs/cn/hba4.q1.tsv new file mode 100644 index 000000000..34a3baeb5 --- /dev/null +++ b/tests/parity/inputs/cn/hba4.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 18788 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/hba5.bedcov.tsv b/tests/parity/inputs/cn/hba5.bedcov.tsv new file mode 100644 index 000000000..860ea454b --- /dev/null +++ b/tests/parity/inputs/cn/hba5.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 1200000 +chr16 172875 173710 HBA2 37575 +chr16 176679 177522 HBA1 25290 +chr16 190000 230000 flank_right 1200000 diff --git a/tests/parity/inputs/cn/hba5.q1.tsv b/tests/parity/inputs/cn/hba5.q1.tsv new file mode 100644 index 000000000..f8980fade --- /dev/null +++ b/tests/parity/inputs/cn/hba5.q1.tsv @@ -0,0 +1,4 @@ +chr16 100000 140000 flank_left 900000 +chr16 172875 173710 HBA2 28181 +chr16 176679 177522 HBA1 18968 +chr16 190000 230000 flank_right 900000 diff --git a/tests/parity/inputs/cn/header-alt.sam b/tests/parity/inputs/cn/header-alt.sam new file mode 100644 index 000000000..fb5b18f5c --- /dev/null +++ b/tests/parity/inputs/cn/header-alt.sam @@ -0,0 +1,5 @@ +@HD VN:1.6 +@SQ SN:chr1 LN:248956422 +@SQ SN:chr1_KI270706v1_random LN:175055 +@SQ SN:chr6_GL000250v2_alt LN:4672374 +@SQ SN:HLA-A*01:01:01:01 LN:3503 diff --git a/tests/parity/inputs/cn/header-clean.sam b/tests/parity/inputs/cn/header-clean.sam new file mode 100644 index 000000000..2ee1fce7e --- /dev/null +++ b/tests/parity/inputs/cn/header-clean.sam @@ -0,0 +1,4 @@ +@HD VN:1.6 +@SQ SN:chr1 LN:248956422 +@SQ SN:chrM LN:16569 +@PG ID:x PN:HLA-x_alt diff --git a/tests/parity/inputs/cn/hp3.bedcov.tsv b/tests/parity/inputs/cn/hp3.bedcov.tsv new file mode 100644 index 000000000..4f4acb262 --- /dev/null +++ b/tests/parity/inputs/cn/hp3.bedcov.tsv @@ -0,0 +1,4 @@ +chr16 72005000 72045000 HP_FLANK_L 1200000 +chr16 72056529 72057466 HP_UNIT1 21082 +chr16 72058253 72059190 HP_UNIT2 21082 +chr16 72085000 72125000 HP_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/hp3.q1.tsv b/tests/parity/inputs/cn/hp3.q1.tsv new file mode 100644 index 000000000..0f4fcbe43 --- /dev/null +++ b/tests/parity/inputs/cn/hp3.q1.tsv @@ -0,0 +1,4 @@ +chr16 72005000 72045000 HP_FLANK_L 600000 +chr16 72056529 72057466 HP_UNIT1 10541 +chr16 72058253 72059190 HP_UNIT2 10541 +chr16 72085000 72125000 HP_FLANK_R 600000 diff --git a/tests/parity/inputs/cn/intervals.bedcov.tsv b/tests/parity/inputs/cn/intervals.bedcov.tsv new file mode 100644 index 000000000..ada5c732f --- /dev/null +++ b/tests/parity/inputs/cn/intervals.bedcov.tsv @@ -0,0 +1,946 @@ +chr1 143034587 143134587 1q21.1_TAR_syndrome_proximal_BP2-BP3_L 3000000 +chr1 145686999 145809382 1q21.1_TAR_syndrome_proximal_BP2-BP3_t1 1835745 +chr1 145823828 145923649 1q21.1_TAR_syndrome_proximal_BP2-BP3_t2 1497315 +chr1 145927476 146048495 1q21.1_TAR_syndrome_proximal_BP2-BP3_t3 1815285 +chr1 149926380 150026380 1q21.1_TAR_syndrome_proximal_BP2-BP3_R 3000000 +chr1 143034587 143134587 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_L 3000000 +chr1 145687000 145809382 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t1 3671460 +chr1 145823828 145923649 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t2 2994630 +chr1 145927476 146048496 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t3 3630600 +chr1 147105903 147473859 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t4 11038680 +chr1 147473859 147841815 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t5 11038680 +chr1 147842996 147917509 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t6 2235390 +chr1 149926380 150026380 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_R 3000000 +chr1 143034587 143134587 1q21.1_distal_BP3-BP4_L 3000000 +chr1 147105904 147473860 1q21.1_distal_BP3-BP4_t1 11038680 +chr1 147473860 147841815 1q21.1_distal_BP3-BP4_t2 11038650 +chr1 147842996 147917509 1q21.1_distal_BP3-BP4_t3 2235390 +chr1 149926380 150026380 1q21.1_distal_BP3-BP4_R 3000000 +chr2 86554206 86654206 2p11.2_L 3000000 +chr2 88022824 88389269 2p11.2_t1 10993350 +chr2 88389269 88755713 2p11.2_t2 10993320 +chr2 90405714 90505714 2p11.2_R 3000000 +chr2 95249415 95349415 2q11.2_L 3000000 +chr2 96073264 96539478 2q11.2_t1 13986420 +chr2 96539478 97005692 2q11.2_t2 13986420 +chr2 97693145 97793145 2q11.2_R 3000000 +chr2 109561396 109661396 2q13_proximal_L 3000000 +chr2 110104531 110185037 2q13_proximal_t1 3622770 +chr2 110186837 110228181 2q13_proximal_t2 1860480 +chr2 112509875 112609875 2q13_proximal_R 3000000 +chr2 109561396 109661396 2q13_distal_L 3000000 +chr2 110634616 110635231 2q13_distal_t1 18450 +chr2 110635407 110695526 2q13_distal_t2 1803570 +chr2 110699475 110967774 2q13_distal_t3 8048970 +chr2 110968796 111223194 2q13_distal_t4 7631940 +chr2 111225336 111252925 2q13_distal_t5 827670 +chr2 111519068 111525222 2q13_distal_t6 184620 +chr2 111615296 111719716 2q13_distal_t7 3132600 +chr2 111882691 112278330 2q13_distal_t8 11869170 +chr2 112287893 112291400 2q13_distal_t9 105210 +chr2 112293808 112347165 2q13_distal_t10 1600710 +chr2 112509875 112609875 2q13_distal_R 3000000 +chr2 129811225 129911225 2q21.1_L 3000000 +chr2 130719936 131172120 2q21.1_t1 13565520 +chr2 132414062 132514062 2q21.1_R 3000000 +chr3 195370170 195470170 3q29_L 3000000 +chr3 196029183 196395167 3q29_t1 10979520 +chr3 196395167 196761150 3q29_t2 10979490 +chr3 196761870 196828315 3q29_t3 1993350 +chr3 196832644 196986594 3q29_t4 4618500 +chr3 196987637 197032456 3q29_t5 1344570 +chr3 197035076 197436093 3q29_t6 12030510 +chr3 197438625 197617791 3q29_t7 5374980 +chr3 197770554 197870554 3q29_R 3000000 +chr5 175752456 175852456 5q35_Sotos_syndrome_L 0 +chr5 176301976 176346064 5q35_Sotos_syndrome_t1 1322640 +chr5 176348356 176609606 5q35_Sotos_syndrome_t2 7837500 +chr5 176609606 176870856 5q35_Sotos_syndrome_t3 7837500 +chr5 176873194 177156949 5q35_Sotos_syndrome_t4 8512650 +chr5 177156949 177440704 5q35_Sotos_syndrome_t5 8512650 +chr5 177442622 177620792 5q35_Sotos_syndrome_t6 5345100 +chr5 178185883 178285883 5q35_Sotos_syndrome_R 3000000 +chr7 72415064 72515064 7q11.23_Williams-Beuren_syndrome_L 3000000 +chr7 73330452 73373003 7q11.23_Williams-Beuren_syndrome_t1 1276530 +chr7 73380068 73398750 7q11.23_Williams-Beuren_syndrome_t2 560460 +chr7 73417318 73440409 7q11.23_Williams-Beuren_syndrome_t3 692730 +chr7 73442040 73900880 7q11.23_Williams-Beuren_syndrome_t4 13765200 +chr7 73911711 73943245 7q11.23_Williams-Beuren_syndrome_t5 946020 +chr7 73973000 73982740 7q11.23_Williams-Beuren_syndrome_t6 292200 +chr7 73990695 73993203 7q11.23_Williams-Beuren_syndrome_t7 75240 +chr7 73995261 74194738 7q11.23_Williams-Beuren_syndrome_t8 5984310 +chr7 74197101 74536524 7q11.23_Williams-Beuren_syndrome_t9 10182690 +chr7 74565485 74567048 7q11.23_Williams-Beuren_syndrome_t10 46890 +chr7 74571222 74576545 7q11.23_Williams-Beuren_syndrome_t11 159690 +chr7 74599408 74650287 7q11.23_Williams-Beuren_syndrome_t12 1526370 +chr7 74650795 74728173 7q11.23_Williams-Beuren_syndrome_t13 2321340 +chr7 77185726 77285726 7q11.23_Williams-Beuren_syndrome_R 3000000 +chr7 72415790 72515790 7q11.23_L 3000000 +chr7 75528718 75922712 7q11.23_t1 11819820 +chr7 75924249 76071493 7q11.23_t2 4417320 +chr7 76073808 76087747 7q11.23_t3 418170 +chr7 76105300 76105897 7q11.23_t4 17910 +chr7 76177734 76433859 7q11.23_t5 7683750 +chr7 77229655 77329655 7q11.23_R 3000000 +chr8 6826263 6926263 8p23.1_L 3000000 +chr8 8242542 8631046 8p23.1_t1 11655120 +chr8 8631046 9019550 8p23.1_t2 11655120 +chr8 9019550 9408054 8p23.1_t3 11655120 +chr8 9408054 9796558 8p23.1_t4 11655120 +chr8 9799208 10151922 8p23.1_t5 10581420 +chr8 10151922 10504636 8p23.1_t6 10581420 +chr8 10506824 10918077 8p23.1_t7 12337590 +chr8 10918077 11329329 8p23.1_t8 12337560 +chr8 11332201 11624615 8p23.1_t9 8772420 +chr8 11657572 11758866 8p23.1_t10 3038820 +chr8 11760002 11790306 8p23.1_t11 909120 +chr8 11793265 11908820 8p23.1_t12 3466650 +chr8 12769873 12869873 8p23.1_R 3000000 +chr10 45521834 45621834 10q11.2_LCR_C-LCR_D_L 3000000 +chr10 48181660 48588063 10q11.2_LCR_C-LCR_D_t1 12192090 +chr10 48588063 48994466 10q11.2_LCR_C-LCR_D_t2 12192090 +chr10 48994466 49400869 10q11.2_LCR_C-LCR_D_t3 12192090 +chr10 49403230 49515152 10q11.2_LCR_C-LCR_D_t4 3357660 +chr10 49517557 49771065 10q11.2_LCR_C-LCR_D_t5 7605240 +chr10 49774433 49778835 10q11.2_LCR_C-LCR_D_t6 132060 +chr10 49781219 49845537 10q11.2_LCR_C-LCR_D_t7 1929540 +chr10 50834350 50934350 10q11.2_LCR_C-LCR_D_R 3000000 +chr10 45521834 45621834 10q11.2_LCR_C-LCR_F_L 3000000 +chr10 48181661 48588064 10q11.2_LCR_C-LCR_F_t1 12192090 +chr10 48588064 48994467 10q11.2_LCR_C-LCR_F_t2 12192090 +chr10 48994467 49400869 10q11.2_LCR_C-LCR_F_t3 12192060 +chr10 49403230 49515152 10q11.2_LCR_C-LCR_F_t4 3357660 +chr10 49517557 49771065 10q11.2_LCR_C-LCR_F_t5 7605240 +chr10 49774433 49778835 10q11.2_LCR_C-LCR_F_t6 132060 +chr10 49781219 49845538 10q11.2_LCR_C-LCR_F_t7 1929570 +chr10 50201000 50628504 10q11.2_LCR_C-LCR_F_t8 12825120 +chr10 50630070 50655358 10q11.2_LCR_C-LCR_F_t9 758640 +chr10 50834350 50934350 10q11.2_LCR_C-LCR_F_R 3000000 +chr10 79400803 79500803 10q22.3q23.2_L 3000000 +chr10 79923087 79982995 10q22.3q23.2_t1 1797240 +chr10 79987131 80009954 10q22.3q23.2_t2 684690 +chr10 80013908 80018239 10q22.3q23.2_t3 129930 +chr10 80045634 80081133 10q22.3q23.2_t4 1064970 +chr10 80103139 80131070 10q22.3q23.2_t5 837930 +chr10 80147027 80209920 10q22.3q23.2_t6 1886790 +chr10 80258685 80700907 10q22.3q23.2_t7 13266660 +chr10 80700907 81143128 10q22.3q23.2_t8 13266630 +chr10 81145195 81603712 10q22.3q23.2_t9 13755510 +chr10 81603712 82062229 10q22.3q23.2_t10 13755510 +chr10 82062229 82520746 10q22.3q23.2_t11 13755510 +chr10 82520746 82979263 10q22.3q23.2_t12 13755510 +chr10 82979263 83437780 10q22.3q23.2_t13 13755510 +chr10 83437780 83896292 10q22.3q23.2_t14 13755360 +chr10 83897369 84081413 10q22.3q23.2_t15 5521320 +chr10 84083161 84390905 10q22.3q23.2_t16 9232320 +chr10 84393707 84679573 10q22.3q23.2_t17 8575980 +chr10 84679573 84965438 10q22.3q23.2_t18 8575950 +chr10 84966462 84968275 10q22.3q23.2_t19 54390 +chr10 84969639 85315330 10q22.3q23.2_t20 10370730 +chr10 85315330 85661021 10q22.3q23.2_t21 10370730 +chr10 85661021 86006710 10q22.3q23.2_t22 10370670 +chr10 86008355 86465864 10q22.3q23.2_t23 13725270 +chr10 86465864 86923373 10q22.3q23.2_t24 13725270 +chr10 86925182 86979631 10q22.3q23.2_t25 1633470 +chr10 87550607 87650607 10q22.3q23.2_R 3000000 +chr11 67556564 67656564 11q13.2q13.4_L 3000000 +chr11 67996175 68021419 11q13.2q13.4_t1 757320 +chr11 68022783 68303534 11q13.2q13.4_t2 8422530 +chr11 68332191 68356675 11q13.2q13.4_t3 734520 +chr11 68392474 68403076 11q13.2q13.4_t4 318060 +chr11 68407506 68890756 11q13.2q13.4_t5 14497500 +chr11 68890756 69374006 11q13.2q13.4_t6 14497500 +chr11 69374006 69857256 11q13.2q13.4_t7 14497500 +chr11 69857256 70340504 11q13.2q13.4_t8 14497440 +chr11 70341528 70344601 11q13.2q13.4_t9 92190 +chr11 70345623 70643160 11q13.2q13.4_t10 8926110 +chr11 70643160 70940696 11q13.2q13.4_t11 8926080 +chr11 70955696 71055696 11q13.2q13.4_t12 3000000 +chr11 71070696 71116951 11q13.2q13.4_t13 1387650 +chr11 71165977 71525885 11q13.2q13.4_t14 10797240 +chr11 71963397 72063397 11q13.2q13.4_R 3000000 +chr13 22651590 22751590 13q12.12_L 3000000 +chr13 22970443 23134172 13q12.12_t1 4911870 +chr13 23135887 23244684 13q12.12_t2 3263910 +chr13 23250456 23422194 13q12.12_t3 5152140 +chr13 23424929 23713633 13q12.12_t4 8661120 +chr13 23715919 23716442 13q12.12_t5 15690 +chr13 23716630 23880085 13q12.12_t6 4903650 +chr13 23904739 23938589 13q12.12_t7 1015500 +chr13 23939257 23940551 13q12.12_t8 38820 +chr13 23946299 23953512 13q12.12_t9 216390 +chr13 23956049 24121032 13q12.12_t10 4949490 +chr13 24123815 24307222 13q12.12_t11 5502210 +chr13 24684489 24784489 13q12.12_R 3000000 +chr15 22782170 23040134 15q11.2_BP1-BP2_t1 7738920 +chr15 34647508 34747508 15q11.2_BP1-BP2_R 3000000 +chr15 22782170 23040135 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t1 7738950 +chr15 23465364 23685679 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t2 6609450 +chr15 23687303 24010500 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t3 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t4 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t5 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t6 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t7 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t8 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t9 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t10 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t11 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t12 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t13 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t14 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t15 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t16 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t17 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t18 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t19 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t20 3099480 +chr15 27999198 28134728 15q11.2q13_PWS_AS_Class_I_BP1-BP3_t21 4065900 +chr15 34647508 34747508 15q11.2q13_PWS_AS_Class_I_BP1-BP3_R 3000000 +chr15 22782171 23040135 15q11.2q13_PWS_AS_BP1-BP4_t1 7738920 +chr15 23465364 23685679 15q11.2q13_PWS_AS_BP1-BP4_t2 6609450 +chr15 23687303 24010500 15q11.2q13_PWS_AS_BP1-BP4_t3 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_BP1-BP4_t4 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_BP1-BP4_t5 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_BP1-BP4_t6 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_BP1-BP4_t7 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_BP1-BP4_t8 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_BP1-BP4_t9 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_BP1-BP4_t10 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_BP1-BP4_t11 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_BP1-BP4_t12 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_BP1-BP4_t13 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_BP1-BP4_t14 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_BP1-BP4_t15 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_BP1-BP4_t16 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_BP1-BP4_t17 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_BP1-BP4_t18 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_BP1-BP4_t19 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_BP1-BP4_t20 3099480 +chr15 27999198 28134729 15q11.2q13_PWS_AS_BP1-BP4_t21 4065930 +chr15 28916885 29053771 15q11.2q13_PWS_AS_BP1-BP4_t22 4106580 +chr15 29055911 29065471 15q11.2q13_PWS_AS_BP1-BP4_t23 286800 +chr15 29069337 29228978 15q11.2q13_PWS_AS_BP1-BP4_t24 4789230 +chr15 29233760 29730610 15q11.2q13_PWS_AS_BP1-BP4_t25 14905500 +chr15 29731821 30005618 15q11.2q13_PWS_AS_BP1-BP4_t26 8213910 +chr15 30006801 30044722 15q11.2q13_PWS_AS_BP1-BP4_t27 1137630 +chr15 30046094 30076787 15q11.2q13_PWS_AS_BP1-BP4_t28 920790 +chr15 34647508 34747508 15q11.2q13_PWS_AS_BP1-BP4_R 3000000 +chr15 23465365 23685679 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t1 6609420 +chr15 23687303 24010500 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t2 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t3 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t4 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t5 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t6 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t7 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t8 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t9 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t10 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t11 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t12 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t13 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t14 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t15 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t16 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t17 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t18 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t19 3099480 +chr15 27999198 28134728 15q11.2q13_PWS_AS_Class_II_BP2-BP3_t20 4065900 +chr15 34647508 34747508 15q11.2q13_PWS_AS_Class_II_BP2-BP3_R 3000000 +chr15 23465366 23685679 15q11.2q13_PWS_AS_BP2-BP4_t1 6609390 +chr15 23687303 24010500 15q11.2q13_PWS_AS_BP2-BP4_t2 9695910 +chr15 24011064 24087204 15q11.2q13_PWS_AS_BP2-BP4_t3 2284200 +chr15 24154836 24157282 15q11.2q13_PWS_AS_BP2-BP4_t4 73380 +chr15 24250757 24256707 15q11.2q13_PWS_AS_BP2-BP4_t5 178500 +chr15 24495017 24496953 15q11.2q13_PWS_AS_BP2-BP4_t6 58080 +chr15 24578602 24754009 15q11.2q13_PWS_AS_BP2-BP4_t7 5262210 +chr15 24754335 24908854 15q11.2q13_PWS_AS_BP2-BP4_t8 4635570 +chr15 24909880 25167285 15q11.2q13_PWS_AS_BP2-BP4_t9 7722150 +chr15 25169694 25269652 15q11.2q13_PWS_AS_BP2-BP4_t10 2998740 +chr15 25271058 25370565 15q11.2q13_PWS_AS_BP2-BP4_t11 2985210 +chr15 25371814 25659393 15q11.2q13_PWS_AS_BP2-BP4_t12 8627370 +chr15 25659393 25946971 15q11.2q13_PWS_AS_BP2-BP4_t13 8627340 +chr15 25951178 26435552 15q11.2q13_PWS_AS_BP2-BP4_t14 14531220 +chr15 26435552 26919926 15q11.2q13_PWS_AS_BP2-BP4_t15 14531220 +chr15 26919926 27404300 15q11.2q13_PWS_AS_BP2-BP4_t16 14531220 +chr15 27404300 27888672 15q11.2q13_PWS_AS_BP2-BP4_t17 14531160 +chr15 27889772 27894580 15q11.2q13_PWS_AS_BP2-BP4_t18 144240 +chr15 27895234 27998550 15q11.2q13_PWS_AS_BP2-BP4_t19 3099480 +chr15 27999198 28134729 15q11.2q13_PWS_AS_BP2-BP4_t20 4065930 +chr15 28916885 29053771 15q11.2q13_PWS_AS_BP2-BP4_t21 4106580 +chr15 29055911 29065471 15q11.2q13_PWS_AS_BP2-BP4_t22 286800 +chr15 29069337 29228978 15q11.2q13_PWS_AS_BP2-BP4_t23 4789230 +chr15 29233760 29730610 15q11.2q13_PWS_AS_BP2-BP4_t24 14905500 +chr15 29731821 30005618 15q11.2q13_PWS_AS_BP2-BP4_t25 8213910 +chr15 30006801 30044722 15q11.2q13_PWS_AS_BP2-BP4_t26 1137630 +chr15 30046094 30076787 15q11.2q13_PWS_AS_BP2-BP4_t27 920790 +chr15 34647508 34747508 15q11.2q13_PWS_AS_BP2-BP4_R 3000000 +chr15 28916886 29053771 15q13_BP3-BP4_t1 4106550 +chr15 29055911 29065471 15q13_BP3-BP4_t2 286800 +chr15 29069337 29228978 15q13_BP3-BP4_t3 4789230 +chr15 29233760 29730610 15q13_BP3-BP4_t4 14905500 +chr15 29731821 30005618 15q13_BP3-BP4_t5 8213910 +chr15 30006801 30044722 15q13_BP3-BP4_t6 1137630 +chr15 30046094 30076787 15q13_BP3-BP4_t7 920790 +chr15 34647508 34747508 15q13_BP3-BP4_R 3000000 +chr15 28916887 29053771 15q13.1q13.3_BP3-BP5_t1 4106520 +chr15 29055911 29065471 15q13.1q13.3_BP3-BP5_t2 286800 +chr15 29069337 29228978 15q13.1q13.3_BP3-BP5_t3 4789230 +chr15 29233760 29730610 15q13.1q13.3_BP3-BP5_t4 14905500 +chr15 29731821 30005618 15q13.1q13.3_BP3-BP5_t5 8213910 +chr15 30006801 30044722 15q13.1q13.3_BP3-BP5_t6 1137630 +chr15 30046094 30076788 15q13.1q13.3_BP3-BP5_t7 920820 +chr15 30900685 31259237 15q13.1q13.3_BP3-BP5_t8 10756560 +chr15 31259237 31617789 15q13.1q13.3_BP3-BP5_t9 10756560 +chr15 31735058 31764141 15q13.1q13.3_BP3-BP5_t10 872490 +chr15 31768318 32153204 15q13.1q13.3_BP3-BP5_t11 11546580 +chr15 34647508 34747508 15q13.1q13.3_BP3-BP5_R 3000000 +chr15 30900686 31259238 15q13.3_BP4-BP5_t1 10756560 +chr15 31259238 31617789 15q13.3_BP4-BP5_t2 10756530 +chr15 31735058 31764141 15q13.3_BP4-BP5_t3 872490 +chr15 31768318 32153204 15q13.3_BP4-BP5_t4 11546580 +chr15 34647508 34747508 15q13.3_BP4-BP5_R 3000000 +chr15 31735058 31764141 15q13.3_D-CHRNA7_to_BP5_t1 872490 +chr15 31768318 32153204 15q13.3_D-CHRNA7_to_BP5_t2 11546580 +chr15 34647508 34747508 15q13.3_D-CHRNA7_to_BP5_R 3000000 +chr15 32607509 33012322 15q13.3q14_BP5-BP6_t1 12144390 +chr15 33013398 33468532 15q13.3q14_BP5-BP6_t2 13654020 +chr15 33468532 33923666 15q13.3q14_BP5-BP6_t3 13654020 +chr15 33923666 34378800 15q13.3q14_BP5-BP6_t4 13654020 +chr15 34690717 34790717 15q13.3q14_BP5-BP6_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_C_L 3000000 +chr15 72671374 72682053 15q24_LCR_A-LCR_C_t1 320370 +chr15 72683710 72899078 15q24_LCR_A-LCR_C_t2 6461040 +chr15 72900361 73284156 15q24_LCR_A-LCR_C_t3 11513850 +chr15 73284156 73667951 15q24_LCR_A-LCR_C_t4 11513850 +chr15 73667951 74051745 15q24_LCR_A-LCR_C_t5 11513820 +chr15 74120301 74595608 15q24_LCR_A-LCR_C_t6 14259210 +chr15 74598046 74907009 15q24_LCR_A-LCR_C_t7 9268890 +chr15 74907009 75215971 15q24_LCR_A-LCR_C_t8 9268860 +chr15 78050834 78150834 15q24_LCR_A-LCR_C_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_D_L 3000000 +chr15 72671374 72682053 15q24_LCR_A-LCR_D_t1 320370 +chr15 72683710 72899078 15q24_LCR_A-LCR_D_t2 6461040 +chr15 72900361 73284156 15q24_LCR_A-LCR_D_t3 11513850 +chr15 73284156 73667951 15q24_LCR_A-LCR_D_t4 11513850 +chr15 73667951 74051745 15q24_LCR_A-LCR_D_t5 11513820 +chr15 74120301 74595608 15q24_LCR_A-LCR_D_t6 14259210 +chr15 74598046 74907009 15q24_LCR_A-LCR_D_t7 9268890 +chr15 74907009 75215972 15q24_LCR_A-LCR_D_t8 9268890 +chr15 75339445 75381921 15q24_LCR_A-LCR_D_t9 1274280 +chr15 75386986 75480313 15q24_LCR_A-LCR_D_t10 2799810 +chr15 75482930 75512491 15q24_LCR_A-LCR_D_t11 886830 +chr15 75514448 75680568 15q24_LCR_A-LCR_D_t12 4983600 +chr15 78050834 78150834 15q24_LCR_A-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_A-LCR_B_L 3000000 +chr15 72671375 72682053 15q24_LCR_A-LCR_B_t1 320340 +chr15 72683710 72899078 15q24_LCR_A-LCR_B_t2 6461040 +chr15 72900361 73284155 15q24_LCR_A-LCR_B_t3 11513820 +chr15 73284155 73667949 15q24_LCR_A-LCR_B_t4 11513820 +chr15 73667949 74051743 15q24_LCR_A-LCR_B_t5 11513820 +chr15 78050834 78150834 15q24_LCR_A-LCR_B_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_C_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_C_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_C_t2 9268890 +chr15 74907009 75215971 15q24_LCR_B-LCR_C_t3 9268860 +chr15 78050834 78150834 15q24_LCR_B-LCR_C_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_D_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_D_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_D_t2 9268890 +chr15 74907009 75215972 15q24_LCR_B-LCR_D_t3 9268890 +chr15 75339445 75381921 15q24_LCR_B-LCR_D_t4 1274280 +chr15 75386986 75480313 15q24_LCR_B-LCR_D_t5 2799810 +chr15 75482930 75512491 15q24_LCR_B-LCR_D_t6 886830 +chr15 75514448 75680568 15q24_LCR_B-LCR_D_t7 4983600 +chr15 78050834 78150834 15q24_LCR_B-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_B-LCR_E_L 3000000 +chr15 74120302 74595608 15q24_LCR_B-LCR_E_t1 14259180 +chr15 74598046 74907009 15q24_LCR_B-LCR_E_t2 9268890 +chr15 74907009 75215972 15q24_LCR_B-LCR_E_t3 9268890 +chr15 75339445 75381921 15q24_LCR_B-LCR_E_t4 1274280 +chr15 75386986 75480313 15q24_LCR_B-LCR_E_t5 2799810 +chr15 75482930 75512491 15q24_LCR_B-LCR_E_t6 886830 +chr15 75514448 75680569 15q24_LCR_B-LCR_E_t7 4983630 +chr15 75822911 75846817 15q24_LCR_B-LCR_E_t8 717180 +chr15 75850869 75881298 15q24_LCR_B-LCR_E_t9 912870 +chr15 75882410 76257102 15q24_LCR_B-LCR_E_t10 11240760 +chr15 76259376 76619173 15q24_LCR_B-LCR_E_t11 10793910 +chr15 76619173 76978969 15q24_LCR_B-LCR_E_t12 10793880 +chr15 76980529 77278454 15q24_LCR_B-LCR_E_t13 8937750 +chr15 77279850 77314069 15q24_LCR_B-LCR_E_t14 1026570 +chr15 77315649 77559142 15q24_LCR_B-LCR_E_t15 7304790 +chr15 77562827 77858909 15q24_LCR_B-LCR_E_t16 8882460 +chr15 78050834 78150834 15q24_LCR_B-LCR_E_R 3000000 +chr15 72455531 72555531 15q24_LCR_C-LCR_D_L 3000000 +chr15 75339446 75381921 15q24_LCR_C-LCR_D_t1 1274250 +chr15 75386986 75480313 15q24_LCR_C-LCR_D_t2 2799810 +chr15 75482930 75512491 15q24_LCR_C-LCR_D_t3 886830 +chr15 75514448 75680568 15q24_LCR_C-LCR_D_t4 4983600 +chr15 78050834 78150834 15q24_LCR_C-LCR_D_R 3000000 +chr15 72455531 72555531 15q24_LCR_D-LCR_E_L 3000000 +chr15 75822911 75846817 15q24_LCR_D-LCR_E_t1 717180 +chr15 75850869 75881298 15q24_LCR_D-LCR_E_t2 912870 +chr15 75882410 76257102 15q24_LCR_D-LCR_E_t3 11240760 +chr15 76259376 76619173 15q24_LCR_D-LCR_E_t4 10793910 +chr15 76619173 76978969 15q24_LCR_D-LCR_E_t5 10793880 +chr15 76980529 77278454 15q24_LCR_D-LCR_E_t6 8937750 +chr15 77279850 77314069 15q24_LCR_D-LCR_E_t7 1026570 +chr15 77315649 77559142 15q24_LCR_D-LCR_E_t8 7304790 +chr15 77562827 77858909 15q24_LCR_D-LCR_E_t9 8882460 +chr15 78050834 78150834 15q24_LCR_D-LCR_E_R 3000000 +chr15 82040662 82140662 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_L 3000000 +chr15 82534139 82613798 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t1 2389770 +chr15 82614158 82809649 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t2 5864730 +chr15 82827843 83233890 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t3 12181410 +chr15 83233890 83639937 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t4 12181410 +chr15 83639937 84045982 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t5 12181350 +chr15 84596420 84634582 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t6 1144860 +chr15 84636031 84801860 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t7 4974870 +chr15 84803285 85169770 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_t8 10994550 +chr15 85526412 85626412 15q25.2q25.3_proximal_and_distal_LCR_A_B-D_R 3000000 +chr15 82040662 82140662 15q25.2_proximal_LCR_A_B-C_L 3000000 +chr15 82534140 82613798 15q25.2_proximal_LCR_A_B-C_t1 2389740 +chr15 82614158 82809649 15q25.2_proximal_LCR_A_B-C_t2 5864730 +chr15 82827843 83233889 15q25.2_proximal_LCR_A_B-C_t3 12181380 +chr15 83233889 83639935 15q25.2_proximal_LCR_A_B-C_t4 12181380 +chr15 83639935 84045981 15q25.2_proximal_LCR_A_B-C_t5 12181380 +chr15 85526412 85626412 15q25.2_proximal_LCR_A_B-C_R 3000000 +chr15 82040662 82140662 15q25.2q25.3_distal_LCR_C-D_L 3000000 +chr15 84596421 84634582 15q25.2q25.3_distal_LCR_C-D_t1 1144830 +chr15 84636031 84801860 15q25.2q25.3_distal_LCR_C-D_t2 4974870 +chr15 84803285 85169770 15q25.2q25.3_distal_LCR_C-D_t3 10994550 +chr15 85526412 85626412 15q25.2q25.3_distal_LCR_C-D_R 3000000 +chr16 14580912 14680912 16p13.11_BP1-BP3_L 3000000 +chr16 15030912 15062395 16p13.11_BP1-BP3_t1 944490 +chr16 15417853 15808131 16p13.11_BP1-BP3_t2 11708340 +chr16 15808131 16198408 16p13.11_BP1-BP3_t3 11708310 +chr16 19002480 19102480 16p13.11_BP1-BP3_R 3000000 +chr16 14580912 14680912 16p13.11_BP1-BP4_L 3000000 +chr16 15030912 15062395 16p13.11_BP1-BP4_t1 944490 +chr16 15417853 15808131 16p13.11_BP1-BP4_t2 11708340 +chr16 15808131 16198409 16p13.11_BP1-BP4_t3 11708340 +chr16 16762938 17093166 16p13.11_BP1-BP4_t4 9906840 +chr16 17096423 17584983 16p13.11_BP1-BP4_t5 14656800 +chr16 17584983 18073542 16p13.11_BP1-BP4_t6 14656770 +chr16 19002480 19102480 16p13.11_BP1-BP4_R 3000000 +chr16 14580912 14680912 16p13.11_BP2-BP3_L 3000000 +chr16 15417854 15808131 16p13.11_BP2-BP3_t1 11708310 +chr16 15808131 16198408 16p13.11_BP2-BP3_t2 11708310 +chr16 19002480 19102480 16p13.11_BP2-BP3_R 3000000 +chr16 14580912 14680912 16p13.11_BP2-BP4_L 3000000 +chr16 15417855 15808132 16p13.11_BP2-BP4_t1 11708310 +chr16 15808132 16198409 16p13.11_BP2-BP4_t2 11708310 +chr16 16762938 17093166 16p13.11_BP2-BP4_t3 9906840 +chr16 17096423 17584983 16p13.11_BP2-BP4_t4 14656800 +chr16 17584983 18073542 16p13.11_BP2-BP4_t5 14656770 +chr16 19002480 19102480 16p13.11_BP2-BP4_R 3000000 +chr16 14580912 14680912 16p12.3_BP3-BP4_L 3000000 +chr16 16762939 17093166 16p12.3_BP3-BP4_t1 9906810 +chr16 17096423 17584983 16p12.3_BP3-BP4_t2 14656800 +chr16 17584983 18073542 16p12.3_BP3-BP4_t3 14656770 +chr16 19002480 19102480 16p12.3_BP3-BP4_R 3000000 +chr16 21192540 21292540 16p12.2_distal_BP1-BP2_L 3000000 +chr16 21558792 21728000 16p12.2_distal_BP1-BP2_t1 5076240 +chr16 30393459 30493459 16p12.2_distal_BP1-BP2_R 3000000 +chr16 21192540 21292540 16p12.2p11.2_L 3000000 +chr16 21558793 21728000 16p12.2p11.2_t1 5076210 +chr16 21937123 22419484 16p12.2p11.2_t2 14470830 +chr16 22613938 22833211 16p12.2p11.2_t3 6578190 +chr16 22834242 23166943 16p12.2p11.2_t4 9981030 +chr16 23166943 23499643 16p12.2p11.2_t5 9981000 +chr16 23500829 23501557 16p12.2p11.2_t6 21840 +chr16 23503728 23667041 16p12.2p11.2_t7 4899390 +chr16 23669861 24115761 16p12.2p11.2_t8 13377000 +chr16 24115761 24561661 16p12.2p11.2_t9 13377000 +chr16 24561661 25007561 16p12.2p11.2_t10 13377000 +chr16 25007561 25453458 16p12.2p11.2_t11 13376910 +chr16 25454717 25743210 16p12.2p11.2_t12 8654790 +chr16 25743210 26031702 16p12.2p11.2_t13 8654760 +chr16 26032921 26111401 16p12.2p11.2_t14 2354400 +chr16 26112444 26452694 16p12.2p11.2_t15 10207500 +chr16 26452694 26792944 16p12.2p11.2_t16 10207500 +chr16 26792944 27133194 16p12.2p11.2_t17 10207500 +chr16 27135251 27503334 16p12.2p11.2_t18 11042490 +chr16 27503334 27871417 16p12.2p11.2_t19 11042490 +chr16 27871417 28239498 16p12.2p11.2_t20 11042430 +chr16 28240648 28327647 16p12.2p11.2_t21 2609970 +chr16 28471892 28590812 16p12.2p11.2_t22 3567600 +chr16 28811313 29035179 16p12.2p11.2_t23 6715980 +chr16 29126773 29127492 16p12.2p11.2_t24 21570 +chr16 29302873 29303936 16p12.2p11.2_t25 31890 +chr16 30393459 30493459 16p12.2p11.2_R 3000000 +chr16 21192540 21292540 16p12.2_proximal_BP2-BP3_L 3000000 +chr16 21937124 22419483 16p12.2_proximal_BP2-BP3_t1 14470770 +chr16 30393459 30493459 16p12.2_proximal_BP2-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_distal_BP1-BP3_L 3000000 +chr16 28471894 28590812 16p11.2_distal_BP1-BP3_t1 3567540 +chr16 28811313 29035178 16p11.2_distal_BP1-BP3_t2 6715950 +chr16 30393459 30493459 16p11.2_distal_BP1-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_BP1-BP4_L 3000000 +chr16 28471894 28590812 16p11.2_proximal_BP1-BP4_t1 3567540 +chr16 28811313 29035179 16p11.2_proximal_BP1-BP4_t2 6715980 +chr16 29126773 29127492 16p11.2_proximal_BP1-BP4_t3 21570 +chr16 29302873 29303936 16p11.2_proximal_BP1-BP4_t4 31890 +chr16 30393459 30493459 16p11.2_proximal_BP1-BP4_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_and_distal_BP1-BP5_L 3000000 +chr16 28471894 28590812 16p11.2_proximal_and_distal_BP1-BP5_t1 3567540 +chr16 28811313 29035179 16p11.2_proximal_and_distal_BP1-BP5_t2 6715980 +chr16 29126773 29127492 16p11.2_proximal_and_distal_BP1-BP5_t3 21570 +chr16 29302873 29303936 16p11.2_proximal_and_distal_BP1-BP5_t4 31890 +chr16 29638675 29903564 16p11.2_proximal_and_distal_BP1-BP5_t5 7946670 +chr16 29903564 30168453 16p11.2_proximal_and_distal_BP1-BP5_t6 7946670 +chr16 30170912 30188531 16p11.2_proximal_and_distal_BP1-BP5_t7 528570 +chr16 30393459 30493459 16p11.2_proximal_and_distal_BP1-BP5_R 3000000 +chr16 21192540 21292540 16p11.2_distal_BP2-BP3_L 3000000 +chr16 28811314 29035178 16p11.2_distal_BP2-BP3_t1 6715920 +chr16 30393459 30493459 16p11.2_distal_BP2-BP3_R 3000000 +chr16 21192540 21292540 16p11.2_proximal_BP4-BP5_L 3000000 +chr16 29638676 29903565 16p11.2_proximal_BP4-BP5_t1 7946670 +chr16 29903565 30168453 16p11.2_proximal_BP4-BP5_t2 7946640 +chr16 30170912 30188531 16p11.2_proximal_BP4-BP5_t3 528570 +chr16 30393459 30493459 16p11.2_proximal_BP4-BP5_R 3000000 +chr17 13869962 13969962 17p12_HNPP_CMT1A_L 3000000 +chr17 14194700 14230542 17p12_HNPP_CMT1A_t1 1075260 +chr17 14241379 14667253 17p12_HNPP_CMT1A_t2 12776220 +chr17 14667253 15093127 17p12_HNPP_CMT1A_t3 12776220 +chr17 15093127 15519000 17p12_HNPP_CMT1A_t4 12776190 +chr17 15930698 16030698 17p12_HNPP_CMT1A_R 3000000 +chr17 16486272 16586272 17p11.2_SMS_PLS_L 3000000 +chr17 16906714 16935461 17p11.2_SMS_PLS_t1 862410 +chr17 16935702 16995496 17p11.2_SMS_PLS_t2 1793820 +chr17 17031424 17463581 17p11.2_SMS_PLS_t3 12964710 +chr17 17463581 17895738 17p11.2_SMS_PLS_t4 12964710 +chr17 17895738 18327894 17p11.2_SMS_PLS_t5 12964680 +chr17 18329376 18375628 17p11.2_SMS_PLS_t6 1387560 +chr17 18848096 19020655 17p11.2_SMS_PLS_t7 5176770 +chr17 19237509 19282949 17p11.2_SMS_PLS_t8 1363200 +chr17 19284122 19581999 17p11.2_SMS_PLS_t9 8936310 +chr17 19583330 19587402 17p11.2_SMS_PLS_t10 122160 +chr17 19588723 19598547 17p11.2_SMS_PLS_t11 294720 +chr17 19601764 19856686 17p11.2_SMS_PLS_t12 7647660 +chr17 19856686 20111608 17p11.2_SMS_PLS_t13 7647660 +chr17 20115149 20233524 17p11.2_SMS_PLS_t14 3551250 +chr17 20235188 20309889 17p11.2_SMS_PLS_t15 2241030 +chr17 20986981 21086981 17p11.2_SMS_PLS_R 3000000 +chr17 16486272 16586272 17p11.2_REPD-REPM_L 3000000 +chr17 16906715 16935461 17p11.2_REPD-REPM_t1 862380 +chr17 16935702 16995496 17p11.2_REPD-REPM_t2 1793820 +chr17 17031424 17463581 17p11.2_REPD-REPM_t3 12964710 +chr17 17463581 17895738 17p11.2_REPD-REPM_t4 12964710 +chr17 17895738 18327894 17p11.2_REPD-REPM_t5 12964680 +chr17 18329376 18375627 17p11.2_REPD-REPM_t6 1387530 +chr17 20986981 21086981 17p11.2_REPD-REPM_R 3000000 +chr17 30380647 30480647 17q11.2_Type_1_REPA-REPC_L 3000000 +chr17 30780078 31008336 17q11.2_Type_1_REPA-REPC_t1 6847740 +chr17 31051210 31067523 17q11.2_Type_1_REPA-REPC_t2 489390 +chr17 31072877 31078593 17q11.2_Type_1_REPA-REPC_t3 171480 +chr17 31080033 31138282 17q11.2_Type_1_REPA-REPC_t4 1747470 +chr17 31149888 31189463 17q11.2_Type_1_REPA-REPC_t5 1187250 +chr17 31191722 31193317 17q11.2_Type_1_REPA-REPC_t6 47850 +chr17 31202173 31212016 17q11.2_Type_1_REPA-REPC_t7 295290 +chr17 31236747 31256172 17q11.2_Type_1_REPA-REPC_t8 582750 +chr17 31268283 31345374 17q11.2_Type_1_REPA-REPC_t9 2312730 +chr17 31348726 31642514 17q11.2_Type_1_REPA-REPC_t10 8813640 +chr17 31642514 31936301 17q11.2_Type_1_REPA-REPC_t11 8813610 +chr17 32170047 32270047 17q11.2_Type_1_REPA-REPC_R 3000000 +chr17 30380647 30480647 17q11.2_Type_2_SUZ12P1-SUZ12_L 3000000 +chr17 30780079 31008336 17q11.2_Type_2_SUZ12P1-SUZ12_t1 6847710 +chr17 31051210 31067523 17q11.2_Type_2_SUZ12P1-SUZ12_t2 489390 +chr17 31072877 31078593 17q11.2_Type_2_SUZ12P1-SUZ12_t3 171480 +chr17 31080033 31138282 17q11.2_Type_2_SUZ12P1-SUZ12_t4 1747470 +chr17 31149888 31189463 17q11.2_Type_2_SUZ12P1-SUZ12_t5 1187250 +chr17 31191722 31193317 17q11.2_Type_2_SUZ12P1-SUZ12_t6 47850 +chr17 31202173 31212016 17q11.2_Type_2_SUZ12P1-SUZ12_t7 295290 +chr17 31236747 31256172 17q11.2_Type_2_SUZ12P1-SUZ12_t8 582750 +chr17 31268283 31345374 17q11.2_Type_2_SUZ12P1-SUZ12_t9 2312730 +chr17 31348726 31642514 17q11.2_Type_2_SUZ12P1-SUZ12_t10 8813640 +chr17 31642514 31936301 17q11.2_Type_2_SUZ12P1-SUZ12_t11 8813610 +chr17 32170047 32270047 17q11.2_Type_2_SUZ12P1-SUZ12_R 3000000 +chr17 30380647 30480647 17q11.2_Type_3_REPB-REPC_L 3000000 +chr17 31051211 31067523 17q11.2_Type_3_REPB-REPC_t1 489360 +chr17 31072877 31078593 17q11.2_Type_3_REPB-REPC_t2 171480 +chr17 31080033 31138282 17q11.2_Type_3_REPB-REPC_t3 1747470 +chr17 31149888 31189463 17q11.2_Type_3_REPB-REPC_t4 1187250 +chr17 31191722 31193317 17q11.2_Type_3_REPB-REPC_t5 47850 +chr17 31202173 31212016 17q11.2_Type_3_REPB-REPC_t6 295290 +chr17 31236747 31256172 17q11.2_Type_3_REPB-REPC_t7 582750 +chr17 31268283 31345374 17q11.2_Type_3_REPB-REPC_t8 2312730 +chr17 31348726 31642514 17q11.2_Type_3_REPB-REPC_t9 8813640 +chr17 31642514 31936301 17q11.2_Type_3_REPB-REPC_t10 8813610 +chr17 32170047 32270047 17q11.2_Type_3_REPB-REPC_R 3000000 +chr17 35818633 35918633 17q12_RCAD_syndrome_L 3000000 +chr17 36458167 36800704 17q12_RCAD_syndrome_t1 10276110 +chr17 36800704 37143240 17q12_RCAD_syndrome_t2 10276080 +chr17 37144370 37488620 17q12_RCAD_syndrome_t3 10327500 +chr17 37488620 37832870 17q12_RCAD_syndrome_t4 10327500 +chr17 37836264 37854616 17q12_RCAD_syndrome_t5 550560 +chr17 38364969 38464969 17q12_RCAD_syndrome_R 3000000 +chr17 45264926 45364926 17q21.31_L 3000000 +chr17 45627800 46087514 17q21.31_t1 13791420 +chr17 46757124 46857124 17q21.31_R 3000000 +chr17 59771943 59871943 17q23.1q23.2_L 3000000 +chr17 60035641 60096247 17q23.1q23.2_t1 1818180 +chr17 60127341 60166682 17q23.1q23.2_t2 1180230 +chr17 60171137 60172722 17q23.1q23.2_t3 47550 +chr17 60221224 60374694 17q23.1q23.2_t4 4604100 +chr17 60382053 60434344 17q23.1q23.2_t5 1568730 +chr17 60435541 60851263 17q23.1q23.2_t6 12471660 +chr17 60852649 61301249 17q23.1q23.2_t7 13458000 +chr17 61301249 61749849 17q23.1q23.2_t8 13458000 +chr17 61749849 62198448 17q23.1q23.2_t9 13457970 +chr17 62618463 62718463 17q23.1q23.2_R 3000000 +chr22 16912063 16922834 22q11.21_CES_t1 323130 +chr22 16928158 17014088 22q11.21_CES_t2 2577900 +chr22 17038567 17214090 22q11.21_CES_t3 5265690 +chr22 17215248 17419415 22q11.21_CES_t4 6125010 +chr22 17420675 17763709 22q11.21_CES_t5 10291020 +chr22 17765328 17984579 22q11.21_CES_t6 6577530 +chr22 17998239 18008719 22q11.21_CES_t7 314400 +chr22 18010388 18159723 22q11.21_CES_t8 4480050 +chr22 24773137 24873137 22q11.21_CES_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-B_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-B_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-B_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-B_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-B_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-B_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-B_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-B_t8 2856540 +chr22 20290267 20299685 22q11.2_DGS_proximal_A-B_t9 282540 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-B_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-C_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-C_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-C_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-C_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-C_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-C_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-C_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-C_t8 2856540 +chr22 20290267 20299686 22q11.2_DGS_proximal_A-C_t9 282570 +chr22 20377695 20388431 22q11.2_DGS_proximal_A-C_t10 322080 +chr22 20390146 20451509 22q11.2_DGS_proximal_A-C_t11 1840890 +chr22 20453240 20482745 22q11.2_DGS_proximal_A-C_t12 885150 +chr22 20484148 20656624 22q11.2_DGS_proximal_A-C_t13 5174280 +chr22 20658941 20667579 22q11.2_DGS_proximal_A-C_t14 259140 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-C_R 3000000 +chr22 18939750 18950240 22q11.2_DGS_proximal_A-D_t1 314700 +chr22 18976730 18977700 22q11.2_DGS_proximal_A-D_t2 29100 +chr22 18981974 18999690 22q11.2_DGS_proximal_A-D_t3 531480 +chr22 19015209 19023255 22q11.2_DGS_proximal_A-D_t4 241380 +chr22 19035499 19404012 22q11.2_DGS_proximal_A-D_t5 11055390 +chr22 19404012 19772525 22q11.2_DGS_proximal_A-D_t6 11055390 +chr22 19772525 20141036 22q11.2_DGS_proximal_A-D_t7 11055330 +chr22 20166817 20262035 22q11.2_DGS_proximal_A-D_t8 2856540 +chr22 20290267 20299686 22q11.2_DGS_proximal_A-D_t9 282570 +chr22 20377695 20388431 22q11.2_DGS_proximal_A-D_t10 322080 +chr22 20390146 20451509 22q11.2_DGS_proximal_A-D_t11 1840890 +chr22 20453240 20482745 22q11.2_DGS_proximal_A-D_t12 885150 +chr22 20484148 20656624 22q11.2_DGS_proximal_A-D_t13 5174280 +chr22 20658941 20667580 22q11.2_DGS_proximal_A-D_t14 259170 +chr22 20738049 21010365 22q11.2_DGS_proximal_A-D_t15 8169480 +chr22 21018352 21042050 22q11.2_DGS_proximal_A-D_t16 710940 +chr22 21050489 21057379 22q11.2_DGS_proximal_A-D_t17 206700 +chr22 21062735 21111383 22q11.2_DGS_proximal_A-D_t18 1459440 +chr22 24773137 24873137 22q11.2_DGS_proximal_A-D_R 3000000 +chr22 20377696 20388431 22q11.2_central_B-D_t1 322050 +chr22 20390146 20451509 22q11.2_central_B-D_t2 1840890 +chr22 20453240 20482745 22q11.2_central_B-D_t3 885150 +chr22 20484148 20656624 22q11.2_central_B-D_t4 5174280 +chr22 20658941 20667580 22q11.2_central_B-D_t5 259170 +chr22 20738049 21010365 22q11.2_central_B-D_t6 8169480 +chr22 21018352 21042050 22q11.2_central_B-D_t7 710940 +chr22 21050489 21057379 22q11.2_central_B-D_t8 206700 +chr22 21062735 21111383 22q11.2_central_B-D_t9 1459440 +chr22 24773137 24873137 22q11.2_central_B-D_R 3000000 +chr22 20738050 21010365 22q11.2_central_C-D_t1 8169450 +chr22 21018352 21042050 22q11.2_central_C-D_t2 710940 +chr22 21050489 21057379 22q11.2_central_C-D_t3 206700 +chr22 21062735 21111383 22q11.2_central_C-D_t4 1459440 +chr22 24773137 24873137 22q11.2_central_C-D_R 3000000 +chr22 20738051 21010365 22q11.2_C-E_t1 8169420 +chr22 21018352 21042050 22q11.2_C-E_t2 710940 +chr22 21050489 21057379 22q11.2_C-E_t3 206700 +chr22 21062735 21111384 22q11.2_C-E_t4 1459470 +chr22 21562827 21954496 22q11.2_C-E_t5 11750070 +chr22 21958494 22114615 22q11.2_C-E_t6 4683630 +chr22 22120610 22122809 22q11.2_C-E_t7 65970 +chr22 22125415 22125942 22q11.2_C-E_t8 15810 +chr22 22126195 22133984 22q11.2_C-E_t9 233670 +chr22 22139837 22220568 22q11.2_C-E_t10 2421930 +chr22 22224566 22249758 22q11.2_C-E_t11 755760 +chr22 22315122 22320797 22q11.2_C-E_t12 170250 +chr22 22327981 22402552 22q11.2_C-E_t13 2237130 +chr22 22410447 22620607 22q11.2_C-E_t14 6304800 +chr22 24773137 24873137 22q11.2_C-E_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_I_D-E_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_I_D-E_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_I_D-E_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_I_D-E_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_I_D-E_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_I_D-E_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_I_D-E_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_I_D-E_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_I_D-E_t9 2237130 +chr22 22410447 22620607 22q11.2_distal_type_I_D-E_t10 6304800 +chr22 24773137 24873137 22q11.2_distal_type_I_D-E_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_I_D-F_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_I_D-F_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_I_D-F_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_I_D-F_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_I_D-F_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_I_D-F_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_I_D-F_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_I_D-F_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_I_D-F_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_I_D-F_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_I_D-F_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_I_D-F_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_I_D-F_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_I_D-F_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_I_D-F_t15 10711530 +chr22 23264988 23306924 22q11.2_distal_type_I_D-F_t16 1258080 +chr22 24773137 24873137 22q11.2_distal_type_I_D-F_R 3000000 +chr22 21562828 21954496 22q11.2_distal_type_III_D-H_t1 11750040 +chr22 21958494 22114615 22q11.2_distal_type_III_D-H_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_III_D-H_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_III_D-H_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_III_D-H_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_III_D-H_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_III_D-H_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_III_D-H_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_III_D-H_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_III_D-H_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_III_D-H_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_III_D-H_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_D-H_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_D-H_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_D-H_t15 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_D-H_t16 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_D-H_t17 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_D-H_t18 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_D-H_t19 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_D-H_t20 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_D-H_t21 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_D-H_t22 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_D-H_t23 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_D-H_t24 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_D-H_t25 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_D-H_t26 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_D-H_t27 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_D-H_t28 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_D-H_t29 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_D-H_t30 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_D-H_t31 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_D-H_t32 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_D-H_t33 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_D-H_R 3000000 +chr22 21562829 21954496 22q11.2_distal_type_III_D-G_t1 11750010 +chr22 21958494 22114615 22q11.2_distal_type_III_D-G_t2 4683630 +chr22 22120610 22122809 22q11.2_distal_type_III_D-G_t3 65970 +chr22 22125415 22125942 22q11.2_distal_type_III_D-G_t4 15810 +chr22 22126195 22133984 22q11.2_distal_type_III_D-G_t5 233670 +chr22 22139837 22220568 22q11.2_distal_type_III_D-G_t6 2421930 +chr22 22224566 22249758 22q11.2_distal_type_III_D-G_t7 755760 +chr22 22315122 22320797 22q11.2_distal_type_III_D-G_t8 170250 +chr22 22327981 22402552 22q11.2_distal_type_III_D-G_t9 2237130 +chr22 22410447 22620608 22q11.2_distal_type_III_D-G_t10 6304830 +chr22 22776923 22789775 22q11.2_distal_type_III_D-G_t11 385560 +chr22 22794414 22820054 22q11.2_distal_type_III_D-G_t12 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_D-G_t13 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_D-G_t14 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_D-G_t15 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_D-G_t16 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_D-G_t17 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_D-G_t18 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_D-G_t19 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_D-G_t20 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_D-G_t21 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_D-G_t22 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_D-G_t23 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_D-G_t24 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_D-G_t25 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_D-G_t26 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_D-G_t27 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_D-G_t28 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_D-G_t29 5337540 +chr22 24184938 24236853 22q11.2_distal_type_III_D-G_t30 1557450 +chr22 24773137 24873137 22q11.2_distal_type_III_D-G_R 3000000 +chr22 22776924 22789775 22q11.2_distal_type_II_E-F_t1 385530 +chr22 22794414 22820054 22q11.2_distal_type_II_E-F_t2 769200 +chr22 22823467 22897363 22q11.2_distal_type_II_E-F_t3 2216880 +chr22 22901536 22902711 22q11.2_distal_type_II_E-F_t4 35250 +chr22 22906902 23263953 22q11.2_distal_type_II_E-F_t5 10711530 +chr22 23264988 23306924 22q11.2_distal_type_II_E-F_t6 1258080 +chr22 24773137 24873137 22q11.2_distal_type_II_E-F_R 3000000 +chr22 22776925 22789775 22q11.2_distal_type_III_E-H_t1 385500 +chr22 22794414 22820054 22q11.2_distal_type_III_E-H_t2 769200 +chr22 22823467 22897363 22q11.2_distal_type_III_E-H_t3 2216880 +chr22 22901536 22902711 22q11.2_distal_type_III_E-H_t4 35250 +chr22 22906902 23263953 22q11.2_distal_type_III_E-H_t5 10711530 +chr22 23264988 23306925 22q11.2_distal_type_III_E-H_t6 1258110 +chr22 23489014 23493585 22q11.2_distal_type_III_E-H_t7 137130 +chr22 23510388 23511082 22q11.2_distal_type_III_E-H_t8 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_E-H_t9 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_E-H_t10 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_E-H_t11 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_E-H_t12 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_E-H_t13 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_E-H_t14 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_E-H_t15 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_E-H_t16 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_E-H_t17 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_E-H_t18 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_E-H_t19 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_E-H_t20 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_E-H_t21 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_E-H_t22 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_E-H_t23 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_E-H_R 3000000 +chr22 23489015 23493585 22q11.2_distal_type_III_F-G_t1 137100 +chr22 23510388 23511082 22q11.2_distal_type_III_F-G_t2 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_F-G_t3 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_F-G_t4 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_F-G_t5 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_F-G_t6 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_F-G_t7 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_F-G_t8 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_F-G_t9 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_F-G_t10 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_F-G_t11 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_F-G_t12 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_F-G_t13 5337540 +chr22 24184938 24236853 22q11.2_distal_type_III_F-G_t14 1557450 +chr22 24773137 24873137 22q11.2_distal_type_III_F-G_R 3000000 +chr22 23489016 23493585 22q11.2_distal_type_III_F-H_t1 137070 +chr22 23510388 23511082 22q11.2_distal_type_III_F-H_t2 20820 +chr22 23552138 23553526 22q11.2_distal_type_III_F-H_t3 41640 +chr22 23577255 23594547 22q11.2_distal_type_III_F-H_t4 518760 +chr22 23602870 23616462 22q11.2_distal_type_III_F-H_t5 407760 +chr22 23644932 23658595 22q11.2_distal_type_III_F-H_t6 409890 +chr22 23661000 23670225 22q11.2_distal_type_III_F-H_t7 276750 +chr22 23679116 23734856 22q11.2_distal_type_III_F-H_t8 1672200 +chr22 23737374 23853822 22q11.2_distal_type_III_F-H_t9 3493440 +chr22 23856326 23929519 22q11.2_distal_type_III_F-H_t10 2195790 +chr22 23932056 23939932 22q11.2_distal_type_III_F-H_t11 236280 +chr22 23970000 23971773 22q11.2_distal_type_III_F-H_t12 53190 +chr22 24001392 24179310 22q11.2_distal_type_III_F-H_t13 5337540 +chr22 24184938 24236854 22q11.2_distal_type_III_F-H_t14 1557480 +chr22 24301118 24310536 22q11.2_distal_type_III_F-H_t15 282540 +chr22 24314778 24377585 22q11.2_distal_type_III_F-H_t16 1884210 +chr22 24389259 24598466 22q11.2_distal_type_III_F-H_t17 6276210 +chr22 24773137 24873137 22q11.2_distal_type_III_F-H_R 3000000 +chrX 6377561 6477561 Xp22.31_L 1500000 +chrX 6537771 6709739 Xp22.31_t1 2579520 +chrX 6712094 6934350 Xp22.31_t2 3333840 +chrX 6935400 7003885 Xp22.31_t3 1027275 +chrX 7044785 7321723 Xp22.31_t4 4154070 +chrX 7321723 7598660 Xp22.31_t5 4154055 +chrX 7599946 7752196 Xp22.31_t6 2283750 +chrX 7761567 7840853 Xp22.31_t7 1189290 +chrX 7852088 7973686 Xp22.31_t8 1823970 +chrX 7984426 8000352 Xp22.31_t9 238890 +chrX 8006947 8156913 Xp22.31_t10 2249490 +chrX 8231347 8331347 Xp22.31_R 1500000 +chrX 47703983 47803983 Xp11.22p11.23_L 1500000 +chrX 48447780 48801813 Xp11.22p11.23_t1 5310495 +chrX 48801813 49155846 Xp11.22p11.23_t2 5310495 +chrX 49157442 49162798 Xp11.22p11.23_t3 80340 +chrX 49164374 49300509 Xp11.22p11.23_t4 2042025 +chrX 49606699 49631626 Xp11.22p11.23_t5 373905 +chrX 49633394 49663751 Xp11.22p11.23_t6 455355 +chrX 49668689 49836185 Xp11.22p11.23_t7 2512440 +chrX 49837392 49877729 Xp11.22p11.23_t8 605055 +chrX 49878948 50174945 Xp11.22p11.23_t9 4439955 +chrX 50175512 50204972 Xp11.22p11.23_t10 441900 +chrX 50209960 50210790 Xp11.22p11.23_t11 12450 +chrX 50228964 50278964 Xp11.22p11.23_t12 750000 +chrX 50293964 50599659 Xp11.22p11.23_t13 4585425 +chrX 50599659 50905354 Xp11.22p11.23_t14 4585425 +chrX 50906368 50990225 Xp11.22p11.23_t15 1257855 +chrX 50993330 50996121 Xp11.22p11.23_t16 41865 +chrX 51040256 51077839 Xp11.22p11.23_t17 563745 +chrX 51081476 51154075 Xp11.22p11.23_t18 1088985 +chrX 51184010 51290913 Xp11.22p11.23_t19 1603545 +chrX 51296426 51314554 Xp11.22p11.23_t20 271920 +chrX 51334611 51494772 Xp11.22p11.23_t21 2402415 +chrX 51526125 51617596 Xp11.22p11.23_t22 1372065 +chrX 51619037 51667324 Xp11.22p11.23_t23 724305 +chrX 51692963 51700363 Xp11.22p11.23_t24 111000 +chrX 51725664 51869486 Xp11.22p11.23_t25 2157330 +chrX 51873046 51903394 Xp11.22p11.23_t26 455220 +chrX 51904535 51919767 Xp11.22p11.23_t27 228480 +chrX 51922376 52037052 Xp11.22p11.23_t28 1720140 +chrX 52068673 52185421 Xp11.22p11.23_t29 1751220 +chrX 52213786 52262297 Xp11.22p11.23_t30 727665 +chrX 52266091 52266756 Xp11.22p11.23_t31 9975 +chrX 52268901 52336321 Xp11.22p11.23_t32 1011300 +chrX 52340891 52444264 Xp11.22p11.23_t33 1550595 +chrX 53026821 53126821 Xp11.22p11.23_R 1500000 +chrX 153905935 154005935 Xq28_L 1500000 +chrX 154396223 154555683 Xq28_t1 2391900 +chrX 154704527 154804527 Xq28_R 1500000 +chrX 154710814 154810814 Xq28_int22h1_int22h2-flanked_L 1500000 +chrX 154890328 155006106 Xq28_int22h1_int22h2-flanked_t1 1736670 +chrX 155007147 155147780 Xq28_int22h1_int22h2-flanked_t2 2109495 +chrX 155148821 155288173 Xq28_int22h1_int22h2-flanked_t3 2090280 +chrX 155291419 155304000 Xq28_int22h1_int22h2-flanked_t4 188715 +chrX 155304900 155335092 Xq28_int22h1_int22h2-flanked_t5 452880 +chrX 155506728 155606728 Xq28_int22h1_int22h2-flanked_R 1500000 +chr1 898702 955921 1p36_terminal_t1 1716570 +chr1 960557 1075856 1p36_terminal_t2 3458970 +chr1 1079032 1444514 1p36_terminal_t3 10964460 +chr1 1448930 1464079 1p36_terminal_t4 454470 +chr1 1502033 1504522 1p36_terminal_t5 74670 +chr1 1534825 1632163 1p36_terminal_t6 2920140 +chr1 1730936 1733195 1p36_terminal_t7 67770 +chr1 1752193 2150309 1p36_terminal_t8 11943480 +chr1 2150309 2548425 1p36_terminal_t9 11943480 +chr1 2550688 2551369 1p36_terminal_t10 20430 +chr1 2554047 2567635 1p36_terminal_t11 407640 +chr1 2570293 2576603 1p36_terminal_t12 189300 +chr1 2578586 2652036 1p36_terminal_t13 2203500 +chr1 2702781 2746290 1p36_terminal_t14 1305270 +chr1 2777361 3235654 1p36_terminal_t15 13748790 +chr1 3235654 3693946 1p36_terminal_t16 13748760 +chr1 3695002 3722655 1p36_terminal_t17 829590 +chr1 3723188 4043198 1p36_terminal_t18 9600300 +chr1 4045223 4447638 1p36_terminal_t19 12072450 +chr1 4447638 4850053 1p36_terminal_t20 12072450 +chr1 4850053 5252468 1p36_terminal_t21 12072450 +chr1 5252468 5654881 1p36_terminal_t22 12072390 +chr1 5657240 5665835 1p36_terminal_t23 257850 +chr1 5673788 5929408 1p36_terminal_t24 7668600 +chr1 5929408 6185028 1p36_terminal_t25 7668600 +chr1 6186818 6229913 1p36_terminal_t26 1292850 +chr1 6279913 6379913 1p36_terminal_R 3000000 +chrX 153858528 153958528 Xq28_MECP2_L 1500000 +chrX 154008528 154110279 Xq28_MECP2_t1 1526265 +chrX 154660279 154760279 Xq28_MECP2_R 1500000 +chr13 20073037 20173037 GJB6_D13S1830_L 3000000 +chr13 20223037 20531806 GJB6_D13S1830_t1 9263070 +chr13 20581806 20681806 GJB6_D13S1830_R 3000000 +chr7 117348312 117448312 CFTR_dele2_3_L 3000000 +chr7 117498312 117519392 CFTR_dele2_3_t1 632400 +chr7 117569392 117669392 CFTR_dele2_3_R 3000000 +chr11 5075254 5175254 HBB_exon3_del_L 3000000 +chr11 5225254 5225875 HBB_exon3_del_t1 18630 +chr11 5275875 5375875 HBB_exon3_del_R 3000000 +chr11 5075388 5175388 HBB_619bp_del_L 3000000 +chr11 5225388 5226007 HBB_619bp_del_t1 18570 +chr11 5276007 5376007 HBB_619bp_del_R 3000000 +chr11 5069877 5169877 HBD_HBB_Sicilian_del_L 3000000 +chr11 5219877 5233280 HBD_HBB_Sicilian_del_t1 402090 +chr11 5283280 5383280 HBD_HBB_Sicilian_del_R 3000000 +chrY 2935200 2982579 AZFa_c1 710685 +chrY 12537649 12860852 AZFa_t1 4848045 +chrY 12904107 12920478 AZFa_t2 245565 +chrY 14522572 14845654 AZFa_c2 4846230 +chrY 2935200 2982579 AZFb_c1 710685 +chrY 14522572 14845654 AZFb_c2 4846230 +chrY 19702297 19745148 AZFb_t1 642765 +chrY 20575693 20593154 AZFb_t2 261915 +chrY 2935200 2982579 AZFc_DAZ_c1 710685 +chrY 14522572 14845654 AZFc_DAZ_c2 4846230 +chrY 23129354 23199010 AZFc_DAZ_t1 1044840 +chrY 23219446 23291356 AZFc_DAZ_t2 1078650 +chrY 24763068 24813492 AZFc_DAZ_t3 756360 +chrY 24833842 24907040 AZFc_DAZ_t4 1097970 diff --git a/tests/parity/inputs/cn/kiv-notseen.pileup b/tests/parity/inputs/cn/kiv-notseen.pileup new file mode 100644 index 000000000..b993050ec --- /dev/null +++ b/tests/parity/inputs/cn/kiv-notseen.pileup @@ -0,0 +1,7 @@ +chr6 160646214 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160640667 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160635123 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160629574 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160624030 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160618484 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III +chr6 160612932 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC III diff --git a/tests/parity/inputs/cn/kiv-present.pileup b/tests/parity/inputs/cn/kiv-present.pileup new file mode 100644 index 000000000..b4131112c --- /dev/null +++ b/tests/parity/inputs/cn/kiv-present.pileup @@ -0,0 +1,7 @@ +chr6 160646214 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTTt+1A III +chr6 160640667 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTT+1A III +chr6 160635123 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCTT+1A III +chr6 160629574 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160624030 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160618484 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III +chr6 160612932 N 40 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC+1A III diff --git a/tests/parity/inputs/cn/lce0.bedcov.tsv b/tests/parity/inputs/cn/lce0.bedcov.tsv new file mode 100644 index 000000000..0418598db --- /dev/null +++ b/tests/parity/inputs/cn/lce0.bedcov.tsv @@ -0,0 +1,3 @@ +chr1 152520000 152560000 LCE3_FLANK_L 1200000 +chr1 152600233 152614098 LCE3BC_DEL 0 +chr1 152640000 152680000 LCE3_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/lce0.q1.tsv b/tests/parity/inputs/cn/lce0.q1.tsv new file mode 100644 index 000000000..0418598db --- /dev/null +++ b/tests/parity/inputs/cn/lce0.q1.tsv @@ -0,0 +1,3 @@ +chr1 152520000 152560000 LCE3_FLANK_L 1200000 +chr1 152600233 152614098 LCE3BC_DEL 0 +chr1 152640000 152680000 LCE3_FLANK_R 1200000 diff --git a/tests/parity/inputs/cn/lpa-floor.bedcov.tsv b/tests/parity/inputs/cn/lpa-floor.bedcov.tsv new file mode 100644 index 000000000..fa5c8e268 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-floor.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 34 +chr6 160612931 160613113 kiv2_e15 39 +chr6 160617116 160617276 kiv2_e14 34 +chr6 160618483 160618665 kiv2_e13 39 +chr6 160622662 160622822 kiv2_e12 34 +chr6 160624029 160624211 kiv2_e11 39 +chr6 160628206 160628366 kiv2_e10 34 +chr6 160629573 160629755 kiv2_e09 39 +chr6 160633752 160633912 kiv2_e08 34 +chr6 160635122 160635304 kiv2_e07 39 +chr6 160639299 160639459 kiv2_e06 34 +chr6 160640666 160640848 kiv2_e05 39 +chr6 160644846 160645006 kiv2_e04 34 +chr6 160646213 160646395 kiv2_e03 39 diff --git a/tests/parity/inputs/cn/lpa-floor.q1.tsv b/tests/parity/inputs/cn/lpa-floor.q1.tsv new file mode 100644 index 000000000..db866d7e5 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-floor.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 3 +chr6 160612931 160613113 kiv2_e15 4 +chr6 160617116 160617276 kiv2_e14 3 +chr6 160618483 160618665 kiv2_e13 4 +chr6 160622662 160622822 kiv2_e12 3 +chr6 160624029 160624211 kiv2_e11 4 +chr6 160628206 160628366 kiv2_e10 3 +chr6 160629573 160629755 kiv2_e09 4 +chr6 160633752 160633912 kiv2_e08 3 +chr6 160635122 160635304 kiv2_e07 4 +chr6 160639299 160639459 kiv2_e06 3 +chr6 160640666 160640848 kiv2_e05 4 +chr6 160644846 160645006 kiv2_e04 3 +chr6 160646213 160646395 kiv2_e03 4 diff --git a/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv b/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv new file mode 100644 index 000000000..da76d893c --- /dev/null +++ b/tests/parity/inputs/cn/lpa-lowflank.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 564 +chr6 160547788 160547937 flank_e32 894 +chr6 160548477 160548659 flank_e31 1092 +chr6 160556024 160556184 flank_e30 960 +chr6 160557389 160557571 flank_e29 1092 +chr6 160577135 160577295 flank_e28 960 +chr6 160578522 160578704 flank_e27 1092 +chr6 160585045 160585205 flank_e26 960 +chr6 160586448 160586630 flank_e25 1092 +chr6 160589552 160589712 flank_e24 960 +chr6 160590943 160591101 flank_e23 948 +chr6 160593957 160594117 flank_e22 960 +chr6 160595353 160595535 flank_e21 1092 +chr6 160599499 160599659 flank_e20 960 +chr6 160600916 160601098 flank_e19 1092 +chr6 160611561 160611721 kiv2_e16 2194 +chr6 160612931 160613113 kiv2_e15 2496 +chr6 160617116 160617276 kiv2_e14 2194 +chr6 160618483 160618665 kiv2_e13 2496 +chr6 160622662 160622822 kiv2_e12 2194 +chr6 160624029 160624211 kiv2_e11 2496 +chr6 160628206 160628366 kiv2_e10 2194 +chr6 160629573 160629755 kiv2_e09 2496 +chr6 160633752 160633912 kiv2_e08 2194 +chr6 160635122 160635304 kiv2_e07 2496 +chr6 160639299 160639459 kiv2_e06 2194 +chr6 160640666 160640848 kiv2_e05 2496 +chr6 160644846 160645006 kiv2_e04 2194 +chr6 160646213 160646395 kiv2_e03 2496 diff --git a/tests/parity/inputs/cn/lpa-lowflank.q1.tsv b/tests/parity/inputs/cn/lpa-lowflank.q1.tsv new file mode 100644 index 000000000..f1f092779 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-lowflank.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 564 +chr6 160547788 160547937 flank_e32 894 +chr6 160548477 160548659 flank_e31 1092 +chr6 160556024 160556184 flank_e30 960 +chr6 160557389 160557571 flank_e29 1092 +chr6 160577135 160577295 flank_e28 960 +chr6 160578522 160578704 flank_e27 1092 +chr6 160585045 160585205 flank_e26 960 +chr6 160586448 160586630 flank_e25 1092 +chr6 160589552 160589712 flank_e24 960 +chr6 160590943 160591101 flank_e23 948 +chr6 160593957 160594117 flank_e22 960 +chr6 160595353 160595535 flank_e21 1092 +chr6 160599499 160599659 flank_e20 960 +chr6 160600916 160601098 flank_e19 1092 +chr6 160611561 160611721 kiv2_e16 219 +chr6 160612931 160613113 kiv2_e15 250 +chr6 160617116 160617276 kiv2_e14 219 +chr6 160618483 160618665 kiv2_e13 250 +chr6 160622662 160622822 kiv2_e12 219 +chr6 160624029 160624211 kiv2_e11 250 +chr6 160628206 160628366 kiv2_e10 219 +chr6 160629573 160629755 kiv2_e09 250 +chr6 160633752 160633912 kiv2_e08 219 +chr6 160635122 160635304 kiv2_e07 250 +chr6 160639299 160639459 kiv2_e06 219 +chr6 160640666 160640848 kiv2_e05 250 +chr6 160644846 160645006 kiv2_e04 219 +chr6 160646213 160646395 kiv2_e03 250 diff --git a/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv b/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv new file mode 100644 index 000000000..cf3ca6f12 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-mapq0.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10971 +chr6 160612931 160613113 kiv2_e15 12480 +chr6 160617116 160617276 kiv2_e14 10971 +chr6 160618483 160618665 kiv2_e13 12480 +chr6 160622662 160622822 kiv2_e12 10971 +chr6 160624029 160624211 kiv2_e11 12480 +chr6 160628206 160628366 kiv2_e10 10971 +chr6 160629573 160629755 kiv2_e09 12480 +chr6 160633752 160633912 kiv2_e08 10971 +chr6 160635122 160635304 kiv2_e07 12480 +chr6 160639299 160639459 kiv2_e06 10971 +chr6 160640666 160640848 kiv2_e05 12480 +chr6 160644846 160645006 kiv2_e04 10971 +chr6 160646213 160646395 kiv2_e03 12480 diff --git a/tests/parity/inputs/cn/lpa-mapq0.q1.tsv b/tests/parity/inputs/cn/lpa-mapq0.q1.tsv new file mode 100644 index 000000000..9039bd44a --- /dev/null +++ b/tests/parity/inputs/cn/lpa-mapq0.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10423 +chr6 160612931 160613113 kiv2_e15 11856 +chr6 160617116 160617276 kiv2_e14 10423 +chr6 160618483 160618665 kiv2_e13 11856 +chr6 160622662 160622822 kiv2_e12 10423 +chr6 160624029 160624211 kiv2_e11 11856 +chr6 160628206 160628366 kiv2_e10 10423 +chr6 160629573 160629755 kiv2_e09 11856 +chr6 160633752 160633912 kiv2_e08 10423 +chr6 160635122 160635304 kiv2_e07 11856 +chr6 160639299 160639459 kiv2_e06 10423 +chr6 160640666 160640848 kiv2_e05 11856 +chr6 160644846 160645006 kiv2_e04 10423 +chr6 160646213 160646395 kiv2_e03 11856 diff --git a/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv b/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv new file mode 100644 index 000000000..cf3ca6f12 --- /dev/null +++ b/tests/parity/inputs/cn/lpa-share1996.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 10971 +chr6 160612931 160613113 kiv2_e15 12480 +chr6 160617116 160617276 kiv2_e14 10971 +chr6 160618483 160618665 kiv2_e13 12480 +chr6 160622662 160622822 kiv2_e12 10971 +chr6 160624029 160624211 kiv2_e11 12480 +chr6 160628206 160628366 kiv2_e10 10971 +chr6 160629573 160629755 kiv2_e09 12480 +chr6 160633752 160633912 kiv2_e08 10971 +chr6 160635122 160635304 kiv2_e07 12480 +chr6 160639299 160639459 kiv2_e06 10971 +chr6 160640666 160640848 kiv2_e05 12480 +chr6 160644846 160645006 kiv2_e04 10971 +chr6 160646213 160646395 kiv2_e03 12480 diff --git a/tests/parity/inputs/cn/lpa-share1996.q1.tsv b/tests/parity/inputs/cn/lpa-share1996.q1.tsv new file mode 100644 index 000000000..fff6ba5cd --- /dev/null +++ b/tests/parity/inputs/cn/lpa-share1996.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 8782 +chr6 160612931 160613113 kiv2_e15 9989 +chr6 160617116 160617276 kiv2_e14 8782 +chr6 160618483 160618665 kiv2_e13 9989 +chr6 160622662 160622822 kiv2_e12 8782 +chr6 160624029 160624211 kiv2_e11 9989 +chr6 160628206 160628366 kiv2_e10 8782 +chr6 160629573 160629755 kiv2_e09 9989 +chr6 160633752 160633912 kiv2_e08 8782 +chr6 160635122 160635304 kiv2_e07 9989 +chr6 160639299 160639459 kiv2_e06 8782 +chr6 160640666 160640848 kiv2_e05 9989 +chr6 160644846 160645006 kiv2_e04 8782 +chr6 160646213 160646395 kiv2_e03 9989 diff --git a/tests/parity/inputs/cn/lpa-table.tsv b/tests/parity/inputs/cn/lpa-table.tsv new file mode 100644 index 000000000..48563d9dd --- /dev/null +++ b/tests/parity/inputs/cn/lpa-table.tsv @@ -0,0 +1,5 @@ +metric value +status not assessed +kiv2_copies 28.8 +mapq0_share 0.199 +message x diff --git a/tests/parity/inputs/cn/lpa12.bedcov.tsv b/tests/parity/inputs/cn/lpa12.bedcov.tsv new file mode 100644 index 000000000..184185374 --- /dev/null +++ b/tests/parity/inputs/cn/lpa12.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 4800 +chr6 160612931 160613113 kiv2_e15 5460 +chr6 160617116 160617276 kiv2_e14 4800 +chr6 160618483 160618665 kiv2_e13 5460 +chr6 160622662 160622822 kiv2_e12 4800 +chr6 160624029 160624211 kiv2_e11 5460 +chr6 160628206 160628366 kiv2_e10 4800 +chr6 160629573 160629755 kiv2_e09 5460 +chr6 160633752 160633912 kiv2_e08 4800 +chr6 160635122 160635304 kiv2_e07 5460 +chr6 160639299 160639459 kiv2_e06 4800 +chr6 160640666 160640848 kiv2_e05 5460 +chr6 160644846 160645006 kiv2_e04 4800 +chr6 160646213 160646395 kiv2_e03 5460 diff --git a/tests/parity/inputs/cn/lpa12.q1.tsv b/tests/parity/inputs/cn/lpa12.q1.tsv new file mode 100644 index 000000000..823c0dbf9 --- /dev/null +++ b/tests/parity/inputs/cn/lpa12.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2820 +chr6 160547788 160547937 flank_e32 4470 +chr6 160548477 160548659 flank_e31 5460 +chr6 160556024 160556184 flank_e30 4800 +chr6 160557389 160557571 flank_e29 5460 +chr6 160577135 160577295 flank_e28 4800 +chr6 160578522 160578704 flank_e27 5460 +chr6 160585045 160585205 flank_e26 4800 +chr6 160586448 160586630 flank_e25 5460 +chr6 160589552 160589712 flank_e24 4800 +chr6 160590943 160591101 flank_e23 4740 +chr6 160593957 160594117 flank_e22 4800 +chr6 160595353 160595535 flank_e21 5460 +chr6 160599499 160599659 flank_e20 4800 +chr6 160600916 160601098 flank_e19 5460 +chr6 160611561 160611721 kiv2_e16 480 +chr6 160612931 160613113 kiv2_e15 546 +chr6 160617116 160617276 kiv2_e14 480 +chr6 160618483 160618665 kiv2_e13 546 +chr6 160622662 160622822 kiv2_e12 480 +chr6 160624029 160624211 kiv2_e11 546 +chr6 160628206 160628366 kiv2_e10 480 +chr6 160629573 160629755 kiv2_e09 546 +chr6 160633752 160633912 kiv2_e08 480 +chr6 160635122 160635304 kiv2_e07 546 +chr6 160639299 160639459 kiv2_e06 480 +chr6 160640666 160640848 kiv2_e05 546 +chr6 160644846 160645006 kiv2_e04 480 +chr6 160646213 160646395 kiv2_e03 546 diff --git a/tests/parity/inputs/cn/lpa29.bedcov.tsv b/tests/parity/inputs/cn/lpa29.bedcov.tsv new file mode 100644 index 000000000..9005e6721 --- /dev/null +++ b/tests/parity/inputs/cn/lpa29.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2115 +chr6 160547788 160547937 flank_e32 5588 +chr6 160548477 160548659 flank_e31 4095 +chr6 160556024 160556184 flank_e30 6000 +chr6 160557389 160557571 flank_e29 4095 +chr6 160577135 160577295 flank_e28 6000 +chr6 160578522 160578704 flank_e27 4095 +chr6 160585045 160585205 flank_e26 6000 +chr6 160586448 160586630 flank_e25 4095 +chr6 160589552 160589712 flank_e24 6000 +chr6 160590943 160591101 flank_e23 3555 +chr6 160593957 160594117 flank_e22 6000 +chr6 160595353 160595535 flank_e21 4095 +chr6 160599499 160599659 flank_e20 6000 +chr6 160600916 160601098 flank_e19 4095 +chr6 160611561 160611721 kiv2_e16 13286 +chr6 160612931 160613113 kiv2_e15 9068 +chr6 160617116 160617276 kiv2_e14 13286 +chr6 160618483 160618665 kiv2_e13 9068 +chr6 160622662 160622822 kiv2_e12 13286 +chr6 160624029 160624211 kiv2_e11 9068 +chr6 160628206 160628366 kiv2_e10 13286 +chr6 160629573 160629755 kiv2_e09 9068 +chr6 160633752 160633912 kiv2_e08 13286 +chr6 160635122 160635304 kiv2_e07 9068 +chr6 160639299 160639459 kiv2_e06 13286 +chr6 160640666 160640848 kiv2_e05 9068 +chr6 160644846 160645006 kiv2_e04 13286 +chr6 160646213 160646395 kiv2_e03 9068 diff --git a/tests/parity/inputs/cn/lpa29.q1.tsv b/tests/parity/inputs/cn/lpa29.q1.tsv new file mode 100644 index 000000000..96a717a9c --- /dev/null +++ b/tests/parity/inputs/cn/lpa29.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2115 +chr6 160547788 160547937 flank_e32 5588 +chr6 160548477 160548659 flank_e31 4095 +chr6 160556024 160556184 flank_e30 6000 +chr6 160557389 160557571 flank_e29 4095 +chr6 160577135 160577295 flank_e28 6000 +chr6 160578522 160578704 flank_e27 4095 +chr6 160585045 160585205 flank_e26 6000 +chr6 160586448 160586630 flank_e25 4095 +chr6 160589552 160589712 flank_e24 6000 +chr6 160590943 160591101 flank_e23 3555 +chr6 160593957 160594117 flank_e22 6000 +chr6 160595353 160595535 flank_e21 4095 +chr6 160599499 160599659 flank_e20 6000 +chr6 160600916 160601098 flank_e19 4095 +chr6 160611561 160611721 kiv2_e16 1329 +chr6 160612931 160613113 kiv2_e15 907 +chr6 160617116 160617276 kiv2_e14 1329 +chr6 160618483 160618665 kiv2_e13 907 +chr6 160622662 160622822 kiv2_e12 1329 +chr6 160624029 160624211 kiv2_e11 907 +chr6 160628206 160628366 kiv2_e10 1329 +chr6 160629573 160629755 kiv2_e09 907 +chr6 160633752 160633912 kiv2_e08 1329 +chr6 160635122 160635304 kiv2_e07 907 +chr6 160639299 160639459 kiv2_e06 1329 +chr6 160640666 160640848 kiv2_e05 907 +chr6 160644846 160645006 kiv2_e04 1329 +chr6 160646213 160646395 kiv2_e03 907 diff --git a/tests/parity/inputs/cn/lpa47.bedcov.tsv b/tests/parity/inputs/cn/lpa47.bedcov.tsv new file mode 100644 index 000000000..b46886e05 --- /dev/null +++ b/tests/parity/inputs/cn/lpa47.bedcov.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2679 +chr6 160547788 160547937 flank_e32 4694 +chr6 160548477 160548659 flank_e31 5187 +chr6 160556024 160556184 flank_e30 5040 +chr6 160557389 160557571 flank_e29 5187 +chr6 160577135 160577295 flank_e28 5040 +chr6 160578522 160578704 flank_e27 5187 +chr6 160585045 160585205 flank_e26 5040 +chr6 160586448 160586630 flank_e25 5187 +chr6 160589552 160589712 flank_e24 5040 +chr6 160590943 160591101 flank_e23 4503 +chr6 160593957 160594117 flank_e22 5040 +chr6 160595353 160595535 flank_e21 5187 +chr6 160599499 160599659 flank_e20 5040 +chr6 160600916 160601098 flank_e19 5187 +chr6 160611561 160611721 kiv2_e16 18480 +chr6 160612931 160613113 kiv2_e15 17199 +chr6 160617116 160617276 kiv2_e14 18480 +chr6 160618483 160618665 kiv2_e13 17199 +chr6 160622662 160622822 kiv2_e12 18480 +chr6 160624029 160624211 kiv2_e11 17199 +chr6 160628206 160628366 kiv2_e10 18480 +chr6 160629573 160629755 kiv2_e09 17199 +chr6 160633752 160633912 kiv2_e08 18480 +chr6 160635122 160635304 kiv2_e07 17199 +chr6 160639299 160639459 kiv2_e06 18480 +chr6 160640666 160640848 kiv2_e05 17199 +chr6 160644846 160645006 kiv2_e04 18480 +chr6 160646213 160646395 kiv2_e03 17199 diff --git a/tests/parity/inputs/cn/lpa47.q1.tsv b/tests/parity/inputs/cn/lpa47.q1.tsv new file mode 100644 index 000000000..6df1103e9 --- /dev/null +++ b/tests/parity/inputs/cn/lpa47.q1.tsv @@ -0,0 +1,29 @@ +chr6 160545439 160545533 flank_e33 2679 +chr6 160547788 160547937 flank_e32 4694 +chr6 160548477 160548659 flank_e31 5187 +chr6 160556024 160556184 flank_e30 5040 +chr6 160557389 160557571 flank_e29 5187 +chr6 160577135 160577295 flank_e28 5040 +chr6 160578522 160578704 flank_e27 5187 +chr6 160585045 160585205 flank_e26 5040 +chr6 160586448 160586630 flank_e25 5187 +chr6 160589552 160589712 flank_e24 5040 +chr6 160590943 160591101 flank_e23 4503 +chr6 160593957 160594117 flank_e22 5040 +chr6 160595353 160595535 flank_e21 5187 +chr6 160599499 160599659 flank_e20 5040 +chr6 160600916 160601098 flank_e19 5187 +chr6 160611561 160611721 kiv2_e16 1848 +chr6 160612931 160613113 kiv2_e15 1720 +chr6 160617116 160617276 kiv2_e14 1848 +chr6 160618483 160618665 kiv2_e13 1720 +chr6 160622662 160622822 kiv2_e12 1848 +chr6 160624029 160624211 kiv2_e11 1720 +chr6 160628206 160628366 kiv2_e10 1848 +chr6 160629573 160629755 kiv2_e09 1720 +chr6 160633752 160633912 kiv2_e08 1848 +chr6 160635122 160635304 kiv2_e07 1720 +chr6 160639299 160639459 kiv2_e06 1848 +chr6 160640666 160640848 kiv2_e05 1720 +chr6 160644846 160645006 kiv2_e04 1848 +chr6 160646213 160646395 kiv2_e03 1720 diff --git a/tests/parity/inputs/cn/oracle.sh b/tests/parity/inputs/cn/oracle.sh new file mode 100644 index 000000000..09cfdb7d7 --- /dev/null +++ b/tests/parity/inputs/cn/oracle.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# The oracle of og cn (tests/parity/oracle.ref): bin/copy_number.py with the +# modules it imported (staged by gen/cn-oracle.sh), each og cn verb turned +# into the old command line. +# sh ./oracle.sh PROG VERB [ARGS] +# call ... --marker LPA_KIV2_4925 --marker-pileup P --marker-out O +# copy_number.py call ..., then lpa_kiv2.py null-allele --table T +# marker-regions NAME traits_report.py --marker-regions NAME +# marker-read NAME --pileup P [--copy-table T] --out O +# traits_report.py --marker NAME ... --marker-out O +d=$(mktemp -d) || exit 70 +trap 'rm -rf "$d"' EXIT +cp "$1" "${d}/copy_number.py" +cp hba_copy_number.py lpa_kiv2.py traits_report.py cn_loci.tsv "$d"/ +shift +verb=$1 +shift +case "$verb" in + marker-regions) python3 -B "${d}/traits_report.py" --marker-regions "$@"; exit $? ;; + marker-read) + name=$1 + shift + set -- "$@" --end + while [ "$1" != --end ]; do + case "$1" in + --out) set -- "$@" --marker-out "$2" ;; + --catalog) ;; + *) set -- "$@" "$1" "$2" ;; + esac + shift 2 + done + shift + python3 -B "${d}/traits_report.py" --marker "$name" "$@" + exit $? + ;; + call) + pileup="" out="" table="" + set -- "$@" --end + while [ "$1" != --end ]; do + case "$1" in + --marker) ;; + --marker-pileup) pileup=$2 ;; + --marker-out) out=$2 ;; + --table) table=$2; set -- "$@" "$1" "$2" ;; + *) set -- "$@" "$1" "$2" ;; + esac + shift 2 + done + shift + python3 -B "${d}/copy_number.py" call "$@" || exit $? + [ -z "$out" ] || python3 -B "${d}/lpa_kiv2.py" null-allele --pileup "$pileup" --table "$table" --out "$out" + exit $? + ;; + *) python3 -B "${d}/copy_number.py" "$verb" "$@"; exit $? ;; +esac diff --git a/tests/parity/inputs/cn/rhd0.bedcov.tsv b/tests/parity/inputs/cn/rhd0.bedcov.tsv new file mode 100644 index 000000000..1783b7afc --- /dev/null +++ b/tests/parity/inputs/cn/rhd0.bedcov.tsv @@ -0,0 +1,13 @@ +chr1 25199999 25260000 RHD_FLANK_UP 1800030 +chr1 25272392 25330445 RHD_GENE 0 +chr1 25272508 25272695 RHD_EX1 0 +chr1 25284572 25284759 RHD_EX2 0 +chr1 25290640 25290791 RHD_EX3 0 +chr1 25300945 25301093 RHD_EX4 0 +chr1 25301519 25301686 RHD_EX5 0 +chr1 25303321 25303459 RHD_EX6 0 +chr1 25306595 25306729 RHD_EX7 0 +chr1 25316999 25317079 RHD_EX8 0 +chr1 25321888 25321962 RHD_EX9 0 +chr1 25328897 25330445 RHD_EX10 0 +chr1 25334999 25360000 RHD_FLANK_DOWN 750030 diff --git a/tests/parity/inputs/cn/rhd1.bedcov.tsv b/tests/parity/inputs/cn/rhd1.bedcov.tsv new file mode 100644 index 000000000..4547d8942 --- /dev/null +++ b/tests/parity/inputs/cn/rhd1.bedcov.tsv @@ -0,0 +1,13 @@ +chr1 25199999 25260000 RHD_FLANK_UP 1800030 +chr1 25272392 25330445 RHD_GENE 870795 +chr1 25272508 25272695 RHD_EX1 5610 +chr1 25284572 25284759 RHD_EX2 5610 +chr1 25290640 25290791 RHD_EX3 4530 +chr1 25300945 25301093 RHD_EX4 4440 +chr1 25301519 25301686 RHD_EX5 5010 +chr1 25303321 25303459 RHD_EX6 4140 +chr1 25306595 25306729 RHD_EX7 4020 +chr1 25316999 25317079 RHD_EX8 2400 +chr1 25321888 25321962 RHD_EX9 2220 +chr1 25328897 25330445 RHD_EX10 46440 +chr1 25334999 25360000 RHD_FLANK_DOWN 750030 diff --git a/tests/parity/inputs/cn/smn-lowq.tsv b/tests/parity/inputs/cn/smn-lowq.tsv new file mode 100644 index 000000000..0d52f91be --- /dev/null +++ b/tests/parity/inputs/cn/smn-lowq.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 LowQual 4 12 LowQual 2,2 12,12 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/smn-notseen.pileup b/tests/parity/inputs/cn/smn-notseen.pileup new file mode 100644 index 000000000..55f07ef90 --- /dev/null +++ b/tests/parity/inputs/cn/smn-notseen.pileup @@ -0,0 +1 @@ +chr5 70952074 N 30 TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT III diff --git a/tests/parity/inputs/cn/smn-present.pileup b/tests/parity/inputs/cn/smn-present.pileup new file mode 100644 index 000000000..b4375e16d --- /dev/null +++ b/tests/parity/inputs/cn/smn-present.pileup @@ -0,0 +1,3 @@ +chr5 70952074 N 30 TTTTTTTTTTTTTTTTTTTTTTTTTGGGGG III +chr5 70076654 N 30 tttttttttttttttttttttttttttttt*<> III +chr5 70000000 N 30 GGGGGGGGGGGGGGGGGGGGGGGGGGGGGG III diff --git a/tests/parity/inputs/cn/smn-two-rows.tsv b/tests/parity/inputs/cn/smn-two-rows.tsv new file mode 100644 index 000000000..9e0f68c72 --- /dev/null +++ b/tests/parity/inputs/cn/smn-two-rows.tsv @@ -0,0 +1,3 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70951000 70953000 SMN1 PASS 4 30 LowQual 1,3 12,40 chr5:70075000-70078000 +chr5 70951900 70952100 SMN1 PASS 5 30 PASS 2,2 40,40 chr5:70076000-70076700 diff --git a/tests/parity/inputs/cn/smn1.tsv b/tests/parity/inputs/cn/smn1.tsv new file mode 100644 index 000000000..b5fe09c78 --- /dev/null +++ b/tests/parity/inputs/cn/smn1.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 PASS 3 30 PASS 1,2 40,40 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/smn2.tsv b/tests/parity/inputs/cn/smn2.tsv new file mode 100644 index 000000000..a8858aa7d --- /dev/null +++ b/tests/parity/inputs/cn/smn2.tsv @@ -0,0 +1,2 @@ +chrom start end locus agCN_filter agCN agCN_qual psCN_filter psCN psCN_qual homologous_regions +chr5 70925000 70955000 SMN1 PASS 4 30 PASS 2,2 40,40 chr5:70049000-70080000 diff --git a/tests/parity/inputs/cn/stdin b/tests/parity/inputs/cn/stdin new file mode 100644 index 000000000..7a4cb7036 --- /dev/null +++ b/tests/parity/inputs/cn/stdin @@ -0,0 +1,2 @@ +chr1 +chr2_KI270769v1_alt diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index ad4c6688b..12f0e0a57 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -12,3 +12,4 @@ sv-inv-filter-py3 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/manta_inv_fi pgx-refblock-fill 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_refblock_fill.py python3 pgx refblock-fill pgx-mito-merge 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_mito_merge.py python3 pgx mito-merge chip 6f2407f9f66cc6bee43ae13c8abae5874b53e1cf file:bin/chip_report.py python3 chip +cn 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/copy_number.py sh ./oracle.sh cn From 097f6c13bc4ad68d8b6c835662ed60c86165e5ec Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:26:40 +0100 Subject: [PATCH 09/53] test(live): check that each demo person page draws the genome map The demo pages must draw Figure 1 from the og map document, place every finding the old karyogram placed, keep closed results off the map, name missing layers, and fall back to a table without a map. The checks run against main first, so they fail until the map lands. Bead: pgp-iqon.8 --- tests/demo/build_live_demo.py | 72 ++++- tests/fixtures/map/karyogram-marks.tsv | 16 + tests/test_live_demo.py | 415 ++++++++++++++++++++++++- tests/test_live_results.py | 50 ++- 4 files changed, 541 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/map/karyogram-marks.tsv diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index f30a624b8..bee9cdb78 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """build_live_demo.py: the live site's public demo, a static folder replaying an invented run. - build_live_demo.py --out DIR [--genome-dir GDIR] [--records N] [--work WDIR] + build_live_demo.py --out DIR [--genome-dir GDIR] [--records N] [--work WDIR] [--og OG] writes DIR, which the docs workflow publishes at /demo/ of the docs site: @@ -13,6 +13,11 @@ runs/DEMO-001/results.json the person's results (bin/live_results.py), as the results plugin serves them at the end of the run; each card shows once the replay reaches the end of its steps + runs/DEMO-001/map.json the genome map of Figure 1: `og map build` with those results on + stdin, as the map plugin (bin/owngenome_live_map.py) sends it + (gzip bytes); the same for each person + map/genes/ `og map genes` of the public GENCODE slice + (tests/fixtures/map/gencode-slice), as /map/genes/ sends them runs/DEMO-001/finding// one page per finding (assets/live/finding.html), as the results plugin's finding route serves it runs/DEMO-001/... the files the plugins point at (the charts, the @@ -56,7 +61,12 @@ meta tag cannot set, left out), and the site's absolute links (/runs/..., /family/...) become relative ones; - the genome viewer opens the reader's own reads on the reader's own - computer, so the demo's viewer/ page says it is not in the demo. + computer, so the demo's viewer/ page says it is not in the demo; + - the map's genes come from the GENCODE slice copied into the run folder's + reference/, since setup has not installed GENCODE there. The map needs og + (OG, default bin/og, which scripts/setup.sh --og builds): without it the + build writes no map.json, says so, and the page lists the findings with a + place in a table instead. Standard library only. tests/test_live_demo.py builds it and checks it. """ @@ -102,6 +112,8 @@ DEMO_RAM = 64 << 30 DEMO_DISK = os.statvfs_result((4096, 4096, 512 << 20, 230 << 20, 230 << 20, 128 << 20, 100 << 20, 100 << 20, 0, 255)) INTERVAL_MS = 600 +OG = os.path.join(BIN, "og") +GENCODE_SLICE = os.path.join(REPO, "tests", "fixtures", "map", "gencode-slice", "reference") # The server's policy; a meta tag cannot set frame-ancestors (the browser ignores it there and warns). CSP = "; ".join(p for p in owngenome_live.CSP.split("; ") if not p.startswith("frame-ancestors")) @@ -167,6 +179,9 @@ def prepare(root, genome_dir=None, records=20000): other_person(root, name, sex, seed, max(2000, records // 4)) fam = os.path.join(root, FAMILY) shutil.copytree(os.path.join(EXTRA, FAMILY), fam) + # The genes the map places a gene-level finding on (and draws later): setup's GENCODE files, here the slice. + if not os.path.isdir(os.path.join(root, "reference")): + shutil.copytree(GENCODE_SLICE, os.path.join(root, "reference")) couple = os.path.join(d, "couple", f"{SAMPLE}_couple_carrier.tsv") if os.path.isfile(couple): a, b = sorted((SAMPLE, PARTNER)) @@ -383,7 +398,44 @@ def person_site(out, root, name, plugin_js, frames, results): "../../../assets/live.css")) -def write_site(out, root, frames, fam): +def og_map(og, args, stdin=b""): + """og map ARGS: its stdout, or None when og is not built on this machine (exit 127).""" + try: + p = subprocess.run([og, "map"] + args, input=stdin, capture_output=True, check=False) + except FileNotFoundError: + return None + if p.returncode == 127: + return None + if p.returncode != 0: + raise SystemExit(f"ERROR: og map {args[0]} failed ({p.returncode}): {p.stderr.decode(errors='replace').strip()}") + return p.stdout + + +def map_files(out, root, people, og): + """runs//map.json for each (name, results) of PEOPLE and map/genes/, as the map plugin sends + them; False (and a note) when og is not built here.""" + cache = os.path.join(root, ".owngenome-live", "map") + assets = os.path.join(REPO, "assets") + for name, results in people: + doc = og_map(og, ["build", "--root", root, "--run", name, "--assets", assets, "--cache-dir", cache], + json.dumps({"results": results}).encode()) + if doc is None: + print(f"NOTE: og is not built here ({og}): the demo has no genome map; scripts/setup.sh --og builds it", + file=sys.stderr) + return False + with open(os.path.join(out, "runs", name, "map.json"), "wb") as f: + f.write(doc) + gd = os.path.join(out, "map", "genes") + os.makedirs(gd) + chroms = [f"chr{c}" for c in list(range(1, 23)) + ["X", "Y"]] + for which in chroms + ["names"]: + data = og_map(og, ["genes", "--root", root, "--cache-dir", cache] + (["--names"] if which == "names" else ["--chrom", which])) + with open(os.path.join(gd, f"{which}.json"), "wb") as f: + f.write(data) + return True + + +def write_site(out, root, frames, fam, og=OG): if os.path.exists(out): shutil.rmtree(out) shutil.copytree(ASSETS, os.path.join(out, "assets"), @@ -400,6 +452,7 @@ def write_site(out, root, frames, fam): for name, _, _ in OTHERS: o = fam["others"][name] person_site(out, root, name, plugin_js, o["frames"], o["results"]) + drawn = map_files(out, root, [(SAMPLE, fam["results"])] + [(n, fam["others"][n]["results"]) for n, _, _ in OTHERS], og) fd = os.path.join(out, "runs", FAMILY) os.makedirs(fd) @@ -414,10 +467,12 @@ def write_site(out, root, frames, fam): f.write(pedigree_svg.draw(order, members, FAMILY)) for name in (fam["state"]["plugins"].get("family") or {}).get("files") or []: shutil.copy(os.path.join(ffiles, name), os.path.join(famdir, name)) + return drawn -def build(out, genome_dir=None, records=20000, work=None, drop_hook=None): - """Write the demo into out; returns {frames, hooks, family} for the tests.""" +def build(out, genome_dir=None, records=20000, work=None, drop_hook=None, og=OG): + """Write the demo into out; returns {frames, hooks, family, map} for the tests (map: the map + documents were written).""" tmp = None if work: os.makedirs(work, exist_ok=True) @@ -430,8 +485,8 @@ def build(out, genome_dir=None, records=20000, work=None, drop_hook=None): try: held = prepare(root, genome_dir, records) frames, fam, app = replay(root, held, drop_hook) - write_site(out, root, frames, fam) - return {"frames": frames, "family": fam, "hooks": [n for n, _ in app.state_hooks]} + drawn = write_site(out, root, frames, fam, og) + return {"frames": frames, "family": fam, "hooks": [n for n, _ in app.state_hooks], "map": drawn} finally: if tmp: shutil.rmtree(tmp, ignore_errors=True) @@ -443,8 +498,9 @@ def main(argv=None): ap.add_argument("--genome-dir", help="a folder holding DEMO-001 (default: write one now)") ap.add_argument("--records", type=int, default=20000, help="variant records of a DEMO-001 written now") ap.add_argument("--work", help="an empty folder for the run folder, kept afterwards (default: a temporary one)") + ap.add_argument("--og", default=OG, help="the og that builds the map documents (default: bin/og)") a = ap.parse_args(argv) - r = build(a.out, a.genome_dir, a.records, a.work) + r = build(a.out, a.genome_dir, a.records, a.work, og=a.og) size = sum(os.path.getsize(os.path.join(p, f)) for p, _, fs in os.walk(a.out) for f in fs) print(f"Demo written: {a.out} ({len(r['frames'])} frames, {size // 1024} KB; plugins {', '.join(r['hooks'])})") return 0 diff --git a/tests/fixtures/map/karyogram-marks.tsv b/tests/fixtures/map/karyogram-marks.tsv new file mode 100644 index 000000000..6104ef186 --- /dev/null +++ b/tests/fixtures/map/karyogram-marks.tsv @@ -0,0 +1,16 @@ +# The marks the findings karyogram (core.js karyogram(), person.js Figure 1) drew on the demo's person pages, +# recorded before the genome map replaced it: one row per mark, from each page at the end of its run. A mark is a +# finding with a viewer place (chrN:pos or chrN:start-end) that the page did not keep closed (DEMO-003 is a child, +# so its results to act on and the findings tagged with them were not drawn). mb is the place's first +# position in Mb, rounded to 0.01. tests/test_live_demo.py checks the map places each one. +run id chr mb level +DEMO-001 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-001 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-001 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-002 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-002 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-002 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-003 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-003 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-003 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-003 sv-chr16-2084000-2096000 16 2.08 review diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 805606cfb..0fedd27d5 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -72,10 +72,13 @@ Run: python3 tests/test_live_demo.py """ +import base64 import copy +import gzip import json import os import shutil +import subprocess import sys import tempfile import threading @@ -314,6 +317,7 @@ def browser_checks(out): fold_checks(b, base, out) veil_checks(b, base, out) planted_checks(b, base, out) + map_checks(b, base, out) finally: b.close() srv.shutdown() @@ -1175,6 +1179,412 @@ def fold_checks(b, base, out): f.write(keep_js) +# --- the genome map (Figure 1, assets/live/map.js) ------------------------------------------ + +MARKS_TSV = os.path.join(REPO, "tests", "fixtures", "map", "karyogram-marks.tsv") + + +def og_binary(): + """An og that runs here: bin/og when scripts/setup.sh --og built it, else one cargo builds now from rust/ + without the BAM feature (the map reads no BAM), kept in CARGO_TARGET_DIR or a folder of the temp dir. + None when neither works.""" + og = os.path.join(REPO, "bin", "og") + if subprocess.run([og, "map", "--help"], capture_output=True, check=False).returncode == 0: + return og + cargo = shutil.which("cargo") or os.path.expanduser("~/.cargo/bin/cargo") + if not os.path.isfile(cargo): + return None + target = os.environ.get("CARGO_TARGET_DIR") or os.path.join(tempfile.gettempdir(), "owngenome-test-og") + p = subprocess.run([cargo, "build", "--locked", "--quiet", "-p", "og", "--no-default-features"], capture_output=True, + text=True, cwd=os.path.join(REPO, "rust"), env=dict(os.environ, CARGO_TARGET_DIR=target), check=False) + exe = os.path.join(target, "debug", "og") + if p.returncode != 0 or not os.path.isfile(exe): + print(f"cargo could not build og ({p.returncode}): {p.stderr[-2000:]}", file=sys.stderr) + return None + return exe + + +def karyogram_marks(): + """(run, id, chrom, mb, level) of each mark the karyogram drew, recorded before the map replaced it.""" + rows = [] + with open(MARKS_TSV) as f: + for line in f: + if line.startswith(("#", "run\t")) or not line.strip(): + continue + run, fid, c, m, lv = line.rstrip("\n").split("\t") + rows.append((run, fid, "chr" + c, float(m), lv)) + return rows + + +def read_map(path): + with open(path, "rb") as f: + return json.loads(gzip.decompress(f.read())) + + +def write_map(path, doc): + with open(path, "wb") as f: + f.write(gzip.compress(json.dumps(doc, separators=(",", ":")).encode(), mtime=0)) + + +# What the map shows: its state, the findings it draws (its list of marks), its timings, level and place, +# its chips, its margin's words and every text and title it holds, and the address. +MAP_JS = r"""JSON.stringify((function () { + var f = document.getElementById('genome-map'); + if (!f) { return { state: 'absent', marks: [] }; } + var t = document.querySelector('.gm-tip'); + return { state: f.dataset.pgpMap || '', level: f.dataset.level || '', chrom: f.dataset.chrom || '', view: f.dataset.view || '', + parse: Number(f.dataset.pgpMapParseMs), paint: Number(f.dataset.pgpMapPaintMs), + marks: Array.prototype.map.call(f.querySelectorAll('.gm-marks a'), function (a) { return { id: a.dataset.id, + level: a.dataset.level, chrom: a.dataset.chrom, s: Number(a.dataset.s), e: Number(a.dataset.e) }; }), + chips: Array.prototype.map.call(f.querySelectorAll('.gm-chip'), function (c) { return c.textContent; }), + title: (document.getElementById('gm-title') || {}).textContent || '', + layers: Array.prototype.map.call(f.querySelectorAll('.gm-layer'), function (l) { return [l.dataset.layer, l.dataset.state, + l.querySelector('dd').textContent]; }), + toast: (f.querySelector('.gm-toast') || { hidden: true }).hidden ? '' : f.querySelector('.gm-toast').textContent, + tip: t && !t.hidden ? t.textContent : '', + text: [f.textContent].concat(Array.prototype.map.call(f.querySelectorAll('[href],[title],[aria-label]'), function (e) { + return (e.getAttribute('href') || '') + ' ' + (e.getAttribute('title') || '') + ' ' + (e.getAttribute('aria-label') || ''); })).join(' '), + hash: location.hash }; +}()))""" +MAP_READY = ("document.documentElement.dataset.pgpResults && (function () { var f = document.getElementById('genome-map'); " + "return f && /^(drawn|failed|nodecompress)$/.test(f.dataset.pgpMap || '') && f.dataset.level !== undefined " + "&& (f.dataset.pgpMap !== 'drawn' || !!f.dataset.level); }())") + + +def map_page(b, base, run, hash_=""): + """A fresh load of RUN's page with HASH_; the map's facts once it settled (None when it never drew).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{run}/{hash_}") + if not b.wait(MAP_READY): + return json.loads(b.eval(MAP_JS)) + time.sleep(0.6) # the margin follows the camera 140 ms after it settles + return json.loads(b.eval(MAP_JS)) + + +def map_now(b): + time.sleep(1.5) # the camera eases for about 0.6 s, then the margin follows 140 ms after it settles + return json.loads(b.eval(MAP_JS)) + + +def parity_problems(b, base, rows): + """The karyogram's marks the map does not place with the same level and chromosome within 0.01 Mb.""" + bad = [] + for run in sorted({r[0] for r in rows}): + got = map_page(b, base, run) + if got["state"] != "drawn": + bad.append(f"{run}: map {got['state']}") + continue + by = {m["id"]: m for m in got["marks"]} + for _, fid, c, m, lv in (r for r in rows if r[0] == run): + g = by.get(fid) + if not g: + bad.append(f"{run}: {fid} not on the map") + elif g["level"] != lv or g["chrom"] != c or abs(g["s"] / 1e6 - m) > 0.01: + bad.append(f"{run}: {fid} at {g['chrom']} {g['s'] / 1e6:.3f} Mb {g['level']}, want {c} {m} {lv}") + return bad + + +def click_canvas(b, x, y, move_only=False): + """A mouse at (x, y) of the map's canvas, in CSS px from its corner: moved there, and clicked unless MOVE_ONLY.""" + b.eval("document.querySelector('.gm-canvas').scrollIntoView({block: 'center'}); true") + time.sleep(0.3) + r = json.loads(b.eval("JSON.stringify(document.querySelector('.gm-canvas').getBoundingClientRect())")) + px, py = r["left"] + x, r["top"] + y + b.call("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": px, "y": py}, b.session) + if not move_only: + for t in ("mousePressed", "mouseReleased"): + b.call("Input.dispatchMouseEvent", {"type": t, "x": px, "y": py, "button": "left", "clickCount": 1}, b.session) + time.sleep(0.6) + + +def genome_xy(b, doc, chrom, bp): + """Where BP of CHROM sits on the genome level of the canvas (map.js gx and rowY, at the desktop width).""" + cw = b.eval("document.querySelector('.gm-canvas').clientWidth") + rows = [c for c in doc["ref"]["chroms"] if not (c["n"] == "chrY" and doc["people"][0].get("has_y") is False)] + i = [c["n"] for c in rows].index(chrom) + ch = b.eval("document.querySelector('.gm-canvas').clientHeight") + rh = max(22, min(44, (ch - 18) / len(rows))) + return 40 + bp * (cw - 40 - 22) / rows[0]["len"], 9 + i * rh + rh / 2 - 1 + + +def state_problems(b, base): + """The genome and chromosome levels, the links that open each, a click on a row, Esc, and a place that + cannot be read.""" + bad = [] + g = map_page(b, base, S) + if not (g["level"] == "genome" and "all 24 chromosomes" in g["title"] and g["view"] == "genome"): + bad.append(f"genome: {g['level']} {g['title']!r} {g['view']}") + for link, chrom, view in (("#chr16", "chr16", "chr16"), ("#chr16:10.0-40.0M", "chr16", "chr16:10.0-40.0M"), + ("#view=chromosome&chr=11", "chr11", "chr11"), ("#chr2:20000001-60000000", "chr2", "chr2:20.0-60.0M")): + c = map_page(b, base, S, link) + if not (c["level"] == "chromosome" and c["chrom"] == chrom and c["view"] == view): + bad.append(f"{link}: {c['level']} {c['chrom']} {c['view']}") + elif any(m["chrom"] != chrom for m in c["marks"]): + bad.append(f"{link}: marks of another chromosome listed: {c['marks']}") + c = map_page(b, base, S, "#chr16") + b.eval("location.hash = '#genome'; true") + c = map_now(b) + if c["level"] != "genome": + bad.append(f"#genome after #chr16: {c['level']}") + b.eval("location.hash = '#chrQ'; true") + c = map_now(b) + if "Could not read the place" not in c["toast"] or c["level"] != "genome": + bad.append(f"#chrQ: toast {c['toast']!r}, level {c['level']}") + g = map_page(b, base, S) + click_canvas(b, 200, 9 + 2 * 22 + 11) + c = map_now(b) + if not (c["level"] == "chromosome" and c["chrom"] == "chr3" and c["hash"] == "#chr3"): + bad.append(f"a click on chromosome 3's row: {c['level']} {c['chrom']} {c['hash']}") + b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', bubbles: true})); true") + c = map_now(b) + if c["level"] != "genome" or c["hash"] != "#genome": + bad.append(f"Esc: {c['level']} {c['hash']}") + return bad + + +def map_veil_problems(b, base, out): + """On the son's closed page: his closed result to act on (the planted deletion) is in no mark, tooltip, + chip, sentence, link or address of the map until opened; the chip's Open (after its confirm) and the card's + "Show the result" each bring back exactly its mark, on the map and in the card; Close it again and a reload + close it. On DEMO-001, an adult's page, APOE stays closed until opened, from the chip or its card's fold.""" + bad = [] + son = OTHERS[1] + doc = read_map(os.path.join(out, "runs", son, "map.json")) + f = doc["per"][son]["findings"] + acts = [i for i, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][i]] + if not acts: + return ["the son's map has no result to act on to close"] + i = acts[0] + fid, gene = f["id"][i], f["gene"][i] + words = [gene, f"{f['chrom'][i]}:{f['s'][i] + 1}", f"{f['s'][i] + 1:,}", fid] + x, y = genome_xy(b, doc, f["chrom"][i], (f["s"][i] + f["e"][i]) / 2) + + def leaks(c): + return [w for w in words if w in c["text"] or w in c["tip"] or w in c["hash"]] + \ + [m["id"] for m in c["marks"] if m["id"] == fid] + c = map_page(b, base, son) + click_canvas(b, x, y, move_only=True) + c = map_now(b) + if c["state"] != "drawn": + return [f"the son's map: {c['state']}"] + if leaks(c): + bad.append(f"closed: the map shows {leaks(c)}") + if not any("1 result closed" in t for t in c["chips"]): + bad.append(f"closed: no chip counting it: {c['chips']}") + b.eval("document.querySelector('.gm-chip[data-closed=act] button').click(); true") + c = map_now(b) + if not b.eval("!!document.querySelector('.gm-chip[data-closed=act] .gm-confirm')") or leaks(c): + bad.append("Open: no confirm step, or it opened at once") + b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-yes').click(); true") + c = map_now(b) + card = b.eval("!!document.querySelector('#f-%s h3') && !document.querySelector('#f-%s .veil')" % (fid, fid)) + if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid] or not card: + bad.append(f"Open: mark {[m['id'] for m in c['marks']]}, card shown {card}") + if not any(t.startswith("Opened: " + f["title"][i]) for t in c["chips"]): + bad.append(f"Open: chips {c['chips']}") + if fid in c["hash"] or gene in c["hash"]: + bad.append(f"Open: the address names it: {c['hash']}") + click_canvas(b, x, y, move_only=True) + c = map_now(b) + if gene not in c["tip"]: + bad.append(f"Open: its tooltip at its place does not name it: {c['tip']!r}") + b.eval("document.querySelector('.gm-chip.opened button').click(); true") + c = map_now(b) + card = b.eval("!!document.querySelector('#f-%s .veil')" % fid) + if leaks(c) or not card: + bad.append(f"Close it again: map shows {leaks(c)}, card closed {card}") + b.eval("document.querySelector('#f-%s .veil button').click(); true" % fid) + c = map_now(b) + if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid]: + bad.append("the card's Show the result does not open it on the map") + c = map_page(b, base, son) + if leaks(c): + bad.append(f"a reload does not close it: {leaks(c)}") + # APOE on an adult's page. + c = map_page(b, base, S) + if any(m["id"] == "apoe" for m in c["marks"]) or not any(t.startswith("APOE closed") for t in c["chips"]): + bad.append(f"APOE on DEMO-001: marks {[m['id'] for m in c['marks']]}, chips {c['chips']}") + b.eval("document.querySelector('.gm-chip[data-closed=gate] button').click(); true") + time.sleep(0.3) + b.eval("document.querySelector('.gm-chip[data-closed=gate] .gm-yes').click(); true") + c = map_now(b) + fold = b.eval("(document.querySelector('details[data-gate=apoe]') || {}).open") + if not any(m["id"] == "apoe" for m in c["marks"]) or fold is not True: + bad.append(f"Open APOE: marks {[m['id'] for m in c['marks']]}, its card's fold open {fold}") + b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + c = map_now(b) + if any(m["id"] == "apoe" for m in c["marks"]): + bad.append("closing APOE's card fold leaves it on the map") + b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + c = map_now(b) + if not any(m["id"] == "apoe" for m in c["marks"]): + bad.append("opening APOE's card fold does not open it on the map") + c = map_page(b, base, S) + if any(m["id"] == "apoe" for m in c["marks"]): + bad.append("a reload does not close APOE") + return bad + + +def missing_layer_problems(b, base, out): + """A map whose ROH layer is missing says so in words, in the margin, never a blank.""" + path = os.path.join(out, "runs", S, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + doc["layers"]["roh"] = dict(doc["layers"]["roh"], state="missing", count=0, note="planted: no ROH text") + doc["per"][S].pop("roh", None) + write_map(path, doc) + c = map_page(b, base, S, "#chr1") + row = [x for x in c["layers"] if x[0] == "roh"] + if not row or row[0][1] != "missing" or "Not on the map: planted: no ROH text" not in row[0][2]: + return [f"the missing ROH layer reads {row}"] + return [] + finally: + with open(path, "wb") as fh: + fh.write(keep) + + +def fallback_problems(b, base, out): + """A browser with no DecompressionStream: the sentence and the findings with a place, each through the veil.""" + son = OTHERS[1] + doc = read_map(os.path.join(out, "runs", son, "map.json")) + f = doc["per"][son]["findings"] + gene = [f["gene"][i] for i, lv in enumerate(f["level"]) if lv == "act"][0] + b.call("Page.enable", session=b.session) # a script for each new document needs the Page domain on + sid = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "delete window.DecompressionStream;"}, b.session)["identifier"] + try: + c = map_page(b, base, son) + rows = b.eval("document.querySelectorAll('.gm-fallback table tbody tr').length") + finally: + b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": sid}, b.session) + bad = [] + if c["state"] != "nodecompress" or "This browser cannot open the map file" not in c["text"]: + bad.append(f"no DecompressionStream: {c['state']}") + if not rows: + bad.append("no DecompressionStream: no findings table") + if gene in c["text"]: + bad.append(f"no DecompressionStream: the table names the closed {gene}") + return bad + + +def map_screens(b, base, dest): + """Screenshots of the map, dark, at 1440 x 900 and 390 x 844: genome and chromosome, a closed page.""" + os.makedirs(dest, exist_ok=True) + taken = [] + try: + for w, h in ((1440, 900), (390, 844)): + b.call("Emulation.setDeviceMetricsOverride", {"width": w, "height": h, "deviceScaleFactor": 1, "mobile": False}, + b.session) + for run, link, name in ((S, "", "genome"), (S, "#chr16", "chromosome"), (OTHERS[1], "", "closed")): + map_page(b, base, run, link) + b.eval("document.getElementById('genome-map').scrollIntoView({block: 'start'}); true") + time.sleep(0.5) + png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] + p = os.path.join(dest, f"map-{name}-{w}x{h}.png") + with open(p, "wb") as fh: + fh.write(base64.b64decode(png)) + taken.append(p) + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + return taken + + +def map_checks(b, base, out): + """Figure 1, the genome map: drawn from map.json with every mark the karyogram drew, in time; its levels, + links and a place it cannot read; the veil and opening on purpose; a missing layer; no DecompressionStream; + no sideways scroll at 390 px.""" + if not os.path.isfile(os.path.join(out, "runs", S, "map.json")): + ci = os.environ.get("CI") == "true" + print(("[FAIL] " if ci else "[SKIP] ") + "the demo has no map.json: no og to build it") + if ci: + FAILS.append("no map.json in CI") + return + b.call("Network.enable", session=b.session) # the controls below change files: never serve them from cache + b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) + # A desktop window, so the whole canvas fits in view for the clicks. + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, + b.session) + rows = karyogram_marks() + bad = parity_problems(b, base, rows) + check(f"map: the DEMO-001 to DEMO-003 person pages draw the map from map.json and every karyogram mark " + f"({len(rows)} marks: same level and chromosome, within 0.01 Mb)", not bad and len(rows) >= 9, bad) + path = os.path.join(out, "runs", S, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + f = doc["per"][S]["findings"] + drop = f["id"].index(next(r[1] for r in rows if r[0] == S)) + for k in f: + f[k] = [v for j, v in enumerate(f[k]) if j != drop] + write_map(path, doc) + control("a map.json without one of the karyogram's findings", not parity_problems(b, base, rows)) + finally: + with open(path, "wb") as fh: + fh.write(keep) + g = map_page(b, base, S) + check(f"map: first paint within 150 ms of the document arriving, parsed within 30 ms ({g.get('paint')} ms, " + f"{g.get('parse')} ms)", g.get("paint", 999) <= 150 and g.get("parse", 999) <= 30, g) + bad = state_problems(b, base) + check("map: genome and chromosome levels; #chr16, #chr16:10.0-40.0M, chr=11 and a 1-based span open their view; " + "a click on a row opens it, Esc closes it; an unknown place gives a toast", not bad, bad) + js = os.path.join(out, "assets", "person.js") + with open(js) as fh: + keep_js = fh.read() + bad = map_veil_problems(b, base, out) + check("map: the son's closed result is in no mark, tooltip, chip, sentence, link or address until opened; Open " + "(after its confirm) and the card's Show the result each bring back exactly its mark there and in the card; " + "Close it again and a reload close it; APOE stays closed on an adult's page until opened, from the chip " + "or its card's fold", not bad, bad) + try: + with open(js, "w") as fh: + fh.write(keep_js.replace("function mapVeil(id, gated) {", "function mapVeil(id, gated) { return 'open';")) + control("person.js whose map veil keeps nothing closed", not map_veil_problems(b, base, out)) + finally: + with open(js, "w") as fh: + fh.write(keep_js) + bad = missing_layer_problems(b, base, out) + check("map: a missing layer says why in words, never a blank", not bad, bad) + bad = fallback_problems(b, base, out) + check("map: a browser with no DecompressionStream gets the sentence and a findings table through the veil", not bad, bad) + b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, b.session) + try: + wide = {} + for link in ("", "#chr16"): + map_page(b, base, S, link) + wide[link or "genome"] = too_wide(b) + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + check("map: at 390 px the map, open or not, scrolls nothing sideways", not any(wide.values()), wide) + if os.environ.get("MAP_SCREENS"): + print("map screenshots: " + ", ".join(map_screens(b, base, os.environ["MAP_SCREENS"]))) + + +def screens_main(dest, site): + """--map-screens DEST SITE: the map's screenshots of an already built demo (the docs workflow's artifact).""" + chrome = find_chrome() + if not chrome: + print("ERROR: no Chrome or Chromium to take the screenshots") + return 1 + handler = type("H", (Static,), {"__init__": lambda self, *a, **k: SimpleHTTPRequestHandler.__init__( + self, *a, directory=site, **k)}) + srv = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + work = tempfile.mkdtemp(prefix="map-screens.") + b = Chrome(chrome, os.path.join(work, "chrome")) + try: + taken = map_screens(b, f"http://127.0.0.1:{srv.server_address[1]}", dest) + finally: + b.close() + srv.shutdown() + srv.server_close() + shutil.rmtree(work, ignore_errors=True) + print("\n".join(taken)) + return 0 if len(taken) == 6 else 1 + + def main(): work = tempfile.mkdtemp(prefix="test_live_demo.") try: @@ -1188,7 +1598,8 @@ def main(): control("a fixture missing its last message", make_demo_events.main(["--check", "--out", changed]) == 0) out = os.path.join(work, "demo") - built = build_live_demo.build(out, work=os.path.join(work, "root")) + og = og_binary() or build_live_demo.OG + built = build_live_demo.build(out, work=os.path.join(work, "root"), og=og) frames = built["frames"] print(f"built: {len(frames)} frames, hooks {built['hooks']}") @@ -1267,4 +1678,6 @@ def main(): if __name__ == "__main__": + if sys.argv[1:2] == ["--map-screens"] and len(sys.argv) == 4: + sys.exit(screens_main(sys.argv[2], sys.argv[3])) sys.exit(main()) diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 8213596a2..3e304b9aa 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -10,9 +10,12 @@ its SECTION_NOTES constants, word for word, never empty; 3. the wording gate: no string of results.json, of family.json (tests/fixtures/families_live family_a) or of the page assets - (assets/live: the text of the pages, the string literals of the scripts) - says a result is fine (live_results.FORBIDDEN), a quoted tool term and a - caveat aside; + (assets/live: the text of the pages, the string literals of the scripts, + map.js among them) says a result is fine (live_results.FORBIDDEN), a quoted + tool term and a caveat aside; nor does any string of the genome map + documents `og map build` writes for DEMO-001 and DEMO-003 (Figure 1 shows + their sentences), a layer's state code aside (skipped without og or + cargo, except in CI); 4. ClinVar: with hits the card lists them in a table; with none it has no table and its empty text says ClinVar lists only part of what causes disease (never an all-clear); @@ -145,6 +148,32 @@ def asset_problems(texts): return out +def map_documents(root, people): + """[(run, document)] of `og map build` for each (run, results) of PEOPLE, as the map plugin builds them; + None without an og (tests/test_live_demo.py og_binary).""" + import gzip + from test_live_demo import og_binary + og = og_binary() + if not og: + return None + ref = os.path.join(root, "reference") + if not os.path.isdir(ref): + shutil.copytree(os.path.join(REPO, "tests", "fixtures", "map", "gencode-slice", "reference"), ref) + out = [] + for run, results in people: + body = {k: v for k, v in results.items() if not k.startswith("_")} + p = subprocess.run([og, "map", "build", "--root", root, "--run", run, "--assets", os.path.join(REPO, "assets"), + "--cache-dir", os.path.join(root, ".map-cache")], input=json.dumps({"results": body}).encode(), + capture_output=True, check=True) + out.append((run, json.loads(gzip.decompress(p.stdout)))) + return out + + +def map_wording(doc): + """The strings of a map document that read as an all-clear; a layer's state is a code, not words.""" + return live_results.wording_problems(doc, allow=("state",)) + + # --- 4. ClinVar with and without hits ------------------------------------------------------- def clinvar_card(summary): @@ -534,6 +563,21 @@ def tagged(sample_dir): finally: live_results.veil_id = keep + # 3, the map documents: og map build of DEMO-001 and DEMO-003, with the results each page gets. + map_docs = map_documents(root, [(SAMPLE, res), ("DEMO-003", tagged(d3)[0])]) + if map_docs is None: + ci = os.environ.get("CI") == "true" + print(("[FAIL] " if ci else "[SKIP] ") + "3. no og (and no cargo to build one) for the map documents") + if ci: + FAILS.append("no og in CI") + else: + bad = [(run, x) for run, doc in map_docs for x in map_wording(doc)] + check(f"3. no string of the genome map documents reads as an all-clear ({len(map_docs)} documents)", + not bad and len(map_docs) == 2, bad) + planted = copy.deepcopy(map_docs[0][1]) + planted["text"]["genome"] = [{"t": "All clear on the map.", "b": False}] + control("'All clear' planted in a map document's sentence", not map_wording(planted)) + # 5. pwfile = os.path.join(work, "password") with open(pwfile, "w") as f: From 6c61e18fdc2b6320917d1300582bb51874ef8615 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:26:59 +0100 Subject: [PATCH 10/53] refactor(cn): steps 32, 35, 41 and 46 call og cn; the Python engine goes The callers and the Nextflow modules run og cn for the copy-number calls, the SMN1 and LPA markers and the region lists. hba_copy_number.py and lpa_kiv2.py are deleted; copy_number.py keeps the catalog reader bloodtype_report needs, and traits_report.py keeps MARKERS for collect_summary. The tests run the engine through og. Bead: pgp-bs9u.24 --- bin/copy_number.py | 535 +-------------------------- bin/hba_copy_number.py | 158 -------- bin/lpa_kiv2.py | 266 ------------- bin/traits_report.py | 215 +---------- modules/local/bloodtype/main.nf | 4 +- modules/local/lpa/main.nf | 32 +- modules/local/parascopy/main.nf | 34 +- modules/local/pypgx/main.nf | 4 +- scripts/32-pypgx.sh | 4 +- scripts/35-paralogs.sh | 46 +-- scripts/41-bloodtype.sh | 4 +- scripts/46-lpa.sh | 42 +-- tests/demo/make_demo_sample.py | 58 +-- tests/e2e/cn-engine-1-parity.sh | 17 +- tests/fake-docker/cn-engine-calls.sh | 38 +- tests/test_callability.py | 62 ++-- tests/test_collect_summary.py | 15 +- tests/test_copy_number.py | 425 ++++++++++++--------- tests/test_hba_copy_number.py | 65 +++- tests/test_lpa_fallback.sh | 12 +- tests/test_lpa_kiv2.py | 223 +++++++---- tests/test_traits_report.py | 46 ++- 22 files changed, 705 insertions(+), 1600 deletions(-) delete mode 100755 bin/hba_copy_number.py delete mode 100755 bin/lpa_kiv2.py diff --git a/bin/copy_number.py b/bin/copy_number.py index 84374c59c..e56144990 100755 --- a/bin/copy_number.py +++ b/bin/copy_number.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 -"""copy_number.py: copy number at any locus of bin/cn_loci.tsv from short-read depth. +"""copy_number.py: the catalog reader and the depth-ratio engine of +bin/cn_loci.tsv, as step 41's report (bin/bloodtype_report.py) imports them. + +The commands are og cn's (rust/crates/og-cn, docs/cn-loci.md): `og cn bed` +prints a locus's segments, `og cn call` turns `samtools bedcov` over them +into the step's table, `og cn intervals` reads the interval rows and `og cn +check-bam` refuses a BAM aligned with ALT contigs. This file keeps what +bloodtype_report.py reads the RHD row with until it moves to og too: +load_catalog, assess, cut_copies, BrokenControl and CatalogError. One method, one catalog row per locus: the depth over the locus's target segments against the depth over single-copy control segments, as @@ -9,73 +17,11 @@ then guarded by a minimum control depth, flank agreement and the share of the target depth from reads of mapping quality 0, and called as the row's `call` column says. docs/cn-loci.md describes the columns and how to add a locus. - - copy_number.py bed [--catalog TSV] [--locus L ...] - print the segments of the loci (all bedcov loci by default) as a BED - (chrom, 0-based start, end, name), sorted by position within a locus - copy_number.py call [--catalog TSV] --locus L --bedcov T [--bedcov-mapq1 T1] - [--json OUT.json] [--table OUT.tsv] - read `samtools bedcov` over those segments (chrom, start, end, name, - summed depth; the row's first bedcov pass) and, for a row with a MAPQ 0 - guard, the MAPQ >= 1 pass; write the locus's JSON block and, with - --table, the step's metricvalue table (bin/hba_copy_number.py and - bin/lpa_kiv2.py lay those out). Exit 0 whatever the status; 2 on - unreadable input; 3 (BROKEN_CONTROL) when a control segment has no - read while the other controls have the depth the row needs: the row's - coordinates are wrong, or the sample lacks that stretch, and no number - is given. - - copy_number.py bed --kind interval - the segments of every interval row (kind interval, one bedcov pass: the - recurrent CNV regions, the carrier deletion alleles and AZF) in one BED - copy_number.py intervals [--catalog TSV] --bedcov T --sex male|female|unknown - --out OUT.tsv - copy_number.py intervals --failed --out OUT.tsv - call every interval row from one `samtools bedcov` over that BED; write - one line per row (locus, name, chrom, start, end, status, copies, - expected, change, estimate, reason). A row on chrX or chrY is read - against the sample's own count there: 1 on a male, 2 on a female's X; - a chrY row on a female is 'not applicable', and without a sex both are - 'not assessed' (sex_unknown). A row whose control has no read is 'not - assessed' (broken_control) and the others still run. --failed writes - every row as 'not assessed' (check_failed), for a step whose depth - pass failed. - - copy_number.py bed --psv --locus L - the paralogous sequence variant (PSV) sites of a psv row (call psv:D): - each site's position on the gene and on its pseudogene, one base each - (name and name_pseudo), for `samtools mpileup -l` - copy_number.py call --locus L --bedcov T --pileup P ... - a psv row (CYP21A2) also reads `samtools mpileup` without a reference - over those sites (P): the module total from depth as above, then at - each site the share of reads with the gene's base, pooled over the gene - and pseudogene positions, times that total gives the copies that carry - the gene's sequence there. The table names a deletion (one module and - one gene copy fewer), a possible conversion (fewer gene copies at some - sites than at the others, or fewer than two) or none; it never says - which chromosome carries the change. - - copy_number.py check-bam --header H - refuse a BAM aligned to a reference with ALT or HLA contigs. H is - `samtools view -H` of the BAM, or its contig names one per line ('-' - reads stdin). On such a BAM a read that fits a primary locus and its - ALT copy equally gets MAPQ 0 or lands on the ALT copy, so the depth of - a duplicated gene drops and a depth-based count reads a deletion that - is not there. Exit 0 and no output without such a contig; exit 4 with - one REFUSED line on stderr with one. `call --bam-header H` does the - same check first. - -Two empty tables mean the BAM has no reads on that contig at all (bedcov is -skipped there): status 'not assessed'. Standard library only: it runs in the -Python and Parascopy images and under tests/test_copy_number.py. +Standard library only. """ -import argparse import csv -import importlib -import json import math import os -import sys CATALOG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cn_loci.tsv") COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", @@ -86,22 +32,9 @@ # scripts/ci/build-cn-intervals.py normalises it, and a row with any other # spelling is refused here). CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} -SEXES = ("male", "female", "unknown") -INTERVAL_COLUMNS = ["locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", - "estimate", "reason"] INPUTS = ("bedcov", "cnvpytor-level") Z95 = 1.96 -# The step table of a locus, by the module that lays it out (same folder). -TABLES = {"HBA": "hba_copy_number", "LPA_KIV2": "lpa_kiv2"} -# A psv row: the reads a site needs (gene and pseudogene bases pooled) before -# its copies are read, and the findings a reading must exist for. -PSV_MIN_READS = 30 -# A site reads as converted when its copies estimate lies this far under the -# others' count (or under the usual count) or more. At 30x, about 70 reads at -# a site, read sampling alone gives an estimate of 2 copies a standard -# deviation of about 0.25; HG002's six sites read 1.62 to 2.82 (35x), so the -# real spread is wider and 0.6 is a screen setting, not a measured error rate. -PSV_MARGIN = 0.6 +# The findings a psv row needs a reading for. PSV_FINDINGS = ("none", "deletion", "conversion", "pseudogene_deletion") BASES = "ACGT" @@ -110,43 +43,6 @@ class CatalogError(ValueError): pass -class AltContigs(ValueError): - """The BAM was aligned to a reference with ALT or HLA contigs.""" - - -def alt_contigs(lines): - """The ALT or HLA contigs of `samtools view -H` lines or of contig names - one per line (the rule of scripts/validate-setup.sh: a name ending in - _alt, or starting with HLA-).""" - out = [] - for line in lines: - line = line.rstrip("\n") - if line.startswith("@"): - if not line.startswith("@SQ"): - continue - name = next((f[3:] for f in line.split("\t") if f.startswith("SN:")), "") - else: - name = line.split("\t")[0].strip() - if name.endswith("_alt") or name.startswith("HLA-"): - out.append(name) - return out - - -def check_bam(path): - """Raise AltContigs, with the refusal line, when the header at path - (or '-' for stdin) names an ALT or HLA contig.""" - if path == "-": - found = alt_contigs(sys.stdin) - else: - with open(path, errors="replace") as f: - found = alt_contigs(f) - if found: - raise AltContigs(f"REFUSED: the BAM was aligned to a reference with {len(found)} ALT or HLA contigs " - f"(first: {found[0]}). Reads that fit a primary locus and its ALT copy equally get MAPQ 0 " - "or land on the ALT copy, so depth-based copy number reads deletions that are not there. " - "Realign to the default no-ALT reference (docs/realignment.md).") - - class BrokenControl(ValueError): """A control segment with no read while the other controls are covered.""" @@ -303,29 +199,6 @@ def regions(row): return sorted(((row["chrom"], s, e, n) for n, s, e, _ in segs), key=lambda r: (r[1], r[2])) -def read_bedcov(path, row): - """{name: mean depth} from `samtools bedcov` output over the row's - segments, which must all be there and nothing else; {} for an empty table.""" - means, seen = {}, set() - with open(path, errors="replace") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5: - raise ValueError(f"{path}: expected chrom, start, end, name, depth sum; got {line.strip()!r}") - chrom, start, end, name = c[0], int(c[1]), int(c[2]), c[3] - if end <= start: - raise ValueError(f"{path}: region {name} {chrom}:{start}-{end} is empty or reversed") - if (chrom, start, end, name) in seen: - raise ValueError(f"{path}: region {name} {chrom}:{start}-{end} appears twice") - seen.add((chrom, start, end, name)) - means[name] = float(c[4]) / (end - start) - if seen and seen != set(regions(row)): - raise ValueError(f"{path}: its regions differ from the ones `copy_number.py bed --locus {row['locus']}` prints") - return means - - def _combine(mode, segs, m): if mode == "sum": return sum(m[n] for n, *_ in segs) @@ -445,7 +318,7 @@ def assess(row, m, q1=None, sex=None): b["ci95"] = (max(0.0, total * math.exp(-Z95 * rel) - row["_offset"]), total * math.exp(Z95 * rel) - row["_offset"]) if mode == "psv": - # the module total (gene plus pseudogene copies); psv_assess reads the sites + # the module total (gene plus pseudogene copies); og cn reads the sites n = round(est) if abs(est - n) > vals[0]: b["reason"] = "between_counts" @@ -467,387 +340,3 @@ def assess(row, m, q1=None, sex=None): b["reading"] = reading(row, b["class"]) b["status"] = "called" return b - - -def count_bases(bases, ref="N"): - """{A, C, G, T: reads} of one `samtools mpileup` bases column: read - starts (^ and its MAPQ character), read ends, indels (+N and -N with - their N bases), deletions and skips are not bases; '.' and ',' are the - reference base when there is one.""" - n = dict.fromkeys(BASES, 0) - i = 0 - while i < len(bases): - ch = bases[i] - if ch == "^": - i += 2 - continue - if ch in "+-": - j = i + 1 - while j < len(bases) and bases[j].isdigit(): - j += 1 - i = j + int(bases[i + 1:j] or 0) - continue - b = ref.upper() if ch in ".," else ch.upper() - if b in n: - n[b] += 1 - i += 1 - return n - - -def read_pileup(path, row): - """{site: (gene-base reads, pseudogene-base reads, other reads)} of a psv - row from `samtools mpileup` over `bed --psv`, each site's two positions - pooled: a read that carries the gene's base counts for the gene wherever - the aligner put it. A position missing from the pileup has no read.""" - pos = {} - for name, g, ps, gb, pb, _ in row["_psv"]: - pos[g + 1] = pos[ps + 1] = name - counts = {name: dict.fromkeys(BASES, 0) for name, *_ in row["_psv"]} - with open(path, errors="replace") as f: - for line in f: - if not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5 or not c[1].isdigit(): - raise ValueError(f"{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); " - f"got {line.strip()[:80]!r}") - if c[0] != row["chrom"] or int(c[1]) not in pos: - continue - for k, v in count_bases(c[4], c[2]).items(): - counts[pos[int(c[1])]][k] += v - out = {} - for name, g, ps, gb, pb, _ in row["_psv"]: - n = counts[name] - gene, pseudo = sum(n[x] for x in gb), sum(n[x] for x in pb) - out[name] = (gene, pseudo, sum(n.values()) - gene - pseudo) - return out - - -def psv_assess(row, b, counts): - """A psv row's block (from assess) with its sites read. With the module - total called, each site of PSV_MIN_READS reads or more gets the copies - carrying the gene's base there (total x gene reads / gene and pseudogene - reads) and its ratio to the median of the sites. A site is low when its - estimate is more than PSV_MARGIN under the larger of the median's count - and the usual. cyp21a2 copies is the least count over the low sites, or - the median's count when none is low. finding: deletion when a site is low - and the module total under the usual (the 30 kb chimeric deletion leaves - one module and one gene copy fewer); conversion at the low sites when the - module total is kept; pseudogene_deletion when the module total is under - the usual with no low site; else none. Which chromosome carries a change - is never said.""" - sites = {} - for name, *_ in row["_psv"]: - g, p, o = counts.get(name, (0, 0, 0)) - sites[name] = {"gene_reads": g, "pseudo_reads": p, "other_reads": o, "copies": None, - "estimate": None, "ratio": None} - b.update(sites=sites, finding=None, conversion_sites=[]) - if b["status"] != "called" or b.get("module_total") is None: - b["status"] = "not assessed" - return b - usual = b["expected"] // len(row["_targets"]) - read = [] - for name, s in sites.items(): - n = s["gene_reads"] + s["pseudo_reads"] - if n >= PSV_MIN_READS: - s["estimate"] = b["total"] * s["gene_reads"] / n - s["copies"] = round(s["estimate"]) - read.append(s) - if not read: - b.update(status="not assessed", reason="psv_low_depth", copies=None) - return b - est = sorted(s["estimate"] for s in read) - mid = len(est) // 2 - median = est[mid] if len(est) % 2 else (est[mid - 1] + est[mid]) / 2 - for s in read: - s["ratio"] = s["estimate"] / median if median > 0 else None - floor = max(round(median), usual) - conv = [name for name, s in sites.items() if s["estimate"] is not None and s["estimate"] < floor - PSV_MARGIN] - # Only a site past the margin lowers the count, so one noisy site never - # gives fewer copies than the finding says. - b["copies"] = min(sites[n]["copies"] for n in conv) if conv else round(median) - if conv and b["module_total"] < b["expected"]: - b["finding"] = "deletion" - elif conv: - b.update(finding="conversion", conversion_sites=conv) - elif b["module_total"] < b["expected"]: - b["finding"] = "pseudogene_deletion" - else: - b["finding"] = "none" - text = {name: t for name, *_, t in row["_psv"]} - b["reading"] = reading(row, b["finding"]).replace( - "{sites}", ", ".join(f"{n} ({text[n]})" for n in conv)) - return b - - -def psv_message(row, b): - """'.' for a called psv row, else 'not assessed: ' and why. A low MAPQ 0 - share is spelled out: the reports print this message, and the reason - code alone would not say that the BAM, not the genome, stopped the call.""" - if b["status"] == "called": - return "." - if b["reason"] == "low_mapq0_share": - targets = " and ".join(n.removesuffix("_BODY") for n, *_ in row["_targets"]) - return (f"not assessed: the BAM has few or no multi-mapping reads here (reads of mapping quality 0 make " - f"up {100 * b['mapq0_share']:.1f}% of the depth over {targets}, under " - f"{100 * row['_mapq0_min']:g}%), a possible sign of an aligner that leaves out the reads that fit both " - "copies (as novoalign does by default); the module total may then come out low and read as a " - "deletion; realign with minimap2 (step 02) or BWA-MEM") - return f"not assessed: {b['reason'] or '.'}" - - -def psv_table(row, b): - """([(metric, value)], one line) of a psv row's step table.""" - f = (lambda x: "." if x is None else f"{x:.2f}") - d = (lambda x: "." if x is None else str(x)) - rows = [("status", b["status"]), ("cyp21a2_copies", d(b["copies"])), ("module_total", d(b.get("module_total"))), - ("expected_total", d(b["expected"])), ("estimate", f(b["estimate"])), ("finding", b.get("finding") or "."), - ("conversion_sites", ",".join(b.get("conversion_sites") or []) or ".")] - for name, s in (b.get("sites") or {}).items(): - rows.append((f"psv_{name}", f"{s['gene_reads']} gene, {s['pseudo_reads']} pseudogene, {s['other_reads']} other" - f" reads; copies {d(s['copies'])} ({f(s['estimate'])}); ratio {f(s['ratio'])}")) - rows += [("target_depth", f(b["target_depth"])), ("control_depth", f(b["control_depth"])), - ("reason", b["reason"] or "."), ("reading", b["reading"] or "."), - ("mapq0_share", "." if b.get("mapq0_share") is None else f"{b['mapq0_share']:.3f}"), - ("message", psv_message(row, b))] - line = (f"{row['locus']}: {b['status']} ({b['reason'] or b.get('finding')}; {d(b['copies'])} gene copies, " - f"module total {d(b.get('module_total'))} of {d(b['expected'])})") - return rows, line - - -def interval_rows(cat): - """The rows `intervals` calls: kind interval with one bedcov pass (LCE3BC, - an interval row with a MAPQ >= 1 pass, stays a gene-dosage row).""" - return [r for r in cat.values() if r["kind"] == "interval" and r["input"] == "bedcov" and r["_passes"] == ["."]] - - -def _bed(args): - cat = load_catalog(args.catalog) - if args.psv: - if not args.locus or args.kind: - raise CatalogError("bed --psv needs --locus (and no --kind)") - for loc in args.locus: - if loc not in cat or not cat[loc]["_psv"]: - raise CatalogError(f"no psv locus {loc!r} in the catalog") - r = cat[loc] - pos = [(s, f"{n}") for n, s, *_ in r["_psv"]] + [(p, f"{n}_pseudo") for n, _, p, *_ in r["_psv"]] - for s, n in sorted(pos): - print(f"{r['chrom']}\t{s}\t{s + 1}\t{n}") - return 0 - if args.kind == "interval": - if args.locus: - raise CatalogError("bed: --kind interval or --locus, not both") - loci = [r["locus"] for r in interval_rows(cat)] - else: - loci = args.locus or [k for k, r in cat.items() if r["input"] == "bedcov"] - for loc in loci: - if loc not in cat or cat[loc]["input"] != "bedcov": - raise CatalogError(f"no bedcov locus {loc!r} in the catalog") - for r in regions(cat[loc]): - print("\t".join(map(str, r))) - return 0 - - -def _call_cmd(args): - try: - if args.bam_header: - check_bam(args.bam_header) - except AltContigs as e: - print(e, file=sys.stderr) - return 4 - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - cat = load_catalog(args.catalog) - if args.locus not in cat or cat[args.locus]["input"] != "bedcov": - raise CatalogError(f"no bedcov locus {args.locus!r} in the catalog") - row = cat[args.locus] - m = read_bedcov(args.bedcov, row) - guarded = row["_mapq0_min"] is not None or row["_mapq0_max"] is not None - if guarded and not args.bedcov_mapq1: - raise ValueError(f"{args.locus} has a MAPQ 0 guard: --bedcov-mapq1 is needed") - q1 = read_bedcov(args.bedcov_mapq1, row) if args.bedcov_mapq1 else None - if q1 is not None and bool(m) != bool(q1): - raise ValueError(f"{args.bedcov_mapq1 if m else args.bedcov} is empty but the other table is not: " - "both must cover the regions, or both be empty (no reads on that contig)") - block = assess(row, m, q1) - if row["_psv"]: - if not args.pileup: - raise ValueError(f"{args.locus} is a psv row: --pileup (samtools mpileup over `bed --psv`) is needed") - block = psv_assess(row, block, read_pileup(args.pileup, row) if m else {}) - except BrokenControl as e: - print(f"ERROR: {e}", file=sys.stderr) - return 3 - except (OSError, ValueError, KeyError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - rows, line = table(row, block) - try: - if args.json: - with open(args.json, "w") as f: - json.dump(block, f, indent=1, sort_keys=True) - f.write("\n") - if args.table: - with open(args.table, "w") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - print(line) - return 0 - - -def read_bedcov_all(path): - """[(chrom, start, end, name, depth sum)] of a `samtools bedcov` table.""" - out = [] - with open(path, errors="replace") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5: - raise ValueError(f"{path}: expected chrom, start, end, name, depth sum; got {line.strip()!r}") - out.append((c[0], int(c[1]), int(c[2]), c[3], float(c[4]))) - return out - - -def interval_line(row, block): - """One line of the interval table for a row and its block.""" - tg = row["_targets"] - f = (lambda x: "." if x is None else f"{x:.2f}") - copies, exp = block["copies"], block["expected"] - change = "." if copies is None or exp is None else \ - "deletion" if copies < exp else "duplication" if copies > exp else "no change" - return [row["locus"], tg[0][3] or row["locus"], row["chrom"], str(min(s for _, s, _, _ in tg)), - str(max(e for _, _, e, _ in tg)), block["status"], "." if copies is None else str(copies), - "." if exp is None else str(exp), change, f(block["estimate"]), block["reason"] or "."] - - -def call_intervals(cat, rows_in, sex): - """[line] of every interval row from one bedcov table (rows_in, as - read_bedcov_all gives it); a broken control makes that row 'not - assessed' (broken_control), never the whole table.""" - by_name = {} - for c, s, e, n, d in rows_in: - if n in by_name: - raise ValueError(f"segment {n} appears twice in the bedcov table") - by_name[n] = (c, s, e, d) - out = [] - for row in interval_rows(cat): - want = regions(row) - have = [w for w in want if w[3] in by_name] - if len(have) != len(want): - raise ValueError(f"the bedcov table lacks some of {row['locus']}'s segments: " - f"its regions differ from `copy_number.py bed --kind interval`") - m = {} - for c, s, e, n in have: - if by_name[n][:3] != (c, s, e): - raise ValueError(f"segment {n} of {row['locus']} has other coordinates in the bedcov table") - m[n] = by_name[n][3] / (e - s) - try: - block = assess(row, m, None, sex) - except BrokenControl as e: - print(f"WARNING: {e}", file=sys.stderr) - block = assess(row, {}, None, sex) - if block["status"] == "not assessed": - block["reason"] = "broken_control" - out.append(interval_line(row, block)) - return out - - -def _intervals(args): - try: - cat = load_catalog(args.catalog) - if args.failed: - lines = [interval_line(r, {"status": "not assessed", "copies": None, "estimate": None, - "reason": "check_failed", "expected": None}) for r in interval_rows(cat)] - else: - if not args.bedcov or not args.sex: - raise ValueError("intervals needs --bedcov and --sex (or --failed)") - lines = call_intervals(cat, read_bedcov_all(args.bedcov), args.sex) - except (OSError, ValueError, KeyError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - with open(args.out + ".tmp", "w") as f: - f.write("\t".join(INTERVAL_COLUMNS) + "\n") - for ln in lines: - f.write("\t".join(ln) + "\n") - os.replace(args.out + ".tmp", args.out) - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - n = {} - for ln in lines: - n[ln[5]] = n.get(ln[5], 0) + 1 - print(f"interval rows: {len(lines)} ({', '.join(f'{v} {k}' for k, v in sorted(n.items()))})") - return 0 - - -def table(row, block): - """([(metric, value)], one summary line) of a locus's step table.""" - if row["_psv"]: - return psv_table(row, block) - mod = TABLES.get(row["locus"]) - if mod: - return importlib.import_module(mod).table(block) - f = (lambda x: "." if x is None else f"{x:.2f}") - rows = [("status", block["status"]), ("copies", "." if block["copies"] is None else block["copies"]), - ("estimate", f(block["estimate"])), ("target_depth", f(block["target_depth"])), - ("control_depth", f(block["control_depth"])), ("reason", block["reason"] or "."), - ("reading", block["reading"] or ".")] - return rows, f"{row['locus']}: {block['status']} ({block['reason'] or block['reading']})" - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="copy_number.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - b = sub.add_parser("bed", help="print the segments of the loci") - b.add_argument("--catalog") - b.add_argument("--locus", action="append") - b.add_argument("--kind", choices=["interval"], help="every interval row (as `intervals` calls them)") - b.add_argument("--psv", action="store_true", help="the PSV sites of a psv row (gene and pseudogene positions)") - c = sub.add_parser("call", help="the copy number of one locus from samtools bedcov over its segments") - c.add_argument("--catalog") - c.add_argument("--locus", required=True) - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", help="samtools bedcov with the row's MAPQ >= 1 pass") - c.add_argument("--json") - c.add_argument("--table") - c.add_argument("--bam-header", help="refuse a BAM with ALT or HLA contigs first (as check-bam)") - c.add_argument("--pileup", help="samtools mpileup over `bed --psv` (a psv row)") - iv = sub.add_parser("intervals", help="call every interval row from one samtools bedcov table") - iv.add_argument("--catalog") - iv.add_argument("--bedcov", help="samtools bedcov over `copy_number.py bed --kind interval`") - iv.add_argument("--sex", choices=SEXES) - iv.add_argument("--failed", action="store_true", help="every row not assessed (check_failed)") - iv.add_argument("--out", required=True) - k = sub.add_parser("check-bam", help="refuse a BAM aligned to a reference with ALT or HLA contigs") - k.add_argument("--header", required=True, help="samtools view -H of the BAM, or contig names; - for stdin") - args = ap.parse_args(argv) - if args.cmd == "check-bam": - try: - check_bam(args.header) - except AltContigs as e: - print(e, file=sys.stderr) - return 4 - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - if args.cmd == "intervals": - return _intervals(args) - if args.cmd == "bed": - try: - return _bed(args) - except (OSError, CatalogError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return _call_cmd(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/hba_copy_number.py b/bin/hba_copy_number.py deleted file mode 100755 index 302e35a50..000000000 --- a/bin/hba_copy_number.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -"""hba_copy_number.py: HBA1 and HBA2 copy number from short-read depth (step 35). - -Parascopy's pinned models have none for the alpha-globin genes, so step 35 -counts them from read depth instead. HBA1 and HBA2 are near-identical, so -short reads cannot be placed on one of them; but an aligner that reports -every read puts each read from either on one of the two (with mapping -quality 0 where it fits both equally), so the depth over both gene bodies -together, against -single-copy sequence on each side of the cluster, counts the alpha-globin -genes: four on a genome without a deletion (aa/aa), three with one gene lost -on one chromosome (-a3.7 or -a4.2), two with both lost on one chromosome -(--SEA and other large deletions) or one on each (-a/-a). - -The method and the regions are the HBA row of bin/cn_loci.tsv, run by -bin/copy_number.py (steps 35 and the PARASCOPY module call that engine); -this file lays out the step's table and keeps its own command line. - - hba_copy_number.py bed - print the regions (GRCh38, 0-based BED, name in column 4) to stdout - hba_copy_number.py check --bedcov BEDCOV.tsv --bedcov-mapq1 BEDCOV_Q1.tsv --out HBA.tsv - read `samtools bedcov` over those regions (chrom, start, end, name, - summed depth; all MAPQ, duplicates left out) and `samtools bedcov -Q 1` - over the same regions (reads of MAPQ 1 or more only), and write HBA.tsv - (metricvalue: status, alpha_globin_genes, estimate, the depths, - mapq0_share and the message). Exit 0 whatever the status; 2 on - unreadable input. - -status is 'called' or 'not assessed'. The estimate is -2 x (HBA2 depth + HBA1 depth) / flank depth. It is called only when it lies -within 0.3 of a whole number, the flanks reach MIN_FLANK_DEPTH and agree -with each other, and (unless the estimate rounds to 0) reads of MAPQ 0 make -up at least MIN_MAPQ0_SHARE of the depth over the two genes; otherwise the -message says why not. Standard library only: -it runs in the Parascopy image and under tests/test_hba_copy_number.py. -""" -import argparse -import sys - -import copy_number as cn - -LOCUS = "HBA" -_ROW = [] - - -def row(): - """The HBA row of the catalog, read once.""" - if not _ROW: - _ROW.append(cn.load_catalog()[LOCUS]) - return _ROW[0] - - -def __getattr__(name): - # The regions and thresholds come from the catalog row, read on first use. - r = row() - vals = {"REGIONS": cn.regions(r), "MIN_FLANK_DEPTH": r["_min_control"], "MAX_FLANK_RATIO": r["_max_ratio"], - "MAX_OFF": r["_call"][1][0], "MIN_MAPQ0_SHARE": r["_mapq0_min"]} - if name in vals: - return vals[name] - raise AttributeError(name) - - -def read_bedcov(path): - """{'flank_left': mean, 'HBA2': mean, ...} from `samtools bedcov` output - over the regions, which must all be there and nothing else.""" - m = cn.read_bedcov(path, row()) - if not m: - raise ValueError(f"{path}: its regions differ from the ones `hba_copy_number.py bed` prints (empty)") - return m - - -def table(b): - """([(metric, value)], summary line) of the engine's HBA block.""" - r, m = row(), b["depths"] - if b["reason"] == "no_data": - out = {k: "." for k in ORDER} - out.update(status="not assessed", message="not assessed: the BAM has no chr16 reads, so HBA1 and HBA2 have none") - return [(k, out[k]) for k in ORDER], f"HBA1/HBA2: {out['message']}" - left, right = m.get("flank_left"), m.get("flank_right") - share = b["mapq0_share"] - out = {"hba2_depth": round(m["HBA2"], 2), "hba1_depth": round(m["HBA1"], 2), - "flank_left_depth": round(left, 2), "flank_right_depth": round(right, 2), - "mapq0_share": f"{share:.3f}", - "estimate": ".", "alpha_globin_genes": ".", "status": "not assessed"} - reason, est = b["reason"], b["estimate"] - if est is not None: - out["estimate"] = f"{est:.2f}" - if reason == "low_control_depth": - out["message"] = (f"not assessed: flank depth under {r['_min_control']:g}x " - f"({left:.1f}x and {right:.1f}x), too low to compare") - elif reason == "controls_disagree": - out["message"] = (f"not assessed: the two flanks disagree ({left:.1f}x and {right:.1f}x); " - "a deletion larger than --SEA, or uneven coverage, would skew the count") - elif reason == "low_mapq0_share": - out["message"] = (f"not assessed: reads of mapping quality 0 make up {100 * share:.1f}% of the depth " - "over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as " - "novoalign does by default) and the count would come out low; realign with " - "minimap2 (step 02) or BWA-MEM") - elif reason == "between_counts": - out["message"] = (f"not assessed: the estimate {est:.2f} lies between {int(est)} and {int(est) + 1} " - "alpha-globin genes") - else: - out.update(status="called", alpha_globin_genes=str(b["copies"]), message=b["reading"]) - line = (f"HBA1/HBA2: {out['message']} (estimate {out['estimate']}; HBA2 {out['hba2_depth']}x, " - f"HBA1 {out['hba1_depth']}x, flanks {out['flank_left_depth']}x and {out['flank_right_depth']}x; " - f"MAPQ 0 share over the genes {out['mapq0_share']})") - return [(k, out[k]) for k in ORDER], line - - -def assess(m, q1): - """The result dict for the mean depths of read_bedcov: m over every - read, q1 over reads of MAPQ 1 or more.""" - return dict(table(cn.assess(row(), m, q1))[0]) - - -ORDER = ("status", "alpha_globin_genes", "estimate", "hba2_depth", "hba1_depth", - "flank_left_depth", "flank_right_depth", "mapq0_share", "message") - - -def check(args): - try: - rows, line = table(cn.assess(row(), read_bedcov(args.bedcov), read_bedcov(args.bedcov_mapq1))) - except cn.BrokenControl as e: - print(f"ERROR: {e}", file=sys.stderr) - return 3 - except (OSError, ValueError, KeyError, ZeroDivisionError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - try: - with open(args.out, "w") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - except OSError as e: - print(f"ERROR: cannot write {args.out}: {e}", file=sys.stderr) - return 2 - print(line) - return 0 - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="hba_copy_number.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - sub.add_parser("bed", help="print the regions") - c = sub.add_parser("check", help="judge samtools bedcov over the regions") - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", required=True, help="samtools bedcov -Q 1 over the same regions") - c.add_argument("--out", required=True) - args = ap.parse_args(argv) - if args.cmd == "bed": - for r in cn.regions(row()): - print("\t".join(map(str, r))) - return 0 - return check(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/lpa_kiv2.py b/bin/lpa_kiv2.py deleted file mode 100755 index ff8183e63..000000000 --- a/bin/lpa_kiv2.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 -"""lpa_kiv2.py: the LPA KIV-2 repeat count from short-read depth (step 46). - -Lp(a) is a mostly inherited cardiovascular risk factor, and the number of -kringle IV type 2 (KIV-2) repeats in LPA sets the size of apo(a): fewer -repeats, a smaller isoform and, on average, a higher Lp(a) level. The KIV-2 -repeat is a 5.5 kb unit with two exons; a Danish population study counted 6 -to 99 on both chromosomes together (Kamstrup et al. 2009, PMID 19509380). -GRCh38 holds six units, and their exons are near-identical, so an aligner -that reports every read places each KIV-2 read on one of the six (with -mapping quality 0). The depth over those exons, against the depth over -single-copy LPA exons, counts the units the sample carries: the approach of -the published short-read estimators (Zekavat et al. 2018, PMID 29973585; -Behera et al. 2024, PMID 39449055, which also gives the GRCh38 region and the -six reference units). - -The regions and the method are the LPA_KIV2 row of bin/cn_loci.tsv, run by -bin/copy_number.py (step 46 and the LPA processes call that engine); this -file lays out the step's table and keeps its own command line. - - lpa_kiv2.py bed - print the regions (GRCh38, 0-based BED, name in column 4) to stdout - lpa_kiv2.py estimate --bedcov BEDCOV.tsv --bedcov-mapq1 BEDCOV_Q1.tsv --out LPA.tsv - read `samtools bedcov -j` over those regions (chrom, start, end, name, - summed depth; all MAPQ, duplicates left out) and `samtools bedcov -j -Q 1` - over the same regions, and write LPA.tsv (metricvalue). Exit 0 - whatever the status; 2 on unreadable input. Two empty tables mean the - BAM has no chr6: status 'not assessed'. - lpa_kiv2.py null-allele --pileup PILEUP.tsv --table LPA.tsv --out NULL.tsv - read `samtools mpileup` (no reference, every MAPQ) at the seven copies - of the KIV-2 4925G>A site (`traits_report.py --marker-regions - LPA_KIV2_4925`) and write its reading to NULL.tsv (metricvalue), - with the count N of LPA.tsv (written by `estimate` or `copy_number.py - call --locus LPA_KIV2`, left as it is). Exit 0 whatever the reading (an - unreadable pileup reads 'not assessed'); 2 when NULL.tsv cannot be - written. - -The regions are exons of the MANE Select transcript ENST00000316300.10, -numbered from its first exon: - kiv2_e03 to kiv2_e16 the six KIV-2 units of GRCh38 (exons 4 to 15), plus - exon 3 (the last exon of KIV-1) and exon 16 (the first - of KIV-3), which are near-identical to KIV-2 exons and - take some of its reads. Seven 160 bp and seven 182 bp - exons, 2,394 bp: six KIV-2 units plus one single-copy - pair per chromosome. - flank_e19 to flank_e33 single-copy kringle exons downstream (the second - exon of KIV-4 to KV), under 90% identical to any KIV-2 - exon. Exons 17 and 18 (about 95% identical) are in - neither group, nor are exons 1 and 2 or the protease - exons, which resemble plasminogen (PLG). - -With D the mean depth over the flank exons (two copies) and M the mean depth -over the KIV-2 block, the block holds N + 2 copies of a 342 bp pair spread -over seven reference pairs, so M = (N + 2) x D / 14 and - N = 14 x M / D - 2 (the total over both chromosomes) -The 95% interval comes from the spread of the per-exon depths in each group: -relative standard error sqrt(cv_kiv2^2 / 14 + cv_flank^2 / 15), applied to -N + 2 on the log scale. It covers sampling noise only, not a systematic bias -of an aligner or a library. - -status is 'estimated', 'too little depth' or 'not assessed'; the message says -why. - -The KIV-2 4925G>A splice variant (Coassin et al. 2017, PMID 28444229) sits at -the donor splice site of the second KIV-2 exon: carriers had a much lower -Lp(a) than non-carriers with the same isoform size, most of all with small -isoforms, and it is usually in one or two of a person's KIV-2 repeats. GRCh38 -holds the site in each of its six units and in KIV-1, so null-allele adds up -the reads at all seven (bin/traits_report.py MARKERS) and reads the share -with the variant as a fraction of copies: status is 'present', 'not seen', -'uncertain', 'too little depth' or 'not assessed', and with a count, copies -is the fraction x (N + 2), the copies that carry it. -A short-read fraction cannot say which chromosome or which repeat carries -it, so it cannot say whether it sits on the small or the large isoform. Research estimate, not a diagnostic test: the actionable test is a blood -Lp(a) measurement. Standard library only: it runs in the Python image and -under tests/test_lpa_kiv2.py. -""" -import argparse -import os -import sys - -import copy_number as cn - -LOCUS = "LPA_KIV2" -_ROW = [] -ADVICE = ("A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 " - "European Atherosclerosis Society consensus advises measuring it at least once in every adult.") - - -def row(): - """The LPA_KIV2 row of the catalog, read once.""" - if not _ROW: - _ROW.append(cn.load_catalog()[LOCUS]) - return _ROW[0] - - -def __getattr__(name): - # The regions and thresholds come from the catalog row, read on first use. - r = row() - if name == "REGIONS": # in catalog order: the KIV-2 block, then the single-copy exons - return [(r["chrom"], s, e, n) for n, s, e, _ in r["_targets"] + r["_controls"]] - vals = {"MIN_FLANK_DEPTH": r["_min_control"], "MIN_MAPQ0_SHARE": r["_mapq0_min"], - "LOW": r["_call"][1][0], "HIGH": r["_call"][1][1], "READINGS": r["_readings"]} - if name in vals: - return vals[name] - raise AttributeError(name) - - -def bed_lines(): - return ["\t".join(map(str, r)) for r in cn.regions(row())] - - -def read_bedcov(path): - """{name: mean depth} from `samtools bedcov` output over the regions, - which must all be there and nothing else; {} for an empty table (no chr6).""" - return cn.read_bedcov(path, row()) - - -def table(b): - """([(metric, value)], summary line) of the engine's LPA_KIV2 block.""" - r = row() - out = {"status": "not assessed", "kiv2_copies": ".", "ci95_low": ".", "ci95_high": ".", "class": ".", - "kiv2_depth": ".", "flank_depth": ".", "mapq0_share": ".", "message": "", "advice": ADVICE} - reason = b["reason"] - if reason == "no_data": - out["message"] = "not assessed: the BAM has no chr6, so LPA has no reads" - else: - kdepth, fdepth, share = b["target_depth"], b["control_depth"], b["mapq0_share"] - out.update(kiv2_depth=f"{kdepth:.2f}", flank_depth=f"{fdepth:.2f}", mapq0_share=f"{share:.3f}") - if reason == "low_control_depth": - out["status"] = "too little depth" - out["message"] = (f"too little depth: {fdepth:.1f}x over the single-copy LPA exons, under the " - f"{r['_min_control']:g}x the count needs") - elif reason == "low_mapq0_share": - out["message"] = (f"not assessed: reads of mapping quality 0 make up {100 * share:.1f}% of the depth " - "over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 " - "copies (as novoalign does by default) and the count would come out low; realign " - "with minimap2 (step 02) or BWA-MEM") - elif reason == "below_floor": - out["message"] = (f"not assessed: the KIV-2 exons have {kdepth:.1f}x against {fdepth:.1f}x over the " - "single-copy exons, too little for any repeat count (a deletion or a mapping problem)") - else: - est, (lo, hi) = b["estimate"], b["ci95"] - out.update(status="estimated", kiv2_copies=f"{est:.1f}", ci95_low=f"{lo:.1f}", ci95_high=f"{hi:.1f}", - **{"class": b["class"]}, - message=(f"about {est:.0f} KIV-2 repeats in total (95% interval {lo:.1f} to {hi:.1f}): " - f"{b['reading']}")) - line = (f"LPA KIV-2: {out['message']} (KIV-2 exons {out['kiv2_depth']}x, single-copy exons " - f"{out['flank_depth']}x, MAPQ 0 share {out['mapq0_share']})") - return [(k, out[k]) for k in ORDER], line - - -def assess(m, q1): - """The result dict for the mean depths of read_bedcov: m over every read, - q1 over reads of MAPQ 1 or more.""" - out = dict(table(cn.assess(row(), m, q1))[0]) - del out["advice"] - return out - - -ORDER = ("status", "kiv2_copies", "ci95_low", "ci95_high", "class", "kiv2_depth", "flank_depth", - "mapq0_share", "message", "advice") - - -NULL_LIMIT = ("A short-read fraction cannot say which chromosome or which repeat carries it, so it cannot say " - "whether it sits on the small or the large isoform.") - - -def null_allele(counts, kiv2_copies=".", why_not=None): - """[(metric, value)] of the KIV-2 4925G>A table from - traits_report.read_marker_pileup's counts (None: unreadable), with the - count N of step 46's table ('.' when there is none). why_not: the reason - the reads cannot be trusted (the aligner left out the reads of MAPQ 0), - which makes it 'not assessed' whatever the counts.""" - import traits_report as tr # bin/: the copy-aware marker catalog - m = tr.MARKERS["LPA_KIV2_4925"] - a = tr.assess_marker("LPA_KIV2_4925", None if why_not else counts) - st = a["status"] - try: - total = float(kiv2_copies) + 2 - except ValueError: - total = None - copies = "." - if st in ("present", "uncertain") and total is not None: - copies = f"{float(a['fraction']) * total:.1f}" - name = f"{m['name']} (a splice variant that lowers the Lp(a) made from its chromosome, Coassin 2017)" - if st == "not assessed": - msg = f"{name}: not assessed, " + (why_not or "the reads at its seven GRCh38 copies could not be read.") - else: - msg = (f"{name}: {st}, {a['alt_reads']} of {a['depth']} reads at its seven GRCh38 copies carry it" - + (f" (about {copies} of the {total:.0f} copies counted)" if copies != "." else "") - + (f", under the {m['min_depth']} reads a reading needs" if st == "too little depth" else "") - + ".") - if st == "present": - msg += (" Carriers had a lower Lp(a) than non-carriers with the same number of repeats, most of " - "all with few repeats. " + NULL_LIMIT) - elif st == "not seen": - msg += (" Not seen in the reads is not proof of absence: one carrying repeat among many gives few " - "reads, and a short-read fraction never says which chromosome or repeat holds a variant.") - else: - msg += " " + NULL_LIMIT - return [("marker", m["name"]), ("rsid", m["dbsnp"][0] + " (first KIV-2 copy)"), ("status", st), - ("alt_reads", a["alt_reads"]), ("depth", a["depth"]), ("fraction", a["fraction"]), ("copies", copies), - ("kiv2_copies", kiv2_copies), ("message", msg)] - - -def write_null_allele(table_path, pileup_path, out_path): - """Write the KIV-2 4925G>A table beside step 46's table (read for the - count N; a table that is missing or unreadable gives no N). Returns the message.""" - import traits_report as tr - t = {} - try: - with open(table_path, errors="replace") as f: - t = dict(ln.rstrip("\n").split("\t", 1) for ln in f if "\t" in ln) - except OSError: - pass - n = t.get("kiv2_copies", ".") - why_not = None - try: - share = float(t.get("mapq0_share", ".")) - except ValueError: - share = None - if share is not None and share < row()["_mapq0_min"]: - # The reads that fit several copies are the ones a carrying repeat - # gives; without them the share at the seven copies is biased. - why_not = (f"reads of mapping quality 0 make up {100 * share:.1f}% of the KIV-2 depth, so the aligner " - "left out the reads that fit several copies, the ones a carrying repeat gives.") - try: - counts = tr.read_marker_pileup(pileup_path, "LPA_KIV2_4925") - except (OSError, ValueError) as e: - print(f"WARNING: {e}; KIV-2 4925G>A is not assessed", file=sys.stderr) - counts = None - rows = null_allele(counts, n, why_not) - tr.write_metric_table(out_path, rows) - return dict(rows)["message"] - - -def main(argv=None): - ap = argparse.ArgumentParser(prog="lpa_kiv2.py", description=__doc__.split("\n", 1)[0]) - sub = ap.add_subparsers(dest="cmd", required=True) - sub.add_parser("bed", help="print the regions") - c = sub.add_parser("estimate", help="count the KIV-2 repeats from samtools bedcov over the regions") - c.add_argument("--bedcov", required=True) - c.add_argument("--bedcov-mapq1", required=True, help="samtools bedcov -j -Q 1 over the same regions") - c.add_argument("--out", required=True) - n = sub.add_parser("null-allele", help="the KIV-2 4925G>A table beside the count") - n.add_argument("--pileup", required=True, help="samtools mpileup at traits_report.py --marker-regions LPA_KIV2_4925") - n.add_argument("--table", required=True, help="the table estimate wrote (for the count)") - n.add_argument("--out", required=True, help="the 4925G>A table") - args = ap.parse_args(argv) - if args.cmd == "null-allele": - try: - print("LPA " + write_null_allele(args.table, args.pileup, args.out)) - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - if args.cmd == "bed": - print("\n".join(bed_lines())) - return 0 - return cn.main(["call", "--locus", LOCUS, "--bedcov", args.bedcov, "--bedcov-mapq1", args.bedcov_mapq1, - "--table", args.out]) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/traits_report.py b/bin/traits_report.py index cd4fbac98..6b984f1cb 100755 --- a/bin/traits_report.py +++ b/bin/traits_report.py @@ -40,27 +40,15 @@ Copy-aware markers (MARKERS): two well-sourced markers of the rule catalog's list sit in sequence the genome holds more than once, so a gVCF genotype at one position cannot read them and they are not rule lines. Each is read from -the BAM instead, by the step that counts its copies: the reads at every +the BAM instead, by the step that counts its copies (og cn marker-regions, +marker-read and call --marker, in rust/crates/og-cn): the reads at every GRCh38 copy of the site are added up, and the share that carries the marker's -base is a fraction of copies, not a diploid genotype. +base is a fraction of copies, not a diploid genotype. MARKERS stays here for +the reports (bin/collect_summary.py reads the marker tables with it). LPA_KIV2_4925 the LPA KIV-2 4925G>A splice variant (Coassin et al. 2017, PMID 28444229), read by step 46 beside the KIV-2 count - (laid out by bin/lpa_kiv2.py) SMN1_27134 SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID 23788250), read by step 35 beside the SMN1 psCN - - traits_report.py --marker-regions NAME - print the marker's copy sites as samtools regions (chrom:pos-pos) - traits_report.py --marker LPA_KIV2_4925 --pileup P --copy-table LPA.tsv --marker-out OUT.tsv - the 4925G>A table beside step 46's table LPA.tsv, whose count gives - the copies that carry it (laid out by bin/lpa_kiv2.py); exit 2 when - OUT.tsv cannot be written - traits_report.py --marker SMN1_27134 --pileup P [--copy-table SMN.tsv] --marker-out OUT.tsv - read `samtools mpileup` at those sites (no reference, every MAPQ) and - write OUT.tsv (metricvalue). SMN.tsv is step 35's SMN table: the - SMN1 psCN at exon 7 and the agCN at the marker site come from it. Exit - 0 whatever the status (an unreadable pileup is 'not assessed'); 2 when - OUT.tsv cannot be written. """ import argparse import itertools @@ -538,7 +526,8 @@ def report(rows, recs, rules=None, phased=None, lines=None): # Ensembl GRCh38 agree, 2026-11-07): the same 21 bases at each copy, which is # why the reads cannot be placed on one copy. tests/test_traits_report.py # checks every copy against the context and the record against the marker, and -# tests/e2e/genotype-rules-2-smn1-marker.sh compares the records with dbSNP. +# that og cn reads the same copies; tests/e2e/genotype-rules-2-smn1-marker.sh +# compares the records with dbSNP. # min_depth fewer reads over all copies together: too little depth # min_alt reads with the marker's base for 'present' ... # min_fraction ... and a share of the reads at least this @@ -551,7 +540,7 @@ def report(rows, recs, rules=None, phased=None, lines=None): # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, - # the block bin/lpa_kiv2.py counts as N + 2 copies (N KIV-2 repeats and + # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. "LPA_KIV2_4925": { "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", @@ -571,9 +560,6 @@ def report(rows, recs, rules=None, phased=None, lines=None): }, } MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") -# SMN1 exon 7 (c.840, chr5:70,951,946): where step 35's report reads the SMN1 psCN. -SMN1_EXON7 = ("chr5", 70951946) -SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." def marker_errors(name, m=None, dbsnp=None): @@ -602,159 +588,6 @@ def marker_errors(name, m=None, dbsnp=None): return errs -def marker_regions(name): - """The copy sites as samtools regions, chrom:pos-pos, in the marker's order.""" - m = MARKERS[name] - return [f"{m['chrom']}:{p}-{p}" for p in m["copies"]] - - -def read_marker_pileup(path, name): - """(ref reads, marker reads, other reads) over every copy of the marker, - from `samtools mpileup` lines (chrom, pos, ref, depth, bases, ...) at its - sites, without a reference: a read that carries the marker's base counts - wherever the aligner put it. Lines at other positions are left out; a - copy with no line has no read. Raises ValueError on a line that is not - an mpileup line.""" - import copy_number # bin/: its mpileup base counter, shared with the CYP21A2 sites - m = MARKERS[name] - want = set(m["copies"]) - n = dict.fromkeys("ACGT", 0) - with open(path, errors="replace") as f: - for line in f: - if not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 5 or not c[1].isdigit(): - raise ValueError(f"{path}: expected samtools mpileup lines (chrom, pos, ref, depth, bases, ...); " - f"got {line.strip()[:80]!r}") - if c[0] != m["chrom"] or int(c[1]) not in want: - continue - for k, v in copy_number.count_bases(c[4], c[2]).items(): - n[k] += v - ref, alt = n[m["ref"]], n[m["alt"]] - return ref, alt, sum(n.values()) - ref - alt - - -def assess_marker(name, counts): - """{status, alt_reads, depth, fraction} of a marker from read_marker_pileup's - counts (None: the pileup could not be read). depth counts the reads with - the ref or the marker's base.""" - m = MARKERS[name] - if counts is None: - return {"status": "not assessed", "alt_reads": ".", "depth": ".", "fraction": "."} - ref, alt, _ = counts - depth = ref + alt - out = {"alt_reads": str(alt), "depth": str(depth), "fraction": f"{alt / depth:.4f}" if depth else "."} - if depth < m["min_depth"]: - out["status"] = "too little depth" - elif alt >= m["min_alt"] and alt / depth >= m["min_fraction"]: - out["status"] = "present" - elif alt <= m["max_absent"]: - out["status"] = "not seen" - else: - out["status"] = "uncertain" - return out - - -def smn_copy_numbers(path): - """(SMN1 psCN at exon 7 or None, its text, agCN at the marker site or None) - from step 35's SMN table. A copy number counts only with filter PASS and - quality 20 or more, the threshold of step 35's reading.""" - def regions(r): - out = [(r["chrom"], int(r["start"]) + 1, int(r["end"]))] - for h in (r.get("homologous_regions") or "").split(","): - mm = re.match(r"^([^:]+):(\d+)-(\d+)", h.strip()) - if mm: - out.append((mm.group(1), int(mm.group(2)), int(mm.group(3)))) - return out - - def value(r, kind, idx): - vals = (r.get(kind) or "").split(",") - quals = (r.get(kind + "_qual") or "").split(",") - v = vals[idx] if idx < len(vals) else "." - q = quals[idx] if idx < len(quals) else "." - try: - ok = (r.get(kind + "_filter") == "PASS" and float(q) >= 20 and v.isdigit()) - except ValueError: - ok = False - return ((int(v) if ok else None), - f"{v} (quality {q}, filter {r.get(kind + '_filter') or '.'})" + ("" if ok else ", not resolved")) - - sm = MARKERS["SMN1_27134"] - ps, ps_text, ag = None, "not read (no SMN table from step 35)", None - if not path or not os.path.exists(path): - return ps, ps_text, ag - ps_text = "not read (no row of the SMN table covers SMN1 exon 7)" - with open(path, errors="replace") as f: - header = f.readline().rstrip("\n").split("\t") - for line in f: - r = dict(zip(header, line.rstrip("\n").split("\t"))) - try: - regs = regions(r) - except (KeyError, ValueError): - continue - for i, (c, s, e) in enumerate(regs): - if c == SMN1_EXON7[0] and s <= SMN1_EXON7[1] <= e: - ps, ps_text = value(r, "psCN", i) - # agCN is one value per row: the row whose region, or a - # homologous region, holds a copy of the marker site - if c == sm["chrom"] and any(s <= p <= e for p in sm["copies"]): - ag = value(r, "agCN", 0)[0] - return ps, ps_text, ag - - -def smn1_marker(counts, smn_table=None): - """[(metric, value)] of step 35's SMN1 marker table: the marker's reading - beside the SMN1 psCN. The message always says the marker does not prove - 2+0 (SMN1_NOT_PROOF).""" - m = MARKERS["SMN1_27134"] - a = assess_marker("SMN1_27134", counts) - try: - ps, ps_text, ag = smn_copy_numbers(smn_table) - except OSError as e: - print(f"WARNING: {e}; the SMN copy numbers are not read", file=sys.stderr) - ps, ps_text, ag = None, "not read (the SMN table could not be read)", None - copies = "." - if a["status"] in ("present", "uncertain") and ag is not None and a["fraction"] != ".": - copies = f"{float(a['fraction']) * ag:.1f}" - st = a["status"] - if st == "not assessed": - head = f"{m['name']} ({m['dbsnp'][0]}): not assessed, the reads at the marker could not be read." - else: - head = (f"{m['name']} ({m['dbsnp'][0]}): {st}, {a['alt_reads']} of {a['depth']} reads carry G at the " - "SMN1 and SMN2 copies of the site together" - + (f" (about {copies} of the {ag} SMN1 and SMN2 copies)" if copies != "." else "") - + (f", under the {m['min_depth']} reads a reading needs" if st == "too little depth" else "") - + ". The reads cannot tell whether a G sits on SMN1 or SMN2.") - if ps == 2: - body = {"present": " With two SMN1 copies, this marker goes with a higher chance that both sit on one " - "chromosome (2+0, a silent carrier): in Luo 2014 it raised the residual carrier risk of a " - "two-copy result (in African Americans from about 1 in 121 to 1 in 34); the figures " - "depend on ancestry.", - "not seen": " With two SMN1 copies and no marker, the chance of 2+0 is lower than for a two-copy " - "result alone (in Luo 2014, in African Americans, about 1 in 396 against 1 in 121), " - "not zero; the figures depend on ancestry."}.get( - st, " With two SMN1 copies the marker would refine the chance of 2+0; here it gives no reading.") - elif ps is not None: - body = (f" SMN1 has {ps} cop{'y' if ps == 1 else 'ies'} here; the marker refines the carrier risk of a " - "two-copy result only.") - else: - body = f" SMN1 copy number: {ps_text}; the marker refines the carrier risk of a two-copy result only." - return [("marker", m["name"]), ("rsid", m["dbsnp"][0]), ("status", st), ("alt_reads", a["alt_reads"]), - ("depth", a["depth"]), ("fraction", a["fraction"]), ("copies", copies), - ("smn1_pscn", ps_text), ("agcn", "." if ag is None else str(ag)), - ("message", head + body + " " + SMN1_NOT_PROOF)] - - -def write_metric_table(path, rows): - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("metric\tvalue\n") - for k, v in rows: - f.write(f"{k}\t{v}\n") - os.replace(tmp, path) - - def write_tsv(path, out, columns=OUT_COLUMNS): tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as f: @@ -775,41 +608,7 @@ def main(argv=None): ap.add_argument("--rules-out", help="the sample's genotype rules (TSV)") ap.add_argument("--phase-sites", metavar="VCF", help="write the sites WhatsHap must phase and exit") ap.add_argument("--check-table", metavar="TABLE", help="check the table's schema (and --rules') and exit") - ap.add_argument("--marker-regions", metavar="NAME", choices=sorted(MARKERS), - help="print a copy-aware marker's sites as samtools regions and exit") - ap.add_argument("--marker", choices=sorted(MARKERS), help="read a copy-aware marker from --pileup") - ap.add_argument("--pileup", help="samtools mpileup at the --marker-regions sites") - ap.add_argument("--copy-table", help="SMN1_27134: step 35's SMN table (the SMN1 psCN and the agCN); " - "LPA_KIV2_4925: step 46's table (the KIV-2 count)") - ap.add_argument("--marker-out", help="the marker's table (metricvalue)") a = ap.parse_args(argv) - if a.marker_regions: - print("\n".join(marker_regions(a.marker_regions))) - return 0 - if a.marker: - if not (a.pileup and a.marker_out): - ap.error("--marker needs --pileup and --marker-out") - if a.marker == "LPA_KIV2_4925": - import lpa_kiv2 # bin/: step 46's table layout - try: - print("LPA " + lpa_kiv2.write_null_allele(a.copy_table or "", a.pileup, a.marker_out)) - except OSError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - return 0 - try: - counts = read_marker_pileup(a.pileup, a.marker) - except (OSError, ValueError) as e: - print(f"WARNING: {e}; the marker is not assessed", file=sys.stderr) - counts = None - rows = smn1_marker(counts, a.copy_table) - try: - write_metric_table(a.marker_out, rows) - except OSError as e: - print(f"ERROR: cannot write: {e}", file=sys.stderr) - return 2 - print(dict(rows)["message"]) - return 0 try: if a.check_table: rows = read_table(a.check_table) diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index 05b3b46d1..640057f79 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -3,7 +3,7 @@ BLOODTYPE — Blood group antigens (red cell, platelet, neutrophil) predicted from the genome ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('bloodtype' in --tools). Three processes, as in scripts/41-bloodtype.sh: - BLOODTYPE_REGIONS bin/copy_number.py bed --locus RHD, once: RHD, its + BLOODTYPE_REGIONS og cn bed --locus RHD, once: RHD, its exons and flanks (the RHD row of bin/cn_loci.tsv) BLOODTYPE_DEPTH samtools bedcov -Q 20 -j over the sites of assets/bloodtype_markers.tsv and those regions, for each @@ -29,7 +29,7 @@ process BLOODTYPE_REGIONS { script: """ - copy_number.py bed --locus RHD > rhd_regions.bed + og cn bed --locus RHD > rhd_regions.bed cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/lpa/main.nf b/modules/local/lpa/main.nf index 97ff4ea1f..c4e061cc8 100644 --- a/modules/local/lpa/main.nf +++ b/modules/local/lpa/main.nf @@ -3,18 +3,17 @@ LPA — the LPA KIV-2 repeat count estimated from read depth (Lp(a)) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('lpa' in --tools). Three processes, as in scripts/46-lpa.sh: - LPA_REGIONS bin/copy_number.py bed --locus LPA_KIV2, once: the KIV-2 - and single-copy LPA exons (the LPA_KIV2 row of bin/cn_loci.tsv), + LPA_REGIONS og cn bed --locus LPA_KIV2, once: the KIV-2 and + single-copy LPA exons (the LPA_KIV2 row of bin/cn_loci.tsv), and the seven GRCh38 copies of the KIV-2 4925G>A site - (bin/traits_report.py --marker-regions LPA_KIV2_4925) + (og cn marker-regions LPA_KIV2_4925) LPA_DEPTH samtools bedcov -j over the exons, all MAPQ and MAPQ >= 1, and samtools mpileup at the 4925G>A sites, for each sample with a BAM; empty tables when the BAM has no chr6 - LPA bin/copy_number.py call --locus LPA_KIV2: the count, its - 95% interval and the message (laid out by bin/lpa_kiv2.py), - then bin/traits_report.py --marker LPA_KIV2_4925: the 4925G>A - reading beside it, _lpa_kiv2_4925.tsv ('not assessed' - when that check fails; the count stays) + LPA og cn call --locus LPA_KIV2 --marker LPA_KIV2_4925: the + count, its 95% interval and the message, and in the same + run the 4925G>A reading beside it, _lpa_kiv2_4925.tsv + ('not assessed' when that check fails; the count stays) Equivalent to: scripts/46-lpa.sh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -32,8 +31,8 @@ process LPA_REGIONS { script: """ - copy_number.py bed --locus LPA_KIV2 > lpa_regions.bed - traits_report.py --marker-regions LPA_KIV2_4925 > lpa_sites.txt + og cn bed --locus LPA_KIV2 > lpa_regions.bed + og cn marker-regions LPA_KIV2_4925 > lpa_sites.txt cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -123,11 +122,14 @@ process LPA { script: """ - copy_number.py call --locus LPA_KIV2 --bedcov ${depth} --bedcov-mapq1 ${depth_q1} --table ${meta.id}_lpa_kiv2.tsv - # KIV-2 4925G>A beside the count (as scripts/46-lpa.sh [4/4]): a failure - # never costs the count; the table then says not assessed. - if ! traits_report.py --marker LPA_KIV2_4925 --pileup ${pileup} --copy-table ${meta.id}_lpa_kiv2.tsv \\ - --marker-out ${meta.id}_lpa_kiv2_4925.tsv; then + # The count and KIV-2 4925G>A beside it in one run (as scripts/46-lpa.sh + # [3/3]): a failure of the marker never costs the count; the table then + # says not assessed. No count table: the task fails. + rc=0 + og cn call --locus LPA_KIV2 --bedcov ${depth} --bedcov-mapq1 ${depth_q1} --table ${meta.id}_lpa_kiv2.tsv \\ + --marker LPA_KIV2_4925 --marker-pileup ${pileup} --marker-out ${meta.id}_lpa_kiv2_4925.tsv || rc=\$? + if [ "\$rc" -ne 0 ]; then + [ -s ${meta.id}_lpa_kiv2.tsv ] || exit "\$rc" echo "WARNING: the KIV-2 4925G>A check failed; ${meta.id}_lpa_kiv2_4925.tsv says not assessed." >&2 printf 'metric\\tvalue\\nmarker\\tLPA KIV-2 4925G>A\\nrsid\\trs1484919544 (first KIV-2 copy)\\nstatus\\tnot assessed\\nalt_reads\\t.\\ndepth\\t.\\nfraction\\t.\\ncopies\\t.\\nkiv2_copies\\t.\\nmessage\\t%s\\n' \\ 'LPA KIV-2 4925G>A: not assessed, the check failed; see the log of step 46.' > ${meta.id}_lpa_kiv2_4925.tsv diff --git a/modules/local/parascopy/main.nf b/modules/local/parascopy/main.nf index 16da880fc..862bbf569 100644 --- a/modules/local/parascopy/main.nf +++ b/modules/local/parascopy/main.nf @@ -26,7 +26,7 @@ HBA1 and HBA2 have no Parascopy model: samtools bedcov (in the same image) sums the depth over both gene bodies and two single-copy flanks, - over all reads and over reads of MAPQ 1 or more, and bin/copy_number.py + over all reads and over reads of MAPQ 1 or more, and og cn call (the HBA row of bin/cn_loci.tsv) turns it into a count of alpha-globin genes (not assessed when no read over the genes has MAPQ 0: the aligner then left out the reads that fit both genes). @@ -46,7 +46,7 @@ Beside the SMN1 psCN, SMN1 g.27134T>G (rs143838139, Luo et al. 2014), a marker of SMN1 duplication alleles: samtools mpileup at its SMN1 and SMN2 - copies, read as a fraction of copies by bin/traits_report.py --marker + copies, read as a fraction of copies by og cn marker-read SMN1_27134 with the SMN table (_smn1_marker.tsv). It never proves a silent 2+0 carrier, and its message says so. @@ -109,10 +109,10 @@ process PARASCOPY { # the locus profile (aggregate and paralog-specific copy number, each with # filter and quality). # A BAM with ALT or HLA contigs is refused, as scripts/35-paralogs.sh does - # (bin/copy_number.py check-bam): its depth splits between a gene and its + # (og cn check-bam): its depth splits between a gene and its # ALT copy, and a depth-based count reads that as a deletion. samtools view -H ${bam} > bam_header.txt - copy_number.py check-bam --header bam_header.txt + og cn check-bam --header bam_header.txt models=${parascopy_data}/models_GRCh38_1KGP/${params.parascopy_population} [ -d "\$models" ] || { echo "ERROR: no Parascopy models \$models (populations: AFR AMR EAS EUR SAS)" >&2; exit 1; } parascopy depth -i ${bam}::${meta.id} -f ${reference} ${background} -o depth -@ ${task.cpus} @@ -158,11 +158,11 @@ process PARASCOPY { # SMN1 g.27134T>G beside the SMN1 psCN (as scripts/35-paralogs.sh): mpileup at its SMN1 and SMN2 # copies, then the reading with the SMN table; not assessed when it fails - if traits_report.py --marker-regions SMN1_27134 > smn1_marker_sites.txt \ + if og cn marker-regions SMN1_27134 > smn1_marker_sites.txt \ && ( while read -r r; do samtools mpileup -A -B -q 0 -Q 13 -d 100000 -r "\$r" ${bam} || exit 1; done \ < smn1_marker_sites.txt ) > smn1_marker_pileup.tsv \ - && traits_report.py --marker SMN1_27134 --pileup smn1_marker_pileup.tsv \ - --copy-table ${meta.id}_smn_copy_number.tsv --marker-out ${meta.id}_smn1_marker.tsv.tmp; then + && og cn marker-read SMN1_27134 --pileup smn1_marker_pileup.tsv \ + --copy-table ${meta.id}_smn_copy_number.tsv --out ${meta.id}_smn1_marker.tsv.tmp; then mv ${meta.id}_smn1_marker.tsv.tmp ${meta.id}_smn1_marker.tsv else rm -f ${meta.id}_smn1_marker.tsv.tmp @@ -173,10 +173,10 @@ process PARASCOPY { rm -f smn1_marker_sites.txt smn1_marker_pileup.tsv # HBA1/HBA2 from depth (as scripts/35-paralogs.sh): all MAPQ and MAPQ 1 or more, then the verdict - if copy_number.py bed --locus HBA > hba_regions.bed \\ + if og cn bed --locus HBA > hba_regions.bed \\ && samtools bedcov hba_regions.bed ${bam} > hba_bedcov.tsv \\ && samtools bedcov -Q 1 hba_regions.bed ${bam} > hba_bedcov_q1.tsv \\ - && copy_number.py call --locus HBA --bedcov hba_bedcov.tsv --bedcov-mapq1 hba_bedcov_q1.tsv \\ + && og cn call --locus HBA --bedcov hba_bedcov.tsv --bedcov-mapq1 hba_bedcov_q1.tsv \\ --table ${meta.id}_hba_copy_number.tsv.tmp; then mv ${meta.id}_hba_copy_number.tsv.tmp ${meta.id}_hba_copy_number.tsv else @@ -192,10 +192,10 @@ process PARASCOPY { dosage_loci="HP LCE3BC CCL3L1" printf 'locus\\tstatus\\tcopies\\testimate\\treason\\treading\\n' > cn_dosage.part for loc in \$dosage_loci; do - if copy_number.py bed --locus "\$loc" > dosage_regions.bed \\ + if og cn bed --locus "\$loc" > dosage_regions.bed \\ && samtools bedcov dosage_regions.bed ${bam} > dosage_bedcov.tsv \\ && samtools bedcov -Q 1 dosage_regions.bed ${bam} > dosage_bedcov_q1.tsv \\ - && copy_number.py call --locus "\$loc" --bedcov dosage_bedcov.tsv --bedcov-mapq1 dosage_bedcov_q1.tsv \\ + && og cn call --locus "\$loc" --bedcov dosage_bedcov.tsv --bedcov-mapq1 dosage_bedcov_q1.tsv \\ --table dosage_table.tsv; then awk -F'\\t' -v OFS='\\t' -v l="\$loc" 'NR > 1 {v[\$1] = \$2} END {print l, v["status"], v["copies"], v["estimate"], v["reason"], v["reading"]}' dosage_table.tsv >> cn_dosage.part @@ -209,14 +209,14 @@ process PARASCOPY { # CYP21A2 from depth and PSVs (as scripts/35-paralogs.sh): bedcov over the row (the RCCX module # total), mpileup without a reference at its PSV sites, then the verdict; not assessed when it fails - if copy_number.py bed --locus CYP21A2 > cyp21a2_regions.bed \\ - && copy_number.py bed --psv --locus CYP21A2 > cyp21a2_psv.bed \\ + if og cn bed --locus CYP21A2 > cyp21a2_regions.bed \\ + && og cn bed --psv --locus CYP21A2 > cyp21a2_psv.bed \\ && samtools bedcov cyp21a2_regions.bed ${bam} > cyp21a2_bedcov.tsv \\ && samtools bedcov -Q 1 cyp21a2_regions.bed ${bam} > cyp21a2_bedcov_q1.tsv \\ && samtools mpileup -A -B -q 0 -Q 13 -d 100000 \\ -r "\$(awk 'NR == 1 {c = \$1; s = \$2 + 1} {e = \$3} END {print c ":" s "-" e}' cyp21a2_psv.bed)" \\ -l cyp21a2_psv.bed ${bam} > cyp21a2_pileup.tsv \\ - && copy_number.py call --locus CYP21A2 --bedcov cyp21a2_bedcov.tsv --bedcov-mapq1 cyp21a2_bedcov_q1.tsv \\ + && og cn call --locus CYP21A2 --bedcov cyp21a2_bedcov.tsv --bedcov-mapq1 cyp21a2_bedcov_q1.tsv \\ --pileup cyp21a2_pileup.tsv \\ --table ${meta.id}_cyp21a2_psv.tsv.tmp; then mv ${meta.id}_cyp21a2_psv.tsv.tmp ${meta.id}_cyp21a2_psv.tsv @@ -231,11 +231,11 @@ process PARASCOPY { # Interval rows of bin/cn_loci.tsv (as scripts/35-paralogs.sh): one bedcov over every row's segments, # then the verdict; a failed pass writes every row as not assessed (check_failed) - if ! { copy_number.py bed --kind interval > interval_regions.bed \ + if ! { og cn bed --kind interval > interval_regions.bed \ && samtools bedcov interval_regions.bed ${bam} > interval_bedcov.tsv \ - && copy_number.py intervals --bedcov interval_bedcov.tsv --sex ${sex} --out ${meta.id}_cn_intervals.tsv; }; then + && og cn intervals --bedcov interval_bedcov.tsv --sex ${sex} --out ${meta.id}_cn_intervals.tsv; }; then echo "WARNING: the interval depth check failed; ${meta.id}_cn_intervals.tsv lists every row as not assessed." >&2 - copy_number.py intervals --failed --out ${meta.id}_cn_intervals.tsv + og cn intervals --failed --out ${meta.id}_cn_intervals.tsv fi rm -f interval_regions.bed interval_bedcov.tsv diff --git a/modules/local/pypgx/main.nf b/modules/local/pypgx/main.nf index 83c4b354e..31e334ae9 100644 --- a/modules/local/pypgx/main.nf +++ b/modules/local/pypgx/main.nf @@ -68,11 +68,11 @@ process PYPGX { def gq_floor = task.ext.gq_floor != null ? task.ext.gq_floor : 20 """ # A BAM with ALT or HLA contigs is refused, as scripts/32-pypgx.sh does - # (bin/copy_number.py check-bam): its depth splits between a gene and its + # (og cn check-bam): its depth splits between a gene and its # ALT copy, and a depth-based count reads that as a deletion. The contig # names come from pysam, which the image has. python3 -c "import pysam, sys; print(chr(10).join(pysam.AlignmentFile(sys.argv[1]).references))" ${bam} > bam_contigs.txt - copy_number.py check-bam --header bam_contigs.txt + og cn check-bam --header bam_contigs.txt OUTBASE="${meta.id}_pypgx_results" mkdir -p "\$OUTBASE" diff --git a/scripts/32-pypgx.sh b/scripts/32-pypgx.sh index 5e3093790..4d0cdee65 100755 --- a/scripts/32-pypgx.sh +++ b/scripts/32-pypgx.sh @@ -83,14 +83,14 @@ fi # A BAM aligned to a reference with ALT or HLA contigs is refused: pypgx # reads copy number from depth, and that depth splits between a gene and its -# ALT copy, which reads as a deletion (bin/copy_number.py check-bam, exit 4). +# ALT copy, which reads as a deletion (og cn check-bam, exit 4). # A header that cannot be read leaves the check undone, with a warning. It runs # after the bundle check, which starts no container. HDR="${OUTPUT_DIR}/bam_header.part" if run_in --cpus 1 --memory 1g "${SAMTOOLS_IMAGE}" samtools view -H "$(cpath "$BAM")" > "$HDR"; then rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PYTHON_IMAGE}" \ - python3 /pgp-bin/copy_number.py check-bam --header "$(cpath "$HDR")" || rc=$? + /pgp-bin/og cn check-bam --header "$(cpath "$HDR")" || rc=$? rm -f "$HDR" if [ "$rc" -eq 4 ]; then exit 1 diff --git a/scripts/35-paralogs.sh b/scripts/35-paralogs.sh index 64d609f9c..2e9c39088 100755 --- a/scripts/35-paralogs.sh +++ b/scripts/35-paralogs.sh @@ -21,8 +21,8 @@ # HBA1 and HBA2 (alpha-thalassaemia deletions) have no Parascopy model, so the # step counts them from read depth: the depth over both gene bodies against # single-copy sequence on each side of the cluster gives the number of -# alpha-globin genes (bin/copy_number.py on the HBA row of bin/cn_loci.tsv, -# laid out by bin/hba_copy_number.py). A failure there writes an HBA +# alpha-globin genes (og cn call on the HBA row of bin/cn_loci.tsv). A +# failure there writes an HBA # table that says not assessed, and does not stop the step. # # The gene-dosage rows of the same catalog (HP, LCE3BC, CCL3L1: loci no @@ -39,7 +39,7 @@ # Beside the SMN1 psCN the step reads SMN1 g.27134T>G (rs143838139; Luo et # al. 2014), a marker of SMN1 duplication alleles: samtools mpileup at its # SMN1 and SMN2 copies (the reads cannot tell them apart there), read as a -# fraction of copies by bin/traits_report.py --marker SMN1_27134. With two +# fraction of copies by og cn marker-read SMN1_27134. With two # SMN1 copies it raises or lowers the chance of a silent 2+0 carrier; it # never proves 2+0, and the table's message says so; # ${SAMPLE}_smn1_marker.tsv. @@ -106,14 +106,14 @@ for f in "$BAM" "${BAM}.bai" "$REF_FASTA" "${REF_FASTA}.fai" "$LOCI"; do done # A BAM aligned to a reference with ALT or HLA contigs is refused: Parascopy # and the HBA depth count read copy number from depth, and that depth splits -# between a gene and its ALT copy (bin/copy_number.py check-bam, exit 4). +# between a gene and its ALT copy (og cn check-bam, exit 4). # A header that cannot be read leaves the check undone, with a warning. mkdir -p "$OUTDIR" HDR="${OUTDIR}/bam_header.part" if run_in --cpus 1 --memory 1g "${PARASCOPY_IMAGE}" samtools view -H "$(cpath "$BAM")" > "$HDR"; then rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py check-bam --header "$(cpath "$HDR")" || rc=$? + /pgp-bin/og cn check-bam --header "$(cpath "$HDR")" || rc=$? rm -f "$HDR" if [ "$rc" -eq 4 ]; then exit 1 @@ -246,7 +246,7 @@ mv "$WORK" "${OUTDIR}/${SAMPLE}_parascopy" # SMN1 g.27134T>G beside the SMN1 psCN (as the PARASCOPY module): samtools # mpileup at its SMN1 and SMN2 copies, one -r per site, every MAPQ, # duplicates left out, bases of quality 13 or more; then -# bin/traits_report.py --marker SMN1_27134 with the SMN table (the SMN1 psCN +# og cn marker-read SMN1_27134 with the SMN table (the SMN1 psCN # at exon 7 and the agCN). A failure writes a table that says not assessed, # and the step goes on. SMN1M_OUT="${OUTDIR}/${SAMPLE}_smn1_marker.tsv" @@ -255,13 +255,13 @@ rm -rf "$SMN1M_OUT" "$SMN1M_WORK" mkdir -p "$SMN1M_WORK" # shellcheck disable=SC2016 # $1 to $3 belong to the inner bash if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/traits_report.py --marker-regions SMN1_27134 > "${SMN1M_WORK}/sites.txt" \ + /pgp-bin/og cn marker-regions SMN1_27134 > "${SMN1M_WORK}/sites.txt" \ && run_in --cpus 1 --memory 2g "${PARASCOPY_IMAGE}" bash -euo pipefail -c ' while read -r r; do samtools mpileup -A -B -q 0 -Q 13 -d 100000 -r "$r" "$1"; done < "$2" > "$3"' \ _ "$(cpath "$BAM")" "$(cpath "${SMN1M_WORK}/sites.txt")" "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/traits_report.py --marker SMN1_27134 --pileup "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ - --copy-table "$(cpath "${OUTDIR}/${SAMPLE}_smn_copy_number.tsv")" --marker-out "$(cpath "${SMN1M_OUT}.tmp")"; then + /pgp-bin/og cn marker-read SMN1_27134 --pileup "$(cpath "${SMN1M_WORK}/pileup.tsv")" \ + --copy-table "$(cpath "${OUTDIR}/${SAMPLE}_smn_copy_number.tsv")" --out "$(cpath "${SMN1M_OUT}.tmp")"; then mv "${SMN1M_OUT}.tmp" "$SMN1M_OUT" else rm -f "${SMN1M_OUT}.tmp" @@ -274,19 +274,19 @@ rm -rf "$SMN1M_WORK" # HBA1/HBA2 from depth (as the PARASCOPY module): samtools bedcov over the # regions of the HBA row of bin/cn_loci.tsv, all MAPQ and MAPQ 1 or more (a # BAM with no MAPQ 0 read over the genes left out the reads that fit both), -# then the engine's verdict (bin/copy_number.py). +# then the engine's verdict (og cn call). HBA_OUT="${OUTDIR}/${SAMPLE}_hba_copy_number.tsv" HBA_WORK="${OUTDIR}/hba_depth.part" rm -rf "$HBA_OUT" "$HBA_WORK" mkdir -p "$HBA_WORK" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus HBA > "${HBA_WORK}/regions.bed" \ + /pgp-bin/og cn bed --locus HBA > "${HBA_WORK}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${HBA_WORK}/regions.bed")" "$(cpath "$BAM")" > "${HBA_WORK}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov -Q 1 "$(cpath "${HBA_WORK}/regions.bed")" "$(cpath "$BAM")" > "${HBA_WORK}/bedcov_q1.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus HBA --bedcov "$(cpath "${HBA_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus HBA --bedcov "$(cpath "${HBA_WORK}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${HBA_WORK}/bedcov_q1.tsv")" --table "$(cpath "${HBA_OUT}.tmp")"; then mv "${HBA_OUT}.tmp" "$HBA_OUT" else @@ -299,7 +299,7 @@ rm -rf "$HBA_WORK" # Gene-dosage rows of bin/cn_loci.tsv that no pinned caller models (as the # PARASCOPY module): per locus samtools bedcov, all MAPQ and MAPQ 1 or more, -# then copy_number.py call; one line per locus in _cn_dosage.tsv. A +# then og cn call; one line per locus in _cn_dosage.tsv. A # locus whose check fails is not assessed there, and the step goes on. DOSAGE_LOCI="HP LCE3BC CCL3L1" DOSAGE_OUT="${OUTDIR}/${SAMPLE}_cn_dosage.tsv" @@ -311,13 +311,13 @@ for loc in $DOSAGE_LOCI; do d="${DOSAGE_WORK}/${loc}" mkdir -p "$d" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus "$loc" > "${d}/regions.bed" \ + /pgp-bin/og cn bed --locus "$loc" > "${d}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${d}/regions.bed")" "$(cpath "$BAM")" > "${d}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov -Q 1 "$(cpath "${d}/regions.bed")" "$(cpath "$BAM")" > "${d}/bedcov_q1.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus "$loc" --bedcov "$(cpath "${d}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus "$loc" --bedcov "$(cpath "${d}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${d}/bedcov_q1.tsv")" --table "$(cpath "${d}/table.tsv")"; then awk -F'\t' -v OFS='\t' -v l="$loc" 'NR > 1 {v[$1] = $2} END {print l, v["status"], v["copies"], v["estimate"], v["reason"], v["reading"]}' "${d}/table.tsv" \ @@ -335,7 +335,7 @@ rm -rf "$DOSAGE_WORK" # the CYP21A2 row of bin/cn_loci.tsv (CYP21A2 and CYP21A1P added up against # single-copy flanks: the RCCX module total), samtools mpileup without a # reference at its paralogous sequence variant sites (each on CYP21A2 and on -# CYP21A1P, every read whatever its MAPQ), then copy_number.py call: the +# CYP21A1P, every read whatever its MAPQ), then og cn call: the # CYP21A2 copies at each site, and a deletion or a possible conversion. The # bedcov runs twice, all MAPQ and MAPQ 1 or more (as HBA): a BAM without the # MAPQ 0 reads that fit both copies reads not assessed, never a deletion. A @@ -345,9 +345,9 @@ CYP21_WORK="${OUTDIR}/CYP21A2.part" rm -rf "$CYP21_OUT" "$CYP21_WORK" mkdir -p "$CYP21_WORK" if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --locus CYP21A2 > "${CYP21_WORK}/regions.bed" \ + /pgp-bin/og cn bed --locus CYP21A2 > "${CYP21_WORK}/regions.bed" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --psv --locus CYP21A2 > "${CYP21_WORK}/psv.bed" \ + /pgp-bin/og cn bed --psv --locus CYP21A2 > "${CYP21_WORK}/psv.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${CYP21_WORK}/regions.bed")" "$(cpath "$BAM")" > "${CYP21_WORK}/bedcov.tsv" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ @@ -357,7 +357,7 @@ if run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ -r "$(awk 'NR == 1 {c = $1; s = $2 + 1} {e = $3} END {print c ":" s "-" e}' "${CYP21_WORK}/psv.bed")" \ -l "$(cpath "${CYP21_WORK}/psv.bed")" "$(cpath "$BAM")" > "${CYP21_WORK}/pileup.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py call --locus CYP21A2 --bedcov "$(cpath "${CYP21_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn call --locus CYP21A2 --bedcov "$(cpath "${CYP21_WORK}/bedcov.tsv")" \ --bedcov-mapq1 "$(cpath "${CYP21_WORK}/bedcov_q1.tsv")" --pileup "$(cpath "${CYP21_WORK}/pileup.tsv")" \ --table "$(cpath "${CYP21_OUT}.tmp")"; then mv "${CYP21_OUT}.tmp" "$CYP21_OUT" @@ -371,7 +371,7 @@ fi rm -rf "$CYP21_WORK" # Interval rows of bin/cn_loci.tsv (as the PARASCOPY module): one samtools -# bedcov over the segments of every row, then copy_number.py intervals; one +# bedcov over the segments of every row, then og cn intervals; one # line per row in _cn_intervals.tsv. A failed pass writes every row as # not assessed (check_failed), and the step goes on; when even that container # fails, the table is the header alone and the log says no row was assessed. @@ -380,15 +380,15 @@ INTERVAL_WORK="${OUTDIR}/cn_intervals.part" rm -rf "$INTERVAL_OUT" "$INTERVAL_WORK" mkdir -p "$INTERVAL_WORK" if ! { run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py bed --kind interval > "${INTERVAL_WORK}/regions.bed" \ + /pgp-bin/og cn bed --kind interval > "${INTERVAL_WORK}/regions.bed" \ && run_in --cpus 2 --memory 2g "${PARASCOPY_IMAGE}" \ samtools bedcov "$(cpath "${INTERVAL_WORK}/regions.bed")" "$(cpath "$BAM")" > "${INTERVAL_WORK}/bedcov.tsv" \ && run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py intervals --bedcov "$(cpath "${INTERVAL_WORK}/bedcov.tsv")" \ + /pgp-bin/og cn intervals --bedcov "$(cpath "${INTERVAL_WORK}/bedcov.tsv")" \ --sex "$SEX" --out "$(cpath "$INTERVAL_OUT")"; }; then echo "WARNING: the interval depth check failed; ${INTERVAL_OUT##*/} lists every row as not assessed." >&2 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "${PARASCOPY_IMAGE}" \ - python3 /pgp-bin/copy_number.py intervals --failed --out "$(cpath "$INTERVAL_OUT")" \ + /pgp-bin/og cn intervals --failed --out "$(cpath "$INTERVAL_OUT")" \ || printf 'locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\testimate\treason\n' > "$INTERVAL_OUT" fi rm -rf "$INTERVAL_WORK" diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index bc687f5e5..061500751 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -49,13 +49,13 @@ if [ -f "$BAM" ] && [ -f "${BAM}.bai" ]; then echo "[1/2] Read depth at the marker sites and over RHD (samtools bedcov, MAPQ >= 20)..." # The marker table is a BED with extra columns; bedcov reads it once the # comment and header lines are gone. The RHD regions come from - # bin/copy_number.py (BLOODTYPE_REGIONS in Nextflow) as a 4-column BED. + # og cn bed (BLOODTYPE_REGIONS in Nextflow) as a 4-column BED. # bedcov stops on a contig the BAM has no @SQ line for, so those rows are # left out: their sites get no depth row and are not determined. The # BLOODTYPE_DEPTH process runs the same commands. # shellcheck disable=SC2016 # $1 belongs to the inner shell run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - sh -c 'python3 /pgp-bin/copy_number.py bed --locus RHD > "$1"' _ "${O}/rhd.bed" + sh -c '/pgp-bin/og cn bed --locus RHD > "$1"' _ "${O}/rhd.bed" # shellcheck disable=SC2016 # $1 to $6 belong to the inner bash, the two programs to awk run_in -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$SAMTOOLS_IMAGE" bash -euo pipefail -c ' samtools view -H "$3" | awk -F "\t" "$5" > "$2/contigs.txt" diff --git a/scripts/46-lpa.sh b/scripts/46-lpa.sh index def07ded0..a682a9180 100755 --- a/scripts/46-lpa.sh +++ b/scripts/46-lpa.sh @@ -6,14 +6,12 @@ # of apo(a); a low total goes with a higher Lp(a) level, a mostly inherited # cardiovascular risk factor. The step counts the repeats from the depth over # the KIV-2 exons against the depth over single-copy LPA exons -# (bin/copy_number.py on the LPA_KIV2 row of bin/cn_loci.tsv, laid out by -# bin/lpa_kiv2.py; the same code as the LPA Nextflow processes), with a 95% -# interval. Beside the count it reads the KIV-2 4925G>A splice variant +# (og cn call on the LPA_KIV2 row of bin/cn_loci.tsv; the same code as the +# LPA Nextflow processes), with a 95% interval. Beside the count it reads the KIV-2 4925G>A splice variant # (Coassin et al. 2017), which lowers the Lp(a) made from the chromosome that # carries it: samtools mpileup at the site's seven GRCh38 copies (the six -# KIV-2 units and KIV-1), read as a fraction of copies by -# bin/traits_report.py --marker LPA_KIV2_4925 into its own table, laid out by -# bin/lpa_kiv2.py. +# KIV-2 units and KIV-1), read as a fraction of copies in the same og cn +# call, into its own table. # The two common risk variants, rs10455872 and rs3798220, are rows # of step 42's traits panel; the report's Lp(a) section reads both. A research # estimate, not a diagnostic test: the actionable test is a blood Lp(a) @@ -59,13 +57,13 @@ echo "BAM: ${BAM}" mkdir -p "$OUTDIR" rm -f "$TABLE" "$DEPTH" "$DEPTH_Q1" "$PILEUP" "$NULL" "$BED" "$SITES" -echo "[1/4] The regions (bin/copy_number.py bed --locus LPA_KIV2) and the 4925G>A sites..." +echo "[1/3] The regions (og cn bed --locus LPA_KIV2) and the 4925G>A sites..." # shellcheck disable=SC2016 # $1 and $2 belong to the inner shell run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - sh -c 'python3 /pgp-bin/copy_number.py bed --locus LPA_KIV2 > "$1" && - python3 /pgp-bin/traits_report.py --marker-regions LPA_KIV2_4925 > "$2"' _ "$(cpath "$BED")" "$(cpath "$SITES")" + sh -c '/pgp-bin/og cn bed --locus LPA_KIV2 > "$1" && + /pgp-bin/og cn marker-regions LPA_KIV2_4925 > "$2"' _ "$(cpath "$BED")" "$(cpath "$SITES")" -echo "[2/4] Read depth over the KIV-2 and single-copy LPA exons (samtools bedcov -j, all MAPQ and MAPQ >= 1)" +echo "[2/3] Read depth over the KIV-2 and single-copy LPA exons (samtools bedcov -j, all MAPQ and MAPQ >= 1)" echo " and the reads at the 4925G>A sites (samtools mpileup, every MAPQ)..." # bedcov stops on a contig the BAM has no @SQ line for: without chr6 the # tables stay empty and the count is not assessed. mpileup runs once per site @@ -86,18 +84,20 @@ run_in --cpus 1 --memory 2g "$SAMTOOLS_IMAGE" bash -euo pipefail -c ' "$(cpath "$PILEUP")" "$(cpath "$SITES")" rm -f "$BED" "$SITES" -echo "[3/4] The count and its interval (bin/copy_number.py call --locus LPA_KIV2)..." +echo "[3/3] The count, its interval and KIV-2 4925G>A beside it (og cn call --locus LPA_KIV2 --marker LPA_KIV2_4925)..." +# One run reads the count and the marker, so the marker is judged on the +# count's own MAPQ 0 share. A failure of the marker never costs the count (as +# step 35's SMN1 marker): when the count table is there, the 4925G>A table +# then says not assessed, and the step goes on. The LPA process writes the +# same table. +rc=0 run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/copy_number.py call --locus LPA_KIV2 --bedcov "$(cpath "$DEPTH")" \ - --bedcov-mapq1 "$(cpath "$DEPTH_Q1")" --table "$(cpath "$TABLE")" - -echo "[4/4] KIV-2 4925G>A beside the count (bin/traits_report.py --marker LPA_KIV2_4925)..." -# A failure here never costs the count (as step 35's SMN1 marker): the -# 4925G>A table then says not assessed, and the step goes on. The LPA -# process writes the same table. -if ! run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/traits_report.py --marker LPA_KIV2_4925 --pileup "$(cpath "$PILEUP")" \ - --copy-table "$(cpath "$TABLE")" --marker-out "$(cpath "$NULL")"; then + /pgp-bin/og cn call --locus LPA_KIV2 --bedcov "$(cpath "$DEPTH")" \ + --bedcov-mapq1 "$(cpath "$DEPTH_Q1")" --table "$(cpath "$TABLE")" \ + --marker LPA_KIV2_4925 --marker-pileup "$(cpath "$PILEUP")" --marker-out "$(cpath "$NULL")" || rc=$? +if [ "$rc" -ne 0 ]; then + # No count table: the count itself failed (exit 3 for a broken control). + [ -s "$TABLE" ] || exit "$rc" echo "WARNING: the KIV-2 4925G>A check failed; ${NULL##*/} says not assessed." >&2 printf 'metric\tvalue\nmarker\tLPA KIV-2 4925G>A\nrsid\trs1484919544 (first KIV-2 copy)\nstatus\tnot assessed\nalt_reads\t.\ndepth\t.\nfraction\t.\ncopies\t.\nkiv2_copies\t.\nmessage\t%s\n' \ 'LPA KIV-2 4925G>A: not assessed, the check failed; see the log of step 46.' > "$NULL" diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 9def678d8..cf6c104fc 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -26,9 +26,10 @@ from bin/ancestry_fine_report.py, step 35b's table from bin/paraphase_report.py, step 39's from bin/couple_report.py and step 42's from bin/traits_report.py, each fed made-up inputs; step 45's rows come - from bin/ystr_report.py's marker_row, step 46's from bin/lpa_kiv2.py's - assess, step 35's HBA1/HBA2 table from bin/hba_copy_number.py's assess, - step 49's depth rows from bin/chip_report.py's detection_limit, step 50's + from bin/ystr_report.py's marker_row, step 46's table and step 35's + HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE + and LPA_TABLE; tests/test_hba_copy_number.py and tests/test_lpa_kiv2.py + check that og still writes them), step 49's depth rows from bin/chip_report.py's detection_limit, step 50's tables from bin/mei_report.py on a made-up VCF and depth, and step 47's from bin/viral_report.py's build_rows, with made-up counts; steps 02, 40 and 41's tables are written here in their writers' columns. @@ -882,13 +883,8 @@ def region(total, rsi, **kw): "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\nchr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t2,2\t33,30\t" "chr5:70049000-70077000:-\n") - # Step 35's HBA1/HBA2 table: bin/hba_copy_number.py's own assess() on - # made-up depths, four alpha-globin genes. - import hba_copy_number as hba - depth = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} - res = hba.assess(depth, {k: v * (0.76 if k.startswith("HBA") else 0.98) for k, v in depth.items()}) - put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), - "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in hba.ORDER)) + # Step 35's HBA1/HBA2 table: four alpha-globin genes. + put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), HBA_TABLE) def couple(d, rng): @@ -1047,18 +1043,38 @@ def viral(d): w.writerows(rows) +# Step 35's HBA1/HBA2 depths (mean depth per region of the HBA row of +# bin/cn_loci.tsv, every read) and the share of each left at MAPQ 1 or more, +# and the table og cn call --locus HBA writes for them: four genes. +HBA_DEPTH = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} +HBA_Q1 = {"flank_left": 0.98, "HBA2": 0.76, "HBA1": 0.76, "flank_right": 0.98} +HBA_TABLE = ("metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t4\nestimate\t3.96\nhba2_depth\t30.9\n" + "hba1_depth\t30.1\nflank_left_depth\t31.2\nflank_right_depth\t30.4\nmapq0_share\t0.240\n" + "message\tfour alpha-globin genes (aa/aa): no deletion seen\n") + + +def lpa_depth(row): + """Step 46's made-up depths over the LPA_KIV2 row's segments (catalog + order, from bin/copy_number.py's load_catalog): every read, and MAPQ 1 or + more. About 37 KIV-2 repeats at a flank depth near 31, half the KIV-2 + depth at MAPQ 0.""" + m = {n: 86.5 * (1.02 if i % 2 == 0 else 0.98) for i, (n, *_) in enumerate(row["_targets"])} + m.update({n: 31.0 * (1.03 if i % 2 == 0 else 0.97) for i, (n, *_) in enumerate(row["_controls"])}) + return m, {n: v * (0.5 if n.startswith("kiv2_") else 0.98) for n, v in m.items()} + + +# The table og cn call --locus LPA_KIV2 writes for lpa_depth. +LPA_TABLE = ("metric\tvalue\nstatus\testimated\nkiv2_copies\t37.0\nci95_low\t36.3\nci95_high\t37.8\n" + "class\ttypical\nkiv2_depth\t86.61\nflank_depth\t31.09\nmapq0_share\t0.500\n" + "message\tabout 37 KIV-2 repeats in total (95% interval 36.3 to 37.8): within or overlapping the range " + "most people have (30 to 45 for about 70% of the 1000 Genomes samples)\n" + "advice\tA research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 " + "European Atherosclerosis Society consensus advises measuring it at least once in every adult.\n") + + def lpa(d, rng): - """Step 46: bin/lpa_kiv2.py's assess on made-up depths, about 37 KIV-2 - repeats at a flank depth near 31, half the KIV-2 depth at MAPQ 0.""" - import lpa_kiv2 - m, q1 = {}, {} - for *_, name in lpa_kiv2.REGIONS: - m[name] = rng.uniform(83, 90) if name.startswith("kiv2_") else rng.uniform(29.5, 32.5) - q1[name] = m[name] * (0.5 if name.startswith("kiv2_") else 0.98) - res = lpa_kiv2.assess(m, q1) - res["advice"] = lpa_kiv2.ADVICE - put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), - "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in lpa_kiv2.ORDER)) + """Step 46: the table og cn call writes for lpa_depth (LPA_TABLE).""" + put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), LPA_TABLE) def chip(d, rng): diff --git a/tests/e2e/cn-engine-1-parity.sh b/tests/e2e/cn-engine-1-parity.sh index d43124710..a4b04c253 100755 --- a/tests/e2e/cn-engine-1-parity.sh +++ b/tests/e2e/cn-engine-1-parity.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The copy-number engine (bin/copy_number.py, bin/cn_loci.tsv) against the +# The copy-number engine (og cn, bin/cn_loci.tsv) against the # per-gene code it replaced, on real reads: HG002's Illumina reads at the # alpha-globin cluster (chr16:95,001-235,000), the RH locus # (chr1:25.15-25.45 Mb) and LPA (chr6:160.53-160.67 Mb) from the 35x BWA-MEM @@ -10,8 +10,8 @@ # - step 41 of this tree and of the old tree write the same RhD row and the # same RhD evidence rows (other systems may change after the engine); # - step 35's HBA block (bedcov all MAPQ and MAPQ >= 1, then the verdict) -# writes the same table through copy_number.py as the old -# hba_copy_number.py does. +# writes the same table through og cn as the old hba_copy_number.py +# does. # The old tree is main at OLD_SHA, the last commit before the engine (the one # tests/test_copy_number.py froze its GOLD tables from), fetched by its sha # at run time, so the comparison keeps its meaning once the engine is on main. @@ -53,19 +53,20 @@ check_ge "HG002 reads at the three loci" "$(dk "$SAMTOOLS_IMAGE" samtools view - # dkb BIN IMAGE CMD...: dk with a tree's bin/ at its own path (this checkout is outside the work folder). dkb() { docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "$1:$1:ro" -w "$T" "$2" "${@:3}"; } -# hba_block TREE OUT BAM: step 35's HBA block with TREE's bin (main: hba_copy_number.py). +# hba_block TREE OUT BAM: step 35's HBA block with TREE's bin (this tree: og cn; +# the old tree: hba_copy_number.py). hba_block() { local bin="$1/bin" out=$2 bam=$3 d="${T}/hba.$$" rm -rf "$d"; mkdir -p "$d" - if [ -e "${bin}/copy_number.py" ]; then - dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/copy_number.py" bed --locus HBA > "${d}/r.bed" + if [ -e "${bin}/og" ]; then + dkb "$bin" "$PARASCOPY_IMAGE" "${bin}/og" cn bed --locus HBA > "${d}/r.bed" else dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/hba_copy_number.py" bed > "${d}/r.bed" fi dkb "$bin" "$PARASCOPY_IMAGE" samtools bedcov "${d}/r.bed" "$bam" > "${d}/c.tsv" dkb "$bin" "$PARASCOPY_IMAGE" samtools bedcov -Q 1 "${d}/r.bed" "$bam" > "${d}/q.tsv" - if [ -e "${bin}/copy_number.py" ]; then - dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/copy_number.py" call --locus HBA --bedcov "${d}/c.tsv" \ + if [ -e "${bin}/og" ]; then + dkb "$bin" "$PARASCOPY_IMAGE" "${bin}/og" cn call --locus HBA --bedcov "${d}/c.tsv" \ --bedcov-mapq1 "${d}/q.tsv" --table "$out" else dkb "$bin" "$PARASCOPY_IMAGE" python3 "${bin}/hba_copy_number.py" check --bedcov "${d}/c.tsv" \ diff --git a/tests/fake-docker/cn-engine-calls.sh b/tests/fake-docker/cn-engine-calls.sh index 043cb99a5..f14492911 100755 --- a/tests/fake-docker/cn-engine-calls.sh +++ b/tests/fake-docker/cn-engine-calls.sh @@ -1,19 +1,26 @@ #!/usr/bin/env bash # Steps 35 (the HBA1/HBA2 block), 41 (RhD) and 46 (LPA KIV-2) count copies -# through one engine, bin/copy_number.py, with a hook that plays Parascopy and -# samtools bedcov (made-up depths: 30x everywhere, the KIV-2 block at 20 -# repeats) and runs the engine for real on the host: -# - each step calls copy_number.py bed once and step 35 and 46 call -# copy_number.py call once (step 41's report imports the engine), and no -# step calls hba_copy_number.py or lpa_kiv2.py any more; +# through one engine, og cn, with a hook that plays Parascopy and samtools +# bedcov (made-up depths: 30x everywhere, the KIV-2 block at 20 repeats) and +# runs og for real on the host (bin/og): +# - each step calls og cn bed once and step 35 and 46 call og cn call once +# (step 41's report imports bin/copy_number.py), step 46 with the 4925G>A +# marker in the same call, and no step calls the old Python any more; # - the tables say what the depths give: four alpha-globin genes, RhD D+ # with two copies, 20.0 KIV-2 repeats; # - a control segment with no read (FAKE_ZERO=flank_e19, the others at 30x) # fails step 46 with BROKEN_CONTROL, never a count. +# It needs og built (scripts/setup.sh --og). The fake-docker job builds no og, +# so there the case says it is skipped; Guard's rust job runs it with og and +# fails on that line. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" # shellcheck source=../../versions.env . "${REPO_ROOT}/versions.env" +if ! "${REPO_ROOT}/bin/og" version > /dev/null 2>&1; then + echo "skip: bin/og is not built here (Guard's rust job runs this case with og)" + exit 0 +fi export GENOME_DIR="${CASE_WORK}/genome" seed_reference "$GENOME_DIR" @@ -70,7 +77,7 @@ case "$*" in "parascopy cn-using"*) o=$(host_path "$(arg -o "$@")"); mkdir -p "$o" printf 'chr5\t70925087\t70953015\tSMN1\t.\tPASS\t4\t40\tPASS\t2,2\t30\t.\tchr5:70049669-70077595\n' | gzip -c > "${o}/res.samples.bed.gz" ;; - "python3 /pgp-bin/"*) + "python3 /pgp-bin/"* | "/pgp-bin/og "*) args=(); for a in "$@"; do args+=("$(map "$a")"); done exec "${args[@]}" ;; "sh -c "*) @@ -101,9 +108,9 @@ val() { awk -F'\t' -v k="$2" '$1 == k {print $2; exit}' "$1"; } # --- step 35 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 paralogs "${SCRIPTS}/35-paralogs.sh" sample1 -[ "$(calls 'copy_number.py bed --locus HBA')" = 1 ] || fail "step 35: copy_number.py bed --locus HBA was not called once" -[ "$(calls 'copy_number.py call --locus HBA')" = 1 ] || fail "step 35: copy_number.py call --locus HBA was not called once" -[ "$(calls 'hba_copy_number.py')" = 0 ] || fail "step 35 still calls hba_copy_number.py" +[ "$(calls 'og cn bed --locus HBA')" = 1 ] || fail "step 35: og cn bed --locus HBA was not called once" +[ "$(calls 'og cn call --locus HBA')" = 1 ] || fail "step 35: og cn call --locus HBA was not called once" +[ "$(calls '.py')" = 0 ] || fail "step 35 still calls Python: $(grep -F '.py' "$FAKE_DOCKER_LOG")" H="${S}/paralogs/sample1_hba_copy_number.tsv" [ "$(val "$H" alpha_globin_genes)" = 4 ] || fail "step 35: four alpha-globin genes expected: $(cat "$H")" [ "$(val "$H" status)" = called ] || fail "step 35: HBA not called: $(cat "$H")" @@ -111,7 +118,7 @@ H="${S}/paralogs/sample1_hba_copy_number.tsv" # --- step 41 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 bloodtype "${SCRIPTS}/41-bloodtype.sh" sample1 -[ "$(calls 'copy_number.py bed --locus RHD')" = 1 ] || fail "step 41: copy_number.py bed --locus RHD was not called once" +[ "$(calls 'og cn bed --locus RHD')" = 1 ] || fail "step 41: og cn bed --locus RHD was not called once" [ "$(awk -F'\t' '$4 == "RHD_GENE" {n++} END {print n + 0}' "${S}/bloodtype/sample1_bloodtype_depth.tsv")" = 1 ] \ || fail "step 41: the depth table has no RHD_GENE row" R=$(awk -F'\t' '$1 == "RhD"' "${S}/bloodtype/sample1_bloodtype.tsv") @@ -120,11 +127,14 @@ case "$R" in *"D+"*"RHD/RHD (RHD copies by read depth 2.00)"*) ;; *) fail "step # --- step 46 --------------------------------------------------------------------------- : > "$FAKE_DOCKER_LOG" run_expect 0 lpa "${SCRIPTS}/46-lpa.sh" sample1 -[ "$(calls 'copy_number.py bed --locus LPA_KIV2')" = 1 ] || fail "step 46: copy_number.py bed --locus LPA_KIV2 was not called once" -[ "$(calls 'copy_number.py call --locus LPA_KIV2')" = 1 ] || fail "step 46: copy_number.py call --locus LPA_KIV2 was not called once" -[ "$(calls 'lpa_kiv2.py')" = 0 ] || fail "step 46 still calls lpa_kiv2.py" +[ "$(calls 'og cn bed --locus LPA_KIV2')" = 1 ] || fail "step 46: og cn bed --locus LPA_KIV2 was not called once" +[ "$(calls 'og cn call --locus LPA_KIV2')" = 1 ] || fail "step 46: og cn call --locus LPA_KIV2 was not called once" +[ "$(calls '--marker LPA_KIV2_4925')" = 1 ] || fail "step 46: the 4925G>A marker was not read in the count's call" +[ "$(calls '.py')" = 0 ] || fail "step 46 still calls Python: $(grep -F '.py' "$FAKE_DOCKER_LOG")" L="${S}/lpa/sample1_lpa_kiv2.tsv" [ "$(val "$L" kiv2_copies)" = 20.0 ] || fail "step 46: 20.0 KIV-2 repeats expected: $(cat "$L")" +N="${S}/lpa/sample1_lpa_kiv2_4925.tsv" +[ -n "$(val "$N" status)" ] || fail "step 46: no 4925G>A table beside the count: $(cat "$N" 2>/dev/null)" # A control exon with no read while the others have 30x: a named error, no table FAKE_ZERO=flank_e19 run_expect 3 lpa-broken "${SCRIPTS}/46-lpa.sh" sample1 diff --git a/tests/test_callability.py b/tests/test_callability.py index f5048e717..b083181ca 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -23,11 +23,12 @@ unreadable, not a reading; a chromosome with no row at all (a BAM whose reference lacks it, as the e2e fixture's) puts its genes under "not on this reference"; - 5. bin/copy_number.py check-bam and call --bam-header refuse a header with - an ALT or an HLA contig (exit 4, a REFUSED line) and pass a header - without one (the positive control). + 5. og cn check-bam and call --bam-header refuse a header with an ALT or + an HLA contig (exit 4, a REFUSED line) and pass a header without one + (the positive control). They need og: OG names the binary; without it + part 5 is skipped, and with OG_REQUIRED set too the test fails. -Run: python3 tests/test_callability.py +Run: OG=rust/target/release/og python3 tests/test_callability.py """ import csv import gzip @@ -203,33 +204,40 @@ def count_matches(reader, tsv): # --- 5. the ALT contig refusal --------------------------------------------------------- +OG = os.environ.get("OG", "") + + def copy_number(*args, stdin=None): - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "copy_number.py"), *args], - capture_output=True, text=True, input=stdin) + p = subprocess.run([OG, "cn", *args], capture_output=True, text=True, input=stdin) return p.returncode, p.stderr -with tempfile.TemporaryDirectory() as work: - noalt = os.path.join(work, "noalt.txt") - with open(noalt, "w") as f: - f.write("@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n") - alt = os.path.join(work, "alt.txt") - with open(alt, "w") as f: - f.write("@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n") - rc, err = copy_number("check-bam", "--header", noalt) - check("5. positive control: a header without ALT contigs passes", rc == 0 and not err, (rc, err)) - rc, err = copy_number("check-bam", "--header", alt) - check("5. a header with an ALT contig is refused (exit 4, a REFUSED line naming it)", - rc == 4 and err.startswith("REFUSED:") and "chr22_KI270879v1_alt" in err, (rc, err)) - rc, err = copy_number("check-bam", "--header", "-", stdin="chr1\nHLA-A*01:01:01:01\n") - check("5. contig names on stdin with an HLA contig are refused", rc == 4 and "HLA-A" in err, (rc, err)) - bed = os.path.join(work, "bedcov.tsv") - open(bed, "w").close() - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", alt) - check("5. call --bam-header refuses before it reads any depth", rc == 4 and err.startswith("REFUSED:"), - (rc, err)) - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", noalt) - check("5. call --bam-header without ALT contigs goes on (two empty tables: not assessed)", rc == 0, (rc, err)) +if not OG: + print("skip: OG is not set (OG=rust/target/release/og): part 5 did not run") + if os.environ.get("OG_REQUIRED"): + FAILS += 1 +else: + with tempfile.TemporaryDirectory() as work: + noalt = os.path.join(work, "noalt.txt") + with open(noalt, "w") as f: + f.write("@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n") + alt = os.path.join(work, "alt.txt") + with open(alt, "w") as f: + f.write("@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n") + rc, err = copy_number("check-bam", "--header", noalt) + check("5. positive control: a header without ALT contigs passes", rc == 0 and not err, (rc, err)) + rc, err = copy_number("check-bam", "--header", alt) + check("5. a header with an ALT contig is refused (exit 4, a REFUSED line naming it)", + rc == 4 and err.startswith("REFUSED:") and "chr22_KI270879v1_alt" in err, (rc, err)) + rc, err = copy_number("check-bam", "--header", "-", stdin="chr1\nHLA-A*01:01:01:01\n") + check("5. contig names on stdin with an HLA contig are refused", rc == 4 and "HLA-A" in err, (rc, err)) + bed = os.path.join(work, "bedcov.tsv") + open(bed, "w").close() + rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", alt) + check("5. call --bam-header refuses before it reads any depth", rc == 4 and err.startswith("REFUSED:"), + (rc, err)) + rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", noalt) + check("5. call --bam-header without ALT contigs goes on (two empty tables: not assessed)", rc == 0, (rc, err)) print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") sys.exit(1 if FAILS else 0) diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 5710bd530..1158f2ef7 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1633,16 +1633,23 @@ def parascopy_section(work, schema): and "not a carrier by copy number" in card_s and "CYP21A2 and the gene-dosage loci have no Parascopy model" in card_s and "step 35 counts them from read depth against flanking sequence" in card_s and "Not checked" not in html, card_s) - # SMN1 g.27134T>G beside the psCN (step 35's _smn1_marker.tsv, bin/traits_report.py smn1_marker) + # SMN1 g.27134T>G beside the psCN (step 35's _smn1_marker.tsv, og cn marker-read SMN1_27134) caveat = render_report.SMN1_MARKER_CAVEAT after_cn = " SMN1 copies: 2 (quality 33, filter PASS)\n SMN1 g.27134T>G silent-carrier marker (rs143838139): " check("parascopy: without step 35's marker table the marker line says not assessed, beside the psCN, with why", (pd.get("smn1_marker") or {}).get("status") == "not assessed" and after_cn + "not assessed\n" in txt and "step 35 wrote no S_smn1_marker.tsv" in txt and f" SMN1 marker caveat: {caveat}\n" in txt, txt) - import traits_report mk = f"{d}/paralogs/S_smn1_marker.tsv" - traits_report.write_metric_table(mk, traits_report.smn1_marker((35, 5, 0), f"{d}/paralogs/S_smn_copy_number.tsv")) + # What og cn marker-read SMN1_27134 writes for 5 of 40 reads with G and this SMN table. + put(mk, "metric\tvalue\nmarker\tSMN1 g.27134T>G\nrsid\trs143838139\nstatus\tpresent\nalt_reads\t5\n" + "depth\t40\nfraction\t0.1250\ncopies\t0.5\nsmn1_pscn\t2 (quality 33, filter PASS)\nagcn\t4\n" + "message\tSMN1 g.27134T>G (rs143838139): present, 5 of 40 reads carry G at the SMN1 and SMN2 copies of the " + "site together (about 0.5 of the 4 SMN1 and SMN2 copies). The reads cannot tell whether a G sits on SMN1 or " + "SMN2. With two SMN1 copies, this marker goes with a higher chance that both sit on one chromosome (2+0, a " + "silent carrier): in Luo 2014 it raised the residual carrier risk of a two-copy result (in African Americans " + "from about 1 in 121 to 1 in 34); the figures depend on ancestry. This marker does not prove 2+0, and its " + "absence does not rule 2+0 out.\n") summ_m, txt_m, html_m = render(d) card_m = html_m.split("

SMN1/SMN2 Copy Number (Parascopy)

", 1)[-1].split("

", 1)[0] check("parascopy: a marker table with 5 of 40 reads reads as present beside the psCN in the text report, " @@ -1669,7 +1676,7 @@ def parascopy_section(work, schema): any(x.startswith("HBA1/HBA2 copy number") for x in summ["not_assessed"]) and "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count" in txt and "HBA1 and HBA2 together" not in txt, summ["not_assessed"]) - # Step 35's HBA table (bin/hba_copy_number.py): four genes, then a planted three. + # Step 35's HBA table (og cn call --locus HBA): four genes, then a planted three. hba_head = "metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t" put(f"{d}/paralogs/S_hba_copy_number.tsv", hba_head + "4\nestimate\t4.02\nhba2_depth\t30.1\n" "hba1_depth\t30.2\nflank_left_depth\t30.0\nflank_right_depth\t30.0\n" diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 9665db09b..cd97bcc7c 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 -"""bin/copy_number.py and bin/cn_loci.tsv: one depth-ratio engine for every -copy-number locus. +"""og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number +locus, and what bin/copy_number.py keeps (the catalog reader and assess(), +which step 41's report imports). + +Without og (OG unset) the checks of the catalog and of assess() run, and the +wiring; with OG naming the og binary (OG=rust/target/release/og, or bin/og) +also those of og cn: the bedcov reader, parity, the CLI, and the blocks of +step 35 cut out and run. With OG_REQUIRED set, a run without og fails. 1. the catalog: every row loads with its columns checked; a broken row (a bad kind, a segment without a span, a class call without an @@ -9,19 +15,19 @@ (main or paralog copy) of a model the pinned Parascopy data holds (assets/parascopy_models_v1.7.txt), except the pairs EXEMPT lists; a planted AMY1C row and a planted row on C4A's copy (C4B) are caught; - 3. parity: the depths of tests/test_hba_copy_number.py and - tests/test_lpa_kiv2.py give, through the engine, the very tables the - code before the engine wrote (GOLD, frozen from it); + 3. parity: made-up HBA and LPA depths give, through og cn call, the very + tables the code before the engine wrote (GOLD, frozen from it); 4. synthetic bedcov tables at 0, 1, 2, 3 and 4 copies of a made-up gene row are each called right; a MAPQ 0 share above the row's ceiling is not assessed; a kind=interval row with pooled depths works the same; 5. a control segment with no read while the other controls are covered raises BrokenControl, and `call` exits 3 naming BROKEN_CONTROL; with no read anywhere it is 'not assessed', not an error; - 6. the CLI: `bed` prints every bedcov locus, `call --json` writes one block; - 7. the wiring: steps 35, 41 and 46 and their Nextflow modules call - copy_number.py, never hba_copy_number.py or lpa_kiv2.py, with the - bedcov options of the catalog row. UGT2B17 has no row: pypgx calls it + 6. the CLI: `bed` prints every bedcov locus, `call --table` writes step + 46's table; og's catalog reader refuses the broken rows the Python + refuses, with the same message; + 7. the wiring: steps 35, 41 and 46 and their Nextflow modules call og cn, + never the old Python, with the bedcov options of the catalog row. UGT2B17 has no row: pypgx calls it in BAM mode (assets/pgx_genes.tsv), and tests/test_pypgx_genes.sh checks that no catalog row names a gene pypgx calls from the BAM; 8. the gene-dosage rows (HP, LCE3BC, CCL3L1): made-up depths at 0 @@ -56,7 +62,7 @@ the same commands; the couple screen counts the deletion and the conversion as CYP21A2 findings and the unmodified table as none. -Run: python3 tests/test_copy_number.py +Run: OG=rust/target/release/og python3 tests/test_copy_number.py """ import json import os @@ -66,17 +72,15 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "copy_number.py") +OG = os.environ.get("OG", "") CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") MODELS = os.path.join(REPO, "assets", "parascopy_models_v1.7.txt") sys.path.insert(0, os.path.join(REPO, "bin")) import copy_number as cn # noqa: E402 -import hba_copy_number as hba # noqa: E402 -import lpa_kiv2 as lpa # noqa: E402 FAILS = 0 # Frozen from bin/hba_copy_number.py and bin/lpa_kiv2.py as they were before the -# engine (main at 6056452): inputs [depths..., MAPQ 0 share] and the dict assess() gave. +# engine (main at 6056452): inputs [depths..., MAPQ 0 share] and the table rows they wrote. GOLD = json.loads(''.join(( '{"hba":[[[30,30,30,30,0.25],{"hba2_depth":30,"hba1_depth":30,"flank_left_depth":30,"flank_right_depth":30,"map' 'q0_share":"0.250","estimate":"4.00","alpha_globin_genes":"4","status":"called","message":"four alpha-globin ge' @@ -192,10 +196,40 @@ def write_catalog(path, rows): f.write("\t".join(r[c] for c in cn.COLUMNS) + "\n") -def bedcov(path, prow, depth): +def bedcov(path, prow, depth, exact=False): + """samtools bedcov over a row's segments at these mean depths: whole sums, + or with exact the sums themselves, so each mean comes back as given.""" with open(path, "w") as f: for c, s, e, n in cn.regions(prow): - f.write(f"{c}\t{s}\t{e}\t{n}\t{round(depth[n] * (e - s))}\n") + v = depth[n] * (e - s) + f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") + + +def og_cn(*args, **kw): + """og cn with these arguments.""" + return subprocess.run([OG, "cn", *args], capture_output=True, text=True, **kw) + + +def table(path): + """{metric: value} of a metricvalue table, {} when it is not there.""" + if not os.path.exists(path): + return {} + return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) + + +# Step 46's table, in its order (what bin/lpa_kiv2.py wrote). +LPA_ORDER = ["status", "kiv2_copies", "ci95_low", "ci95_high", "class", "kiv2_depth", "flank_depth", "mapq0_share", + "message", "advice"] +# Step 35's interval table (what copy_number.py intervals wrote). +INTERVAL_COLUMNS = ["locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", + "reason"] + + +def change(b): + """The change column of an interval row for an assess() block.""" + if b["copies"] is None or b["expected"] is None: + return "." + return "deletion" if b["copies"] < b["expected"] else "duplication" if b["copies"] > b["expected"] else "no change" # The only rows allowed to overlap a Parascopy model, each for a stated reason. @@ -255,6 +289,7 @@ def main(): want = {"low", "typical", "high"} if mode == "class" else {str(i) for i in range(int(len(vals) + 1))} \ if mode == "cut" else None check(f"catalog {loc}: a reading for every call", want is None or want <= set(readings), readings) + broken = [] with tempfile.TemporaryDirectory() as t: p = os.path.join(t, "c.tsv") for desc, kw, err in (("a bad kind", {"kind": "exon"}, "kind"), @@ -272,6 +307,7 @@ def main(): except cn.CatalogError as e: got = str(e) check(f"catalog: {desc} is refused", err in got, got) + broken.append((desc, [row(**kw)], got)) write_catalog(p, [row(), row(locus="TESTH", targets="TESTG:80000-81000", controls="fl_c:1-2;fl_d:3-4")]) try: cn.load_catalog(p) @@ -280,30 +316,6 @@ def main(): got = str(e) check("catalog: one segment name in two loci is refused", "used by two loci" in got, got) - g0 = cn.parse_row(row()) - b = os.path.join(t, "dup.bedcov") - bedcov(b, g0, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0}) - with open(b) as f: - first = f.readline() - with open(b, "a") as f: - f.write(first.rsplit("\t", 1)[0] + "\t0\n") - try: - cn.read_bedcov(b, g0) - got = "read" - except ValueError as e: - got = str(e) - check("bedcov: a region listed twice is refused, not overwritten", "appears twice" in got, got) - with open(b, "w") as f: - f.write(first.split("\t")[0] + "\t50000\t50000\tTESTG\t0\n") - try: - cn.read_bedcov(b, g0) - got = "read" - except ValueError as e: - got = str(e) - except ZeroDivisionError as e: - got = f"ZeroDivisionError: {e}" - check("bedcov: an empty region is refused with its name", "is empty or reversed" in got, got) - # 2. one owner per locus models = {} with open(MODELS) as f: @@ -355,23 +367,6 @@ def main(): check("owner: RHD overlaps RHCE's model only (a paralog copy), the exempt pair", {m for m, _ in overlaps(cat["RHD"], models)} == {"RHCE"}, overlaps(cat["RHD"], models)) - # 3. parity with the code before the engine - for (h2, h1, left, right, share), want in GOLD["hba"]: - m = {"flank_left": left, "HBA2": h2, "HBA1": h1, "flank_right": right} - q = {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in m.items()} - got = hba.assess(m, q) - check(f"parity HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {want['status']} {want['alpha_globin_genes']}", - got == want, (got, want)) - kiv = [n for *_, n in lpa.REGIONS if n.startswith("kiv2_")] - fl = [n for *_, n in lpa.REGIONS if n.startswith("flank_")] - for (n, flank, ks, fs, share), want in GOLD["lpa"]: - k = flank * (n + 2) / 14 - m = {x: k * (1 + (ks if i % 2 == 0 else -ks)) for i, x in enumerate(kiv)} - m.update({x: flank * (1 + (fs if i % 2 == 0 else -fs)) for i, x in enumerate(fl)}) - q = {x: v * (1 - share) if x.startswith("kiv2_") else v for x, v in m.items()} - got = lpa.assess(m, q) - check(f"parity LPA {n} copies at {flank}x: {want['status']} {want['kiv2_copies']}", got == want, (got, want)) - # 4. synthetic copies, the MAPQ 0 ceiling, a kind=interval row g = cn.parse_row(row()) for copies in (0, 1, 2, 3, 4): @@ -430,43 +425,138 @@ def main(): (b["status"], b["reason"]) == ("not assessed", "low_control_depth"), b) b = cn.assess(g, {"fl_a": 0.0, "fl_b": 5.0, "TESTG": 5.0}, {}) check("one flank empty beside a thin one (5x): too little depth, not an error", b["reason"] == "low_control_depth", b) - lrow = cat["LPA_KIV2"] + # 7. the wiring + def read(p): + with open(os.path.join(REPO, p)) as f: + return f.read() + for path, loci in (("scripts/35-paralogs.sh", ["HBA"]), ("modules/local/parascopy/main.nf", ["HBA"]), + ("scripts/41-bloodtype.sh", ["RHD"]), ("modules/local/bloodtype/main.nf", ["RHD"]), + ("scripts/46-lpa.sh", ["LPA_KIV2"]), ("modules/local/lpa/main.nf", ["LPA_KIV2"])): + text = read(path) + for loc in loci: + check(f"{path}: og cn bed --locus {loc}", f"og cn bed --locus {loc}" in text) + if loc != "RHD": # step 41's report imports the engine + check(f"{path}: og cn call --locus {loc}", f"og cn call --locus {loc}" in text) + passes = ["" if p == "." else p for p in cat[loc]["_passes"]] + if loc == "HBA": # and the dosage loop's two passes (section 8), the interval pass (section 9) + passes += ["", "-Q 1", "", ""] # and the CYP21A2 passes (section 10) + passes += ["-Q 1"] + # the commands only: options, then a path, a quoted argument or a variable + calls = [o for o, arg in re.findall(r"samtools bedcov((?: -[A-Za-z](?: \d+)?)*) (\S+)", text) + if re.match(r'["$/]|\S+\.bed\b', arg)] + check(f"{path}: its bedcov passes are the catalog's ({' and '.join(repr(p) for p in passes)})", + sorted(c.strip() for c in calls) == sorted(passes), calls) + check(f"{path}: no call of the old Python", + not re.search(r"(copy_number|hba_copy_number|lpa_kiv2)\.py |traits_report\.py --marker", text)) + + dosage(cat, read) + intervals(cat, read) + psv(cat, read) + if OG: + og_engine(cat, broken) + else: + print("skip: OG is not set (OG=rust/target/release/og): the og cn checks did not run") + + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") + return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 + + +def og_engine(cat, broken): + """Sections 1, 3, 5 and 6 through og cn: its catalog reader, bedcov + reader, the frozen tables, the broken control and the CLI.""" with tempfile.TemporaryDirectory() as t: - a, q, j = (os.path.join(t, x) for x in ("a.tsv", "q.tsv", "o.json")) + p = os.path.join(t, "c.tsv") + for desc, rows, want in broken: + write_catalog(p, rows) + out = og_cn("bed", "--catalog", p) + check(f"og catalog: {desc} is refused with the Python's message", + out.returncode == 2 and out.stderr == f"ERROR: {want}\n", (out.returncode, out.stderr, want)) + + write_catalog(p, [row()]) + g0 = cn.parse_row(row()) + b = os.path.join(t, "dup.bedcov") + bedcov(b, g0, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0}) + with open(b) as f: + first = f.readline() + with open(b, "a") as f: + f.write(first.rsplit("\t", 1)[0] + "\t0\n") + out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) + check("bedcov: a region listed twice is refused, not overwritten", + out.returncode == 2 and "appears twice" in out.stderr, out) + with open(b, "w") as f: + f.write(first.split("\t")[0] + "\t50000\t50000\tTESTG\t0\n") + out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) + check("bedcov: an empty region is refused with its name", + out.returncode == 2 and "is empty or reversed" in out.stderr, out) + + # 3. parity with the code before the engine: every row of the frozen + # table, the depths as numbers + def same(got, want): + def eq(k, v): + if isinstance(v, (int, float)): + try: + return abs(float(got.get(k)) - v) < 1e-9 + except (TypeError, ValueError): + return False + return got.get(k) == v + return all(eq(k, v) for k, v in want.items()) + + a, q, tab = (os.path.join(t, x) for x in ("a.tsv", "q.tsv", "t.tsv")) + hrow = cat["HBA"] + for (h2, h1, left, right, share), want in GOLD["hba"]: + m = {"flank_left": left, "HBA2": h2, "HBA1": h1, "flank_right": right} + bedcov(a, hrow, m, exact=True) + bedcov(q, hrow, {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in m.items()}, exact=True) + out = og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", a, "--bedcov-mapq1", q, + "--table", tab) + got = table(tab) + check(f"parity HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {want['status']} " + f"{want['alpha_globin_genes']}", out.returncode == 0 and same(got, want), (out.stderr, got, want)) + lrow = cat["LPA_KIV2"] + # in catalog order, as bin/lpa_kiv2.py's REGIONS listed them + kiv = [n for n, *_ in lrow["_targets"]] + fl = [n for n, *_ in lrow["_controls"]] + for (n, flank, ks, fs, share), want in GOLD["lpa"]: + k = flank * (n + 2) / 14 + m = {x: k * (1 + (ks if i % 2 == 0 else -ks)) for i, x in enumerate(kiv)} + m.update({x: flank * (1 + (fs if i % 2 == 0 else -fs)) for i, x in enumerate(fl)}) + bedcov(a, lrow, m, exact=True) + bedcov(q, lrow, {x: v * (1 - share) if x.startswith("kiv2_") else v for x, v in m.items()}, exact=True) + out = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, + "--table", tab) + got = table(tab) + check(f"parity LPA {n} copies at {flank}x: {want['status']} {want['kiv2_copies']}", + out.returncode == 0 and same(got, want), (out.stderr, got, want)) + + # 5. a broken control + os.remove(tab) depth = {n: 30.0 for *_, n in cn.regions(lrow)} depth["flank_e19"] = 0.0 bedcov(a, lrow, depth) bedcov(q, lrow, depth) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, - "--bedcov-mapq1", q, "--json", j], capture_output=True, text=True) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) check("broken control: `call` exits 3 with BROKEN_CONTROL and writes nothing", out.returncode == 3 and "BROKEN_CONTROL: control segment flank_e19 of LPA_KIV2" in out.stderr - and not os.path.exists(j), out) + and not os.path.exists(tab), out) # 6. the CLI - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed") names = [x.split("\t")[3] for x in out.stdout.splitlines()] want = sum(len(cn.regions(r)) for r in cat.values() if r["input"] == "bedcov") - check("bed: every segment of the bedcov loci, once", out.returncode == 0 and len(names) == len(set(names)) == want, - (out.returncode, len(names), want)) + check("bed: every segment of the bedcov loci, once (the catalog beside og)", + out.returncode == 0 and len(names) == len(set(names)) == want, (out.returncode, out.stderr, len(names), want)) depth = {n: 30.0 for *_, n in cn.regions(lrow)} for n in depth: if n.startswith("kiv2_"): depth[n] = 30.0 * 22 / 14 bedcov(a, lrow, depth) bedcov(q, lrow, {n: v * (0.1 if n.startswith("kiv2_") else 1) for n, v in depth.items()}) - tab = os.path.join(t, "t.tsv") - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, - "--bedcov-mapq1", q, "--json", j, "--table", tab], capture_output=True, text=True) - block = json.load(open(j)) if os.path.exists(j) else {} - check("call --json: one block, 20 KIV-2 repeats, called", - out.returncode == 0 and block.get("locus") == "LPA_KIV2" and block.get("status") == "called" - and abs(block.get("estimate", 0) - 20) < 0.01, (out, block)) - tab_rows = dict(x.split("\t", 1) for x in open(tab).read().splitlines()[1:]) if os.path.exists(tab) else {} - check("call --table: step 46's table", tab_rows.get("kiv2_copies") == "20.0" and list(tab_rows) == list(lpa.ORDER), - tab_rows) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", "LPA_KIV2", "--bedcov", a, "--json", j], - capture_output=True, text=True) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) + tab_rows = table(tab) + check("call --table: step 46's table, 20 KIV-2 repeats, in its order", + out.returncode == 0 and tab_rows.get("status") == "estimated" and tab_rows.get("kiv2_copies") == "20.0" + and list(tab_rows) == LPA_ORDER, (out, tab_rows)) + out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a) check("call: a row with a MAPQ 0 guard needs --bedcov-mapq1", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) empty = os.path.join(t, "empty.tsv") @@ -474,44 +564,12 @@ def main(): for loc, key in (("HBA", "alpha_globin_genes"), ("LPA_KIV2", "kiv2_copies")): if os.path.exists(tab): os.remove(tab) - out = subprocess.run([sys.executable, SCRIPT, "call", "--locus", loc, "--bedcov", empty, - "--bedcov-mapq1", empty, "--table", tab], capture_output=True, text=True) - tab_rows = dict(x.split("\t", 1) for x in open(tab).read().splitlines()[1:]) if os.path.exists(tab) else {} + out = og_cn("call", "--locus", loc, "--bedcov", empty, "--bedcov-mapq1", empty, "--table", tab) + tab_rows = table(tab) check(f"call {loc} on two empty tables (no reads on the contig): exit 0, not assessed", out.returncode == 0 and tab_rows.get("status") == "not assessed" and tab_rows.get(key) == "." and "no chr" in tab_rows.get("message", ""), (out, tab_rows)) - # 7. the wiring - def read(p): - with open(os.path.join(REPO, p)) as f: - return f.read() - for path, loci in (("scripts/35-paralogs.sh", ["HBA"]), ("modules/local/parascopy/main.nf", ["HBA"]), - ("scripts/41-bloodtype.sh", ["RHD"]), ("modules/local/bloodtype/main.nf", ["RHD"]), - ("scripts/46-lpa.sh", ["LPA_KIV2"]), ("modules/local/lpa/main.nf", ["LPA_KIV2"])): - text = read(path) - for loc in loci: - check(f"{path}: copy_number.py bed --locus {loc}", f"copy_number.py bed --locus {loc}" in text) - if loc != "RHD": # step 41's report imports the engine - check(f"{path}: copy_number.py call --locus {loc}", f"copy_number.py call --locus {loc}" in text) - passes = ["" if p == "." else p for p in cat[loc]["_passes"]] - if loc == "HBA": # and the dosage loop's two passes (section 8), the interval pass (section 9) - passes += ["", "-Q 1", "", ""] # and the CYP21A2 passes (section 10) - passes += ["-Q 1"] - # the commands only: options, then a path, a quoted argument or a variable - calls = [o for o, arg in re.findall(r"samtools bedcov((?: -[A-Za-z](?: \d+)?)*) (\S+)", text) - if re.match(r'["$/]|\S+\.bed\b', arg)] - check(f"{path}: its bedcov passes are the catalog's ({' and '.join(repr(p) for p in passes)})", - sorted(c.strip() for c in calls) == sorted(passes), calls) - check(f"{path}: no call of the old per-gene scripts", - not re.search(r"(hba_copy_number|lpa_kiv2)\.py (bed|check|estimate)", text)) - - dosage(cat, read) - intervals(cat, read) - psv(cat, read) - - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - def dosage_depths(r, copies, q1_share=0.5): """Mean depths of a dosage row at a copy number: controls at 30x (two @@ -522,14 +580,14 @@ def dosage_depths(r, copies, q1_share=0.5): return m, {n: v * q1_share if n in tg else v for n, v in m.items()} -# run_in and cpath for the dosage block of step 35: copy_number.py runs for -# real, samtools bedcov prints tests/fixtures/cn/[.q1].bedcov.tsv. +# run_in and cpath for the dosage block of step 35: og cn runs for real (OG), +# samtools bedcov prints tests/fixtures/cn/[.q1].bedcov.tsv. STUB = r""" run_in() { while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") shift 2; q=""; if [ "$1" = -Q ]; then q=.q1; shift 2; fi d=$(dirname "$1"); cat "${FIX}/${d##*/}${q}.bedcov.tsv" ;; *) return 1 ;; @@ -549,8 +607,8 @@ def dosage(cat, read): m and m.group(1)) for path, text, q in (("scripts/35-paralogs.sh", step, '"'), ("modules/local/parascopy/main.nf", module, '"\\')): for cmd in ("bed", "call"): - check(f"{path}: copy_number.py {cmd} --locus on each dosage row", - f"copy_number.py {cmd} --locus {q}$loc" in text) + check(f"{path}: og cn {cmd} --locus on each dosage row", + f"og cn {cmd} --locus {q}$loc" in text) for loc in DOSAGE: r = cat[loc] check(f"dosage {loc}: two bedcov passes, all reads then MAPQ >= 1 (what the loop runs)", @@ -580,6 +638,8 @@ def dosage(cat, read): # The block of step 35, cut out and run on the fixtures. blk = re.search(r'\n(DOSAGE_LOCI=.*?\nrm -rf "\$DOSAGE_WORK"\n)', step, re.S) check("step 35 holds the dosage block", blk is not None) + if not OG: + return fix = os.path.join(REPO, "tests", "fixtures", "cn") with tempfile.TemporaryDirectory() as t: env = dict(os.environ, REPO=REPO, FIX=fix, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", @@ -601,16 +661,16 @@ def dosage(cat, read): got[1:3] == ["called", str(n)] and got[5:] == [cn.reading(cat[loc], n)], got) # A row shifted by 1 kb no longer matches the fixture made for its place. - for loc in DOSAGE: - r = dict(cat[loc]) - r["targets"] = ";".join(f"{n}:{s + 1000}-{e + 1000}" for n, s, e, _ in r["_targets"]) - shifted = cn.parse_row({k: v for k, v in r.items() if not k.startswith("_")}) - try: - cn.read_bedcov(os.path.join(fix, f"{loc}.bedcov.tsv"), shifted) - got = "read" - except ValueError as e: - got = str(e) - check(f"dosage {loc}: shifted 1 kb, its fixture is refused", "regions differ" in got, got) + with tempfile.TemporaryDirectory() as t: + p = os.path.join(t, "c.tsv") + for loc in DOSAGE: + r = {k: v for k, v in cat[loc].items() if not k.startswith("_")} + r["targets"] = ";".join(f"{n}:{s + 1000}-{e + 1000}" for n, s, e, _ in cat[loc]["_targets"]) + write_catalog(p, [r]) + f = os.path.join(fix, f"{loc}.bedcov.tsv") + out = og_cn("call", "--catalog", p, "--locus", loc, "--bedcov", f, "--bedcov-mapq1", f) + check(f"dosage {loc}: shifted 1 kb, its fixture is refused", + out.returncode == 2 and "regions differ" in out.stderr, out) # Section 9: the interval rows. @@ -630,7 +690,7 @@ def dosage(cat, read): while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") shift 2; python3 "${FAKECOV}" "$1" ;; *) return 1 ;; esac @@ -709,12 +769,11 @@ def intervals(cat, read): not hits, hits) a_d, p17 = cat["22q11.2_DGS_proximal_A-D"], cat["17p12_HNPP_CMT1A"] - for r, target, want, change in ((a_d, 15.0, 1, "deletion"), (p17, 45.0, 3, "duplication"), + for r, target, want, change_ in ((a_d, 15.0, 1, "deletion"), (p17, 45.0, 3, "duplication"), (a_d, 30.0, 2, "no change"), (p17, 30.0, 2, "no change")): b = cn.assess(r, depths(r, target)) - line = cn.interval_line(r, b) - check(f"interval {r['locus']}: targets at {target}x over 30x controls is {want} copies, {change}", - (b["status"], b["copies"], line[8]) == ("called", want, change), (b, line)) + check(f"interval {r['locus']}: targets at {target}x over 30x controls is {want} copies, {change_}", + (b["status"], b["copies"], change(b)) == ("called", want, change_), b) az = cat["AZFc_DAZ"] m = depths(az, 15.0 / 4, 15.0) check("AZFc on a male: four DAZ copies at a single copy's depth in all is 4 copies", @@ -723,12 +782,12 @@ def intervals(cat, read): half = {n: (7.5 if n in {x for x, *_ in az["_targets"]} else 15.0) for n in m} b = cn.assess(az, half, None, "male") check("AZFc on a male at half its DAZ depth: 2 copies, a deletion (gr/gr or b2/b3)", - (b["copies"], cn.interval_line(az, b)[8]) == (2, "deletion"), b) + (b["copies"], change(b)) == (2, "deletion"), b) b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 15.0, 15.0), None, "male") check("AZFa on a male: one copy, no change", (b["copies"], b["expected"]) == (1, 1), b) b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 0.0, 15.0), None, "male") check("AZFa on a male with no read over it: 0 copies, a deletion", - (b["copies"], cn.interval_line(cat["AZFa"], b)[8]) == (0, "deletion"), b) + (b["copies"], change(b)) == (0, "deletion"), b) for loc in ("AZFa", "AZFb", "AZFc_DAZ"): b = cn.assess(cat[loc], depths(cat[loc], 15.0, 15.0), None, "female") check(f"{loc} on a female: not applicable", (b["status"], b["reason"]) == ("not applicable", "no_y"), b) @@ -740,12 +799,20 @@ def intervals(cat, read): b = cn.assess(xp, depths(xp, 0.0, 15.0), None, "male") check("Xp22.31 on a male with no read there: 0 copies, a deletion (the STS deletion)", b["copies"] == 0, b) + step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") + for path, text in (("scripts/35-paralogs.sh", step), ("modules/local/parascopy/main.nf", module)): + for cmd in ("og cn bed --kind interval", "og cn intervals --bedcov", "og cn intervals --failed"): + check(f"{path}: {cmd}", cmd in text) + check("PARASCOPY module: the samplesheet sex goes to the interval rows", + "def sex = meta.sex in ['male', 'female'] ? meta.sex : 'unknown'" in module and "--sex ${sex}" in module) + if not OG: + return with tempfile.TemporaryDirectory() as t: fake = os.path.join(t, "fakecov.py") with open(fake, "w") as f: f.write(FAKECOV) bed = os.path.join(t, "r.bed") - out = subprocess.run([sys.executable, SCRIPT, "bed", "--kind", "interval"], capture_output=True, text=True) + out = og_cn("bed", "--kind", "interval") open(bed, "w").write(out.stdout) check("bed --kind interval: every segment of every interval row, once", out.returncode == 0 and len(out.stdout.splitlines()) == sum(len(cn.regions(r)) for r in iv), out.stderr) @@ -755,12 +822,11 @@ def intervals(cat, read): with open(cov, "w") as f: subprocess.run([sys.executable, fake, bed], stdout=f, env=env, check=True) tab = os.path.join(t, "iv.tsv") - out = subprocess.run([sys.executable, SCRIPT, "intervals", "--bedcov", cov, "--sex", "female", "--out", tab], - capture_output=True, text=True) + out = og_cn("intervals", "--bedcov", cov, "--sex", "female", "--out", tab) lines = open(tab).read().splitlines() if os.path.exists(tab) else [] rows = {x.split("\t")[0]: x.split("\t") for x in lines[1:]} check("intervals: exit 0, the header, one line per interval row", - out.returncode == 0 and lines[:1] == ["\t".join(cn.INTERVAL_COLUMNS)] and list(rows) == names, + out.returncode == 0 and lines[:1] == ["\t".join(INTERVAL_COLUMNS)] and list(rows) == names, (out.returncode, out.stderr[-400:], lines[:1])) check("intervals: the 22q11.2 A-D deletion at 1 copy", rows.get(a_d["locus"], [""] * 11)[5:9] == ["called", "1", "2", "deletion"], rows.get(a_d["locus"])) @@ -769,12 +835,11 @@ def intervals(cat, read): and rows.get("3q29", [""] * 11)[5:9] == ["called", "2", "2", "no change"], rows.get(p17["locus"])) check("intervals: chrY rows on a female are not applicable", all(rows.get(x, [""] * 11)[5] == "not applicable" for x in ("AZFa", "AZFb", "AZFc_DAZ"))) - out = subprocess.run([sys.executable, SCRIPT, "intervals", "--failed", "--out", tab], capture_output=True, text=True) + out = og_cn("intervals", "--failed", "--out", tab) lines = open(tab).read().splitlines()[1:] check("intervals --failed: every row not assessed, check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines)) - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") blk = re.search(r'\n(INTERVAL_OUT=.*?\nrm -rf "\$INTERVAL_WORK"\n)', step, re.S) check("step 35 holds the interval block", blk is not None) env = dict(os.environ, REPO=REPO, FAKECOV=fake, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", @@ -799,10 +864,6 @@ def intervals(cat, read): check("step 35 interval block with a failed bedcov: exit 0, every row check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines), (out.returncode, out.stderr[-400:])) - for path, text in (("scripts/35-paralogs.sh", step), ("modules/local/parascopy/main.nf", module)): - for cmd in ("copy_number.py bed --kind interval", "copy_number.py intervals --bedcov", - "copy_number.py intervals --failed"): - check(f"{path}: {cmd}", cmd in text) # collect_summary: step 35's section carries the interval rows with a change import collect_summary as cs sd = os.path.join(t, "sample") @@ -821,8 +882,6 @@ def intervals(cat, read): ci = (data or {}).get("cn_intervals") or {} check("collect_summary: the interval rows stay when step 35 wrote no other table", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) - check("PARASCOPY module: the samplesheet sex goes to the interval rows", - "def sex = meta.sex in ['male', 'female'] ? meta.sex : 'unknown'" in module and "--sex ${sex}" in module) # Section 10: the CYP21A2 row, call psv. @@ -866,26 +925,42 @@ def cyp_fixture(r, total, a2): def cyp_call(r, bedcov_text, pileup_text, mapq0=0.1): - """The block of a CYP21A2 genome; mapq0 is the share of every segment's + """og cn call on a CYP21A2 genome; mapq0 is the share of every segment's depth from MAPQ 0 reads (0.1 as HG002's BWA-MEM and minimap2 reads give - over the two targets; 0 for an aligner that leaves those reads out).""" + over the two targets; 0 for an aligner that leaves those reads out). The + table read back: status, copies, module_total, finding and reason (None + for '.'), reading, conversion_sites, each site's copies estimate and + ratio, and the table and stdout line themselves.""" with tempfile.TemporaryDirectory() as t: - b, p = os.path.join(t, "b.tsv"), os.path.join(t, "p.tsv") + b, q, p, tab = (os.path.join(t, x) for x in ("b.tsv", "q.tsv", "p.tsv", "t.tsv")) open(b, "w").write(bedcov_text) + open(q, "w").write("".join( + "\t".join(c[:4] + [repr(float(c[4]) * (1 - mapq0))]) + "\n" + for c in (ln.split("\t") for ln in bedcov_text.splitlines()))) open(p, "w").write(pileup_text) - m = cn.read_bedcov(b, r) - q1 = {n: v * (1 - mapq0) for n, v in m.items()} - return cn.psv_assess(r, cn.assess(r, m, q1), cn.read_pileup(p, r)) - - -# run_in and cpath for the CYP21A2 block of step 35: copy_number.py runs for real, + out = og_cn("call", "--catalog", CATALOG, "--locus", "CYP21A2", "--bedcov", b, "--bedcov-mapq1", q, + "--pileup", p, "--table", tab) + rows = table(tab) + val = (lambda k, f=str: None if rows.get(k, ".") == "." else f(rows[k])) + sites = {} + for k, v in rows.items(): + m = re.search(r"copies \S+ \(([0-9]+\.[0-9]+)\); ratio ([0-9]+\.[0-9]+)", v) if k.startswith("psv_") else None + if m: + sites[k[4:]] = {"estimate": float(m.group(1)), "ratio": float(m.group(2))} + return {"status": rows.get("status"), "copies": val("cyp21a2_copies", int), "module_total": val("module_total", int), + "finding": val("finding"), "reason": val("reason"), "reading": rows.get("reading", ""), + "conversion_sites": (val("conversion_sites") or "").split(",") if val("conversion_sites") else [], + "sites": sites, "table": rows, "line": out.stdout.strip(), "exit": out.returncode} + + +# run_in and cpath for the CYP21A2 block of step 35: og cn runs for real (OG), # samtools bedcov and mpileup print tests/fixtures/cn/CYP21A2_${CYP_CASE}.*. STUB_CYP = r""" run_in() { while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done shift case "$1 $2" in - "python3 /pgp-bin/copy_number.py") shift 2; python3 "${REPO}/bin/copy_number.py" "$@" ;; + "/pgp-bin/og cn") shift; "${OG}" "$@" ;; "samtools bedcov") # the MAPQ >= 1 pass: 90% of each depth (HG002's share), or all of it for # an aligner that left the MAPQ 0 reads out (CYP_NO_MAPQ0) @@ -938,16 +1013,29 @@ def psv(cat, read): got = str(e) check(f"CYP21A2: {desc} is refused", err in got, got) - got = cn.count_bases("^]C.,c$+2AGg-1tG*<>N", "C") - check("pileup: read starts, ends and indels are not bases; '.' and ',' are the reference base", - got == {"A": 0, "C": 4, "G": 2, "T": 0}, got) + step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") + for path, text, q in (("scripts/35-paralogs.sh", step, True), ("modules/local/parascopy/main.nf", module, False)): + for cmd in ("og cn bed --locus CYP21A2", "og cn bed --psv --locus CYP21A2", + "--bedcov-mapq1", "cyp21a2_bedcov_q1.tsv" if not q else 'bedcov_q1.tsv")"', + "samtools mpileup -A -B -q 0 -Q 13 -d 100000", "og cn call --locus CYP21A2 --bedcov", + "--pileup", "_cyp21a2_psv.tsv"): + check(f"{path}: {cmd}", cmd in text) + check("PARASCOPY module: its CYP21A2 table is an output", 'path("${meta.id}_cyp21a2_psv.tsv"), emit: cyp21a2' + in module) + check("CYP21A2 row: a MAPQ 0 guard (two bedcov passes, all reads and MAPQ >= 1; minimum share 0.02)", + r["_passes"] == [".", "-Q 1"] and r["_mapq0_min"] == 0.02 and r["_mapq0_max"] is None, + (r["_passes"], r["_mapq0_min"], r["_mapq0_max"])) + if not OG: + return - out = subprocess.run([sys.executable, SCRIPT, "bed", "--psv", "--locus", "CYP21A2"], capture_output=True, text=True) + # og's pileup reader (read starts, ends and indels are not bases) is + # checked by the crate's own test and by the parity cases. + out = og_cn("bed", "--psv", "--locus", "CYP21A2") lines = out.stdout.splitlines() check("bed --psv: twelve one-base positions (each site on CYP21A2 and on CYP21A1P), sorted", out.returncode == 0 and len(lines) == 12 and all(int(x.split("\t")[2]) - int(x.split("\t")[1]) == 1 for x in lines) and [int(x.split("\t")[1]) for x in lines] == sorted(int(x.split("\t")[1]) for x in lines), out) - out = subprocess.run([sys.executable, SCRIPT, "bed", "--psv", "--locus", "HBA"], capture_output=True, text=True) + out = og_cn("bed", "--psv", "--locus", "HBA") check("bed --psv on a row without PSV sites is refused", out.returncode == 2 and "no psv locus" in out.stderr, out) # The committed fixtures are cyp_fixture's, so a change to either shows. @@ -1008,9 +1096,6 @@ def psv(cat, read): # The MAPQ 0 guard: an aligner that left out the reads fitting both copies lowers # the module total, so the deletion's tables from such a BAM (no MAPQ 0 read) read # not assessed, not a deletion; with HG002's 10% share they still give the deletion. - check("CYP21A2 row: a MAPQ 0 guard (two bedcov passes, all reads and MAPQ >= 1; minimum share 0.02)", - r["_passes"] == [".", "-Q 1"] and r["_mapq0_min"] == 0.02 and r["_mapq0_max"] is None, - (r["_passes"], r["_mapq0_min"], r["_mapq0_max"])) total, a2 = CYP_CASES["deletion"] bed, pile = cyp_fixture(r, total, a2) for share, want in ((0.0, ("not assessed", "low_mapq0_share", None, None)), @@ -1020,19 +1105,17 @@ def psv(cat, read): check(f"CYP21A2: the deletion's tables with a MAPQ 0 share of {share}: {want[0]} ({want[1] or want[2]})", (b["status"], b["reason"], b["finding"], b["copies"]) == want, b) b = cyp_call(r, bed, pile, 0.0) - rows, line = cn.psv_table(r, b) - tab = dict(rows) + tab, line = b["table"], b["line"] check("CYP21A2 without MAPQ 0 reads: the table says so (message, share 0.000), no copies, module total or finding", tab["message"].startswith("not assessed: the BAM has few or no multi-mapping reads here") and "CYP21A2 and CYP21A1P" in tab["message"] and tab["mapq0_share"] == "0.000" and (tab["cyp21a2_copies"], tab["module_total"], tab["finding"]) == (".", ".", ".") and "low_mapq0_share" in line, (tab, line)) bed, pile = cyp_fixture(r, 4, {}) - tab = dict(cn.psv_table(r, cyp_call(r, bed, pile))[0]) + tab = cyp_call(r, bed, pile)["table"] check("CYP21A2 unmodified with HG002's MAPQ 0 share: called, share 0.100, message '.'", (tab["status"], tab["finding"], tab["mapq0_share"], tab["message"]) == ("called", "none", "0.100", "."), tab) - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") blk = re.search(r'\n(CYP21_OUT=.*?\nrm -rf "\$CYP21_WORK"\n)', step, re.S) check("step 35 holds the CYP21A2 block", blk is not None) tabs = {} @@ -1071,14 +1154,6 @@ def psv(cat, read): (nl.get("mapq0_share"), nl.get("message")) == ("0.100", "."), nl) check("step 35 CYP21A2 block with a failed mpileup: the rows of a called table but the sites'", list(rd("fail")) == [k for k in nl if not k.startswith("psv_")], (list(rd("fail")), list(nl))) - for path, text, q in (("scripts/35-paralogs.sh", step, True), ("modules/local/parascopy/main.nf", module, False)): - for cmd in ("copy_number.py bed --locus CYP21A2", "copy_number.py bed --psv --locus CYP21A2", - "--bedcov-mapq1", "cyp21a2_bedcov_q1.tsv" if not q else 'bedcov_q1.tsv")"', - "samtools mpileup -A -B -q 0 -Q 13 -d 100000", "copy_number.py call --locus CYP21A2 --bedcov", - "--pileup", "_cyp21a2_psv.tsv"): - check(f"{path}: {cmd}", cmd in text) - check("PARASCOPY module: its CYP21A2 table is an output", 'path("${meta.id}_cyp21a2_psv.tsv"), emit: cyp21a2' - in module) # the couple screen genes = {"CYP21A2": {"gene": "CYP21A2"}} diff --git a/tests/test_hba_copy_number.py b/tests/test_hba_copy_number.py index 973821e72..a31afb5cd 100755 --- a/tests/test_hba_copy_number.py +++ b/tests/test_hba_copy_number.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -"""bin/hba_copy_number.py: step 35's HBA1/HBA2 count from short-read depth. +"""og cn call --locus HBA: step 35's HBA1/HBA2 count from short-read depth. +It needs og: OG names the binary (OG=rust/target/release/og, or bin/og); +without it the test is skipped, and with OG_REQUIRED set too it fails. On `samtools bedcov` tables made up for each case (summed depth = mean depth x length), at 30x in the flanks, with a quarter of the gene-body depth from @@ -17,11 +19,12 @@ 15.5x, HBA1 25.7x, flanks 30x and 31x, which main counted as three or left between two and three) is 'not assessed', naming the cause; 1% is too, 3% is called; no gene at all (no reads to judge by) is still 0; - 7. the CLI: `bed` prints the four regions, `check` writes the metric table - and exits 0; a table with other regions, or a short row, in either - input exits 2, and so does a missing --bedcov-mapq1. + 7. the CLI: `og cn bed --locus HBA` prints the four regions, `og cn call + --table` writes the metric table and exits 0; a table with other + regions, or a short row, in either input exits 2, and so does a missing + --bedcov-mapq1. -Run: python3 tests/test_hba_copy_number.py +Run: OG=rust/target/release/og python3 tests/test_hba_copy_number.py """ import os import subprocess @@ -29,10 +32,15 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "hba_copy_number.py") +CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import hba_copy_number as hba # noqa: E402 +import copy_number as cn # noqa: E402 +REGIONS = cn.regions(cn.load_catalog(CATALOG)["HBA"]) +# Step 35's HBA table, in its order (what bin/hba_copy_number.py wrote). +ORDER = ["status", "alpha_globin_genes", "estimate", "hba2_depth", "hba1_depth", "flank_left_depth", + "flank_right_depth", "mapq0_share", "message"] FAILS = 0 @@ -46,7 +54,7 @@ def check(desc, ok, detail=""): def bedcov(path, depth): """A bedcov table with the given mean depth per region name.""" with open(path, "w") as f: - for c, s, e, n in hba.REGIONS: + for c, s, e, n in REGIONS: f.write(f"{c}\t{s}\t{e}\t{n}\t{int(depth[n] * (e - s))}\n") @@ -56,12 +64,18 @@ def mapq1(depth, share=0.25): return {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in depth.items()} +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + def run(depth, share=0.25): + """Step 35's HBA table for these depths, as og cn call writes it.""" with tempfile.TemporaryDirectory() as t: - p, q = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv") + p, q, o = (os.path.join(t, x) for x in ("cov.tsv", "cov_q1.tsv", "hba.tsv")) bedcov(p, depth) bedcov(q, mapq1(depth, share)) - return hba.assess(hba.read_bedcov(p), hba.read_bedcov(q)) + og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", p, "--bedcov-mapq1", q, "--table", o) + return dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) if os.path.exists(o) else {} def depths(hba2, hba1, left=30.0, right=30.0): @@ -69,6 +83,9 @@ def depths(hba2, hba1, left=30.0, right=30.0): def main(): + if not OG: + print("skip: OG is not set (OG=rust/target/release/og)") + return 1 if os.environ.get("OG_REQUIRED") else 0 r = run(depths(30, 30)) check("no deletion: four alpha-globin genes, called", (r["status"], r["alpha_globin_genes"], r["estimate"]) == ("called", "4", "4.00") @@ -113,33 +130,45 @@ def main(): (r["status"], r["alpha_globin_genes"]) == ("called", "0") and "Bart" in r["message"], r) with tempfile.TemporaryDirectory() as t: - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed", "--catalog", CATALOG, "--locus", "HBA") rows = [line.split("\t") for line in out.stdout.splitlines()] check("bed: four regions, the two genes between the flanks", out.returncode == 0 and [x[3] for x in rows] == ["flank_left", "HBA2", "HBA1", "flank_right"], out) cov, covq, res = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv"), os.path.join(t, "hba.tsv") bedcov(cov, depths(15, 30)) bedcov(covq, mapq1(depths(15, 30))) - cli = [sys.executable, SCRIPT, "check", "--bedcov", cov, "--bedcov-mapq1", covq, "--out", res] + cli = [OG, "cn", "call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", cov, "--bedcov-mapq1", covq, + "--table", res] out = subprocess.run(cli, capture_output=True, text=True) table = dict(line.split("\t", 1) for line in open(res).read().splitlines()[1:]) if os.path.exists(res) else {} - check("check: exit 0, the table has the status, the count and the message", + check("call: exit 0, the table has the status, the count and the message", out.returncode == 0 and table.get("status") == "called" and table.get("alpha_globin_genes") == "3" and table.get("mapq0_share") == "0.250" - and list(table) == list(hba.ORDER) and "HBA1/HBA2: three" in out.stdout, (out, table)) - out = subprocess.run(cli[:4] + cli[6:], capture_output=True, text=True) - check("check: without --bedcov-mapq1 exits 2", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) + and list(table) == ORDER and "HBA1/HBA2: three" in out.stdout, (out, table)) + out = subprocess.run(cli[:9] + cli[11:], capture_output=True, text=True) + check("call: without --bedcov-mapq1 exits 2", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, out) for name, path in (("all reads", cov), ("MAPQ 1 or more", covq)): bedcov(cov, depths(15, 30)) bedcov(covq, mapq1(depths(15, 30))) with open(path, "a") as f: f.write("chr16\t1\t100\tother\t500\n") out = subprocess.run(cli, capture_output=True, text=True) - check(f"check: a table ({name}) with other regions exits 2", out.returncode == 2 and "differ" in out.stderr, out) + check(f"call: a table ({name}) with other regions exits 2", out.returncode == 2 and "differ" in out.stderr, out) with open(path, "w") as f: f.write("chr16\t100000\t140000\n") out = subprocess.run(cli, capture_output=True, text=True) - check(f"check: a short row ({name}) exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + check(f"call: a short row ({name}) exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + + # The demo's HBA table (tests/demo/make_demo_sample.py) is what og writes. + sys.path.insert(0, os.path.join(REPO, "tests", "demo")) + import make_demo_sample as demo + with tempfile.TemporaryDirectory() as t: + cov, covq, res = os.path.join(t, "cov.tsv"), os.path.join(t, "cov_q1.tsv"), os.path.join(t, "hba.tsv") + bedcov(cov, demo.HBA_DEPTH) + bedcov(covq, {n: v * demo.HBA_Q1[n] for n, v in demo.HBA_DEPTH.items()}) + og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", cov, "--bedcov-mapq1", covq, "--table", res) + got = open(res).read() if os.path.exists(res) else "" + check("the demo's HBA table is the one og cn call writes for its depths", got == demo.HBA_TABLE, got) print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") return 1 if FAILS else 0 diff --git a/tests/test_lpa_fallback.sh b/tests/test_lpa_fallback.sh index acb0f7e2f..c16dc7e53 100755 --- a/tests/test_lpa_fallback.sh +++ b/tests/test_lpa_fallback.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# Step 46 [4/4]: a KIV-2 4925G>A reader that crashes never costs the KIV-2 -# count. scripts/46-lpa.sh then writes lpa/_lpa_kiv2_4925.tsv saying +# Step 46 [3/3]: a KIV-2 4925G>A reader that crashes never costs the KIV-2 +# count (one og cn call reads both; it writes the count table first). scripts/46-lpa.sh then writes lpa/_lpa_kiv2_4925.tsv saying # 'not assessed', exits 0 and keeps lpa/_lpa_kiv2.tsv; the LPA Nextflow # process (modules/local/lpa/main.nf) writes the same fallback table, byte for # byte. The e2e case genotype-rules-2-lpa-null runs the module's block with a # crashing reader in its image. # -# A fake `docker` on PATH stands in for the containers: the count call writes -# a table, the 4925G>A call exits 1 (or 0 for the control, writing a -# 'present' table), everything else succeeds. +# A fake `docker` on PATH stands in for the containers: the og cn call writes +# the count table, then exits 1 at --marker-out (or 0 for the control, +# writing a 'present' table), everything else succeeds. # # Controls: with the reader working the table is the reader's ('present'), so # the not-assessed check can fail; with the fallback removed from a copy of @@ -97,7 +97,7 @@ mkdir -p "${WORK}/scripts/lib" cp "${REPO}/scripts/lib/"* "${WORK}/scripts/lib/" cp "${REPO}/versions.env" "${WORK}/" 2>/dev/null || true # shellcheck disable=SC2016 # the $(cpath ...) is text of the step, not to expand here -sed -e 's/^if ! run_in -v /run_in -v /' -e 's/--marker-out "$(cpath "$NULL")"; then$/--marker-out "$(cpath "$NULL")"\nif false; then/' \ +sed -e 's/--marker-out "$(cpath "$NULL")" || rc=\$?$/--marker-out "$(cpath "$NULL")"/' \ "${REPO}/scripts/46-lpa.sh" > "${WORK}/scripts/46-lpa.sh" bash -n "${WORK}/scripts/46-lpa.sh" || fail "control: the copy of step 46 without the fallback is not valid bash" if cmp -s "${REPO}/scripts/46-lpa.sh" "${WORK}/scripts/46-lpa.sh"; then diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index a60ed5fba..1cbf1065e 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 -"""bin/lpa_kiv2.py: step 46's LPA KIV-2 repeat count from short-read depth, -and its Lp(a) section in both reports. +"""og cn call --locus LPA_KIV2: step 46's LPA KIV-2 repeat count from +short-read depth, with KIV-2 4925G>A beside it, and its Lp(a) section in both +reports. Part 6 runs without og; the rest needs it: OG names the binary +(OG=rust/target/release/og, or bin/og); without it they are skipped, and with +OG_REQUIRED set too the test fails. On `samtools bedcov` tables made up for each case (summed depth = mean depth x exon length), 30x over the single-copy LPA exons, with 90% of the KIV-2 @@ -19,9 +22,10 @@ count; no read of MAPQ 0 over KIV-2 (novoalign's default) is 'not assessed', naming the cause; two empty tables (a BAM with no chr6) is 'not assessed'; - 5. the CLI: `bed` prints the 29 regions, 14 KIV-2 exons of 2,394 bp and 15 - single-copy exons; `estimate` writes the metric table and exits 0; a - table with other regions or a short row exits 2; + 5. the CLI: `og cn bed --locus LPA_KIV2` prints the 29 regions, 14 KIV-2 + exons of 2,394 bp and 15 single-copy exons; `og cn call --table` writes + the metric table and exits 0; a table with other regions or a short row + exits 2; 6. assets/traits.tsv has the two LPA risk variants with the alleles bin/collect_summary.py counts, and the blood-test advice; 7. both reports: a low count with one risk allele is flagged with both @@ -42,9 +46,11 @@ column are not counted; when step 46's table shows the aligner left out the reads of MAPQ 0, 'not assessed'; the reading goes to its own table and step 46's table is left byte for byte; without a count the copies are '.'; an - output that cannot be written exits 2. + output that cannot be written exits 2. `og cn marker-read` on step 46's + table and `og cn call --marker` (what step 46 and the LPA process run) + write the same 4925G>A table. -Run: python3 tests/test_lpa_kiv2.py +Run: OG=rust/target/release/og python3 tests/test_lpa_kiv2.py """ import math import os @@ -53,15 +59,22 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "lpa_kiv2.py") +CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import lpa_kiv2 as lpa # noqa: E402 +import copy_number as cn # noqa: E402 FAILS = 0 # The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. R1, R2 = "rs10455872", "rs3798220" -KIV = [n for *_, n in lpa.REGIONS if n.startswith("kiv2_")] -FLANK = [n for *_, n in lpa.REGIONS if n.startswith("flank_")] +ROW = cn.load_catalog(CATALOG)["LPA_KIV2"] +REGIONS = cn.regions(ROW) +# in catalog order: the KIV-2 block, then the single-copy exons +KIV = [n for n, *_ in ROW["_targets"]] +FLANK = [n for n, *_ in ROW["_controls"]] +# The advice row of step 46's table. +ADVICE = ("A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 European " + "Atherosclerosis Society consensus advises measuring it at least once in every adult.") def check(desc, ok, detail=""): @@ -87,10 +100,41 @@ def q1(m, share=0.9): return {n: v * (1 - share) if n.startswith("kiv2_") else v for n, v in m.items()} -def bedcov(path, m): +def bedcov(path, m, exact=False): + """samtools bedcov at these mean depths: whole sums, or with exact the + sums themselves, so each mean comes back as given.""" with open(path, "w") as f: - for c, s, e, n in lpa.REGIONS: - f.write(f"{c}\t{s}\t{e}\t{n}\t{round(m[n] * (e - s))}\n") + for c, s, e, n in REGIONS: + v = m[n] * (e - s) + f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") + + +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + +def read_table(path): + return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) if os.path.exists(path) else {} + + +def estimate(a, b, out): + """og cn call on step 46's two bedcov tables, writing its table.""" + return og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, + "--table", out) + + +def assess(m, q): + """Step 46's table for mean depths m (every read) and q (MAPQ >= 1).""" + with tempfile.TemporaryDirectory() as t: + a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) + if m: + bedcov(a, m, exact=True) + bedcov(b, q, exact=True) + else: + open(a, "w").close() + open(b, "w").close() + estimate(a, b, o) + return read_table(o) def cv(vals): @@ -98,22 +142,52 @@ def cv(vals): return math.sqrt(sum((v - mu) ** 2 for v in vals) / (len(vals) - 1)) / mu +def pileup(path, per_site, extra=""): + """mpileup lines at the seven copies of 4925G>A (traits_report.MARKERS); + per_site is (C reads, T reads) at each.""" + import traits_report + with open(path, "w") as f: + for p in traits_report.MARKERS["LPA_KIV2_4925"]["copies"]: + c, t_ = per_site + bases = "^]" + "C" * c + "t" * t_ + "$" + f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") + f.write(extra) + + +def traits(): + """6. assets/traits.tsv holds the two LPA risk variants the report counts.""" + import collect_summary + import traits_report + table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} + for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): + r = table.get(rs, {}) + check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", + r.get("gene") == "LPA" and r.get("alt") == alt, r) + check(f"traits.tsv: {rs} says the actionable test is a blood Lp(a) measurement", + "measure" in (r.get("note") or "").lower() and "blood" in (r.get("note") or ""), r) + + def main(): + if not OG: + traits() + print("skip: OG is not set (OG=rust/target/release/og): only part 6 ran") + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") + return 1 if FAILS or os.environ.get("OG_REQUIRED") else 0 # 1. arithmetic for n in (12, 20, 40): - r = lpa.assess(depths(n), q1(depths(n))) + r = assess(depths(n), q1(depths(n))) check(f"arithmetic: a block at 30 x {n + 2} / 14 is {n} KIV-2 copies", r["status"] == "estimated" and r["kiv2_copies"] == f"{n:.1f}", r) # 2. interval - r = lpa.assess(depths(20), q1(depths(20))) + r = assess(depths(20), q1(depths(20))) check("interval: equal depths give a zero-width interval", r["ci95_low"] == r["ci95_high"] == "20.0", r) m = depths(20, kiv_spread=0.10, flank_spread=0.15) - r = lpa.assess(m, q1(m)) + r = assess(m, q1(m)) rel = math.sqrt(cv([m[n] for n in KIV]) ** 2 / 14 + cv([m[n] for n in FLANK]) ** 2 / 15) - klen = sum(e - s for _, s, e, n in lpa.REGIONS if n in KIV) - flen = sum(e - s for _, s, e, n in lpa.REGIONS if n in FLANK) - k = sum(m[n] * (e - s) for _, s, e, n in lpa.REGIONS if n in KIV) / klen - f = sum(m[n] * (e - s) for _, s, e, n in lpa.REGIONS if n in FLANK) / flen + klen = sum(e - s for _, s, e, n in REGIONS if n in KIV) + flen = sum(e - s for _, s, e, n in REGIONS if n in FLANK) + k = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in KIV) / klen + f = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in FLANK) / flen total = 14 * k / f want = (f"{total - 2:.1f}", f"{total * math.exp(-1.96 * rel) - 2:.1f}", f"{total * math.exp(1.96 * rel) - 2:.1f}") got = (r["kiv2_copies"], r["ci95_low"], r["ci95_high"]) @@ -121,38 +195,38 @@ def main(): check("interval: it contains the estimate", float(r["ci95_low"]) < float(r["kiv2_copies"]) < float(r["ci95_high"]), r) m2 = depths(20, kiv_spread=0.20, flank_spread=0.30) - r2 = lpa.assess(m2, q1(m2)) + r2 = assess(m2, q1(m2)) check("interval: twice the spread is wider", float(r2["ci95_high"]) - float(r2["ci95_low"]) > float(r["ci95_high"]) - float(r["ci95_low"]), (r, r2)) # 3. class for n, sp, cls in ((20, 0.03, "low"), (29, 0.03, "low"), (29, 0.25, "typical"), (38, 0.03, "typical"), (47, 0.25, "typical"), (60, 0.03, "high")): m = depths(n, kiv_spread=sp, flank_spread=sp) - r = lpa.assess(m, q1(m)) + r = assess(m, q1(m)) check(f"class: {n} copies ({r['ci95_low']} to {r['ci95_high']}) is {cls}", r["class"] == cls, r) - r = lpa.assess(depths(20), q1(depths(20))) + r = assess(depths(20), q1(depths(20))) check("class: the low message says what a low count means", "a low count goes with small apo(a) isoforms and a higher Lp(a) level" in r["message"], r) # 4. not assessed - r = lpa.assess(depths(20, flank=8.0), q1(depths(20, flank=8.0))) + r = assess(depths(20, flank=8.0), q1(depths(20, flank=8.0))) check("8x over the single-copy exons: too little depth, no count", r["status"] == "too little depth" and r["kiv2_copies"] == "." and "8.0x" in r["message"], r) - r = lpa.assess(depths(20, flank=10.0), q1(depths(20, flank=10.0))) + r = assess(depths(20, flank=10.0), q1(depths(20, flank=10.0))) check("10x: estimated", r["status"] == "estimated", r) - r = lpa.assess(depths(20), q1(depths(20), share=0.0)) + r = assess(depths(20), q1(depths(20), share=0.0)) check("no read of MAPQ 0 over KIV-2: not assessed, naming the cause", r["status"] == "not assessed" and r["kiv2_copies"] == "." and "novoalign" in r["message"], r) - r = lpa.assess(depths(20), q1(depths(20), share=0.25)) + r = assess(depths(20), q1(depths(20), share=0.25)) check("a quarter of the depth from MAPQ 0: estimated", r["status"] == "estimated", r) - r = lpa.assess({}, {}) + r = assess({}, {}) check("no chr6: not assessed", r["status"] == "not assessed" and "no chr6" in r["message"], r) - r = lpa.assess(depths(-1.5), q1(depths(-1.5))) + r = assess(depths(-1.5), q1(depths(-1.5))) check("KIV-2 depth under the floor of any count: not assessed, never a negative count", r["status"] == "not assessed" and r["kiv2_copies"] == "." and "too little for any repeat count" in r["message"], r) # 5. CLI - out = subprocess.run([sys.executable, SCRIPT, "bed"], capture_output=True, text=True) + out = og_cn("bed", "--catalog", CATALOG, "--locus", "LPA_KIV2") rows = [x.split("\t") for x in out.stdout.splitlines()] check("bed: 29 regions", out.returncode == 0 and len(rows) == 29, out) check("bed: 14 KIV-2 exons of 2,394 bp in total", @@ -163,42 +237,37 @@ def main(): a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) bedcov(a, depths(20)) bedcov(b, q1(depths(20))) - cli = [sys.executable, SCRIPT, "estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", o] + cli = [OG, "cn", "call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, + "--table", o] out = subprocess.run(cli, capture_output=True, text=True) tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) if os.path.exists(o) else {} - check("estimate: exit 0 and the metric table", out.returncode == 0 and tab.get("kiv2_copies") == "20.0" + check("call: exit 0 and the metric table", out.returncode == 0 and tab.get("kiv2_copies") == "20.0" and tab.get("status") == "estimated" and "blood" in tab.get("advice", ""), (out, tab)) open(a, "w").close() open(b, "w").close() out = subprocess.run(cli, capture_output=True, text=True) tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) - check("estimate: two empty tables (no chr6) exit 0, not assessed", + check("call: two empty tables (no chr6) exit 0, not assessed", out.returncode == 0 and tab.get("status") == "not assessed", (out, tab)) bedcov(a, depths(20)) out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: an empty MAPQ 1 table beside a full one exits 2", + check("call: an empty MAPQ 1 table beside a full one exits 2", out.returncode == 2 and "is empty but the other table is not" in out.stderr, out) bedcov(a, depths(20)) with open(a, "a") as f: f.write("chr6\t1\t100\tother\t500\n") out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: other regions exit 2", out.returncode == 2 and "differ" in out.stderr, out) + check("call: other regions exit 2", out.returncode == 2 and "differ" in out.stderr, out) with open(a, "w") as f: f.write("chr6\t160646213\t160646395\n") out = subprocess.run(cli, capture_output=True, text=True) - check("estimate: a short row exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) + check("call: a short row exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) # 6. the traits table + traits() import collect_summary import render_report import traits_report - table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} - for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): - r = table.get(rs, {}) - check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", - r.get("gene") == "LPA" and r.get("alt") == alt, r) - check(f"traits.tsv: {rs} says the actionable test is a blood Lp(a) measurement", - "measure" in (r.get("note") or "").lower() and "blood" in (r.get("note") or ""), r) # 7. reports def sample(t, name, n_copies, flank=30.0, genotypes=None): @@ -208,7 +277,7 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): m = depths(n_copies, flank=flank, kiv_spread=0.03, flank_spread=0.03) bedcov(a, m) bedcov(b, q1(m)) - lpa.main(["estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", os.path.join(d, "lpa", "S_lpa_kiv2.tsv")]) + estimate(a, b, os.path.join(d, "lpa", "S_lpa_kiv2.tsv")) if genotypes: os.makedirs(os.path.join(d, "traits")) with open(os.path.join(d, "traits", "S_traits.tsv"), "w") as f: @@ -231,10 +300,10 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): check("report: the text section has the count, both variants and the advice", "## Lp(a): LPA KIV-2 Repeat Count and Risk Variants" in txt and "KIV-2 repeats (both chromosomes): 20.0" in txt and f"{R1}: A/G (1 risk allele)" in txt and f"{R2}: T/T (0 risk alleles)" in txt - and lpa.ADVICE in txt, txt) + and ADVICE in txt, txt) check("report: the HTML card has the flag and the advice", "

Lp(a): LPA KIV-2 Repeat Count

" in html and "Genetic sign of high Lp(a)" in html - and render_report.E(lpa.ADVICE) in html, html[-3000:]) + and render_report.E(ADVICE) in html, html[-3000:]) summ, txt, html = sample(t, "typ", 38, genotypes={R1: "A/A", R2: "T|T"}) check("report: a typical count with no risk allele is not flagged", summ["sections"]["lpa"]["data"]["flag"] == "." and "No genetic sign of a high Lp(a)" in txt @@ -261,8 +330,9 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): and "KIV-2 4925G>A null allele (rs1484919544): not assessed\n" in txt.split(count, 1)[-1].split(R1, 1)[0] and "step 46 wrote no S_lpa_kiv2_4925.tsv" in txt and cav in txt, txt) d = os.path.join(t, "nul0", "S", "lpa") - traits_report.write_metric_table(os.path.join(d, "S_lpa_kiv2_4925.tsv"), - lpa.null_allele((665, 35, 0), "38.0")) + pileup(os.path.join(t, "nul0.pileup"), (95, 5)) + og_cn("marker-read", "LPA_KIV2_4925", "--pileup", os.path.join(t, "nul0.pileup"), "--copy-table", + os.path.join(d, "S_lpa_kiv2.tsv"), "--out", os.path.join(d, "S_lpa_kiv2_4925.tsv")) summ = collect_summary.collect("S", os.path.join(t, "nul0", "S")) txt, html = render_report.text_report(summ), render_report.html_report(summ) beside = txt.split(count, 1)[-1].split(R1, 1)[0] @@ -291,16 +361,6 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): "

Lp(a)" not in render_report.html_report(nothing) and "## Lp(a)" not in render_report.text_report(nothing)) # 8. KIV-2 4925G>A beside the count - sites = traits_report.MARKERS["LPA_KIV2_4925"]["copies"] - - def pileup(path, per_site, extra=""): - """mpileup lines at the seven copies; per_site is (C reads, T reads) at each.""" - with open(path, "w") as f: - for p in sites: - c, t_ = per_site - bases = "^]" + "C" * c + "t" * t_ + "$" - f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") - f.write(extra) def table_after(t, name, per_site, n_copies=38, extra=""): """Step 46's table, a pileup, then null-allele; returns (rc, the 4925G>A table, LPA table, pileup, out).""" @@ -309,10 +369,9 @@ def table_after(t, name, per_site, n_copies=38, extra=""): a, b, o, p, n = (os.path.join(d, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv", "p.tsv", "S_lpa_kiv2_4925.tsv")) bedcov(a, depths(n_copies)) bedcov(b, q1(depths(n_copies))) - lpa.main(["estimate", "--bedcov", a, "--bedcov-mapq1", b, "--out", o]) + estimate(a, b, o) pileup(p, per_site, extra) - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, "--out", n], - capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) if os.path.exists(n) else {} return rc, tab, o, p, n @@ -333,15 +392,17 @@ def table_after(t, name, per_site, n_copies=38, extra=""): a0, b0, o0 = (os.path.join(d0, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv")) bedcov(a0, depths(38)) bedcov(b0, q1(depths(38))) - lpa.main(["estimate", "--bedcov", a0, "--bedcov-mapq1", b0, "--out", o0]) + estimate(a0, b0, o0) check("4925G>A: step 46's table is left byte for byte", open(o).read() == open(o0).read(), open(o).read()) - n3 = os.path.join(t, "viatr.tsv") - res = subprocess.run([sys.executable, os.path.join(REPO, "bin", "traits_report.py"), "--marker", "LPA_KIV2_4925", - "--pileup", p, "--copy-table", o, "--marker-out", n3], capture_output=True, text=True) - check("4925G>A: traits_report.py --marker LPA_KIV2_4925 (what step 46 and the LPA process run) writes " - "the same table as null-allele", res.returncode == 0 and open(n3).read() == open(n).read(), res) - res = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", - os.path.join(t, "none.tsv"), "--out", n3], capture_output=True, text=True) + n3, o3 = os.path.join(t, "viacall.tsv"), os.path.join(t, "viacall_lpa.tsv") + res = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a0, "--bedcov-mapq1", b0, + "--table", o3, "--marker", "LPA_KIV2_4925", "--marker-pileup", p, "--marker-out", n3) + check("4925G>A: og cn call --marker LPA_KIV2_4925 (what step 46 and the LPA process run) writes the same " + "count table and the same 4925G>A table as marker-read on that table", + res.returncode == 0 and open(o3).read() == open(o0).read() and open(n3).read() == open(n).read() + and res.stdout.splitlines()[-1].startswith("LPA LPA KIV-2 4925G>A"), res) + res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", os.path.join(t, "none.tsv"), + "--out", n3) tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) check("4925G>A: without step 46's table the reading stands and the copies are '.'", res.returncode == 0 and tab.get("status") == "present" and tab.get("copies") == "." @@ -350,8 +411,7 @@ def table_after(t, name, per_site, n_copies=38, extra=""): lt = os.path.join(t, f"share{share}.tsv") with open(lt, "w") as f: f.write(f"metric\tvalue\nstatus\tnot assessed\nkiv2_copies\t.\nmapq0_share\t{share}\n") - res = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", lt, "--out", n3], - capture_output=True, text=True) + res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", lt, "--out", n3) tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) check(f"4925G>A: a MAPQ 0 share of {share} in step 46's table gives {want} on the same reads", res.returncode == 0 and tab.get("status") == want @@ -374,15 +434,26 @@ def table_after(t, name, per_site, n_copies=38, extra=""): rc, tab, o, p, n = table_after(t, "bad", (95, 5)) with open(p, "w") as f: f.write("not a pileup\n") - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, "--out", n], - capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) check("4925G>A: an unreadable pileup is not assessed, exit 0", rc.returncode == 0 and tab.get("status") == "not assessed", (rc, tab)) - rc = subprocess.run([sys.executable, SCRIPT, "null-allele", "--pileup", p, "--table", o, - "--out", os.path.join(t, "no", "such", "dir.tsv")], capture_output=True, text=True) + rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, + "--out", os.path.join(t, "no", "such", "dir.tsv")) check("4925G>A: an output that cannot be written exits 2", rc.returncode == 2, rc) + # The demo's step 46 table (tests/demo/make_demo_sample.py) is what og writes. + sys.path.insert(0, os.path.join(REPO, "tests", "demo")) + import make_demo_sample as demo + m, q = demo.lpa_depth(ROW) + with tempfile.TemporaryDirectory() as t: + a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) + bedcov(a, m, exact=True) + bedcov(b, q, exact=True) + estimate(a, b, o) + got = open(o).read() if os.path.exists(o) else "" + check("the demo's step 46 table is the one og cn call writes for its depths", got == demo.LPA_TABLE, got) + print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") return 1 if FAILS else 0 diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index 375926cb2..cd9b75470 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -50,8 +50,10 @@ 11. the copy-aware markers (MARKERS: LPA KIV-2 4925G>A, SMN1 g.27134T>G): each pinned dbSNP record is one of the marker's copies with its ref and alt, and a planted swap, a minus-strand pair and a position off the - copies are each caught; the context's middle base is the ref; - --marker-regions prints one region per copy; on mpileup lines made up + copies are each caught; the context's middle base is the ref. With OG + naming the og binary (skipped without it, a failure with OG_REQUIRED + set too): og cn marker-regions prints one region per copy of MARKERS; + og cn marker-read SMN1_27134 on mpileup lines made up for each case at the SMN1 and SMN2 copies: a quarter of the reads with G is 'present' (copies = fraction x agCN), T only (the reference) 'not seen', 2 reads with G 'uncertain', under 10 reads 'too little depth', an @@ -78,6 +80,9 @@ RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") SCRIPT = os.path.join(REPO, "bin", "traits_report.py") CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") +OG = os.environ.get("OG", "") +# The sentence every SMN1 marker message ends with. +SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." FAILS = 0 # The sha256 of the APOE line (its OUT_COLUMNS joined by tabs) the hand code of @@ -147,6 +152,10 @@ def run(*args): return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) +def og_cn(*args): + return subprocess.run([OG, "cn", *args], capture_output=True, text=True) + + def dbsnp_checks(rules): """10. scripts/ci/check-rule-alleles.sh on the rules and on planted copies. Runs first, so a rules table that breaks the other checks still shows what dbSNP says.""" @@ -294,10 +303,14 @@ def marker_checks(): e = tr.marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) check(f"marker {name} control: a context whose middle base is not the ref is caught", any("middle base" in x for x in e), e) - res = run("--marker-regions", name) - check(f"marker {name}: --marker-regions prints one region per copy", - res.returncode == 0 and res.stdout.split() == [f"{chrom}:{p}-{p}" for p in m["copies"]], res) check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) + if not OG: + print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") + return + for name, m in tr.MARKERS.items(): + res = og_cn("marker-regions", name) + check(f"marker {name}: og cn marker-regions prints one region per copy of MARKERS", + res.returncode == 0 and res.stdout.split() == [f"{m['chrom']}:{p}-{p}" for p in m["copies"]], res) sm = tr.MARKERS["SMN1_27134"] work = tempfile.mkdtemp() @@ -315,7 +328,9 @@ def smn(pscn="2,2", pqual="30,30", pfilt="PASS", agcn="4", homolog="chr5:7004900 def table(p, smn_path): out = os.path.join(work, "o.tsv") - res = run("--marker", "SMN1_27134", "--pileup", p, "--copy-table", smn_path, "--marker-out", out) + if os.path.exists(out): + os.remove(out) + res = og_cn("marker-read", "SMN1_27134", "--pileup", p, "--copy-table", smn_path, "--out", out) return res, dict(x.split("\t", 1) for x in open(out).read().splitlines()[1:]) if os.path.exists(out) else {} res, t = table(pileup(15, 5), smn()) @@ -362,13 +377,20 @@ def table(p, smn_path): # Every status with every psCN: the message says the marker does not prove 2+0. msgs = [] - for counts in [(30, 10, 0), (40, 0, 0), (38, 2, 0), (3, 1, 0), None]: - for path in (smn(), smn(pscn="1,3"), smn(pqual="5,5"), os.path.join(work, "missing.tsv")): - msgs.append(dict(tr.smn1_marker(counts, path))["message"]) + for counts in [(30, 10), (40, 0), (38, 2), (3, 1), None]: + if counts is None: + p = bad + else: # every read at the SMN1 copy + p = os.path.join(work, "all.tsv") + write(p, f"chr5\t{sm['copies'][0]}\tN\t{sum(counts)}\t{'T' * counts[0]}{'G' * counts[1]}\t.\n") + for path in ("2,2", "1,3", "5,5", None): + smn_path = (os.path.join(work, "missing.tsv") if path is None + else smn(pqual=path) if path == "5,5" else smn(pscn=path)) + msgs.append(table(p, smn_path)[1].get("message", "")) check(f"SMN1 marker: every message ({len(msgs)}) says it does not prove 2+0", - all(says_not_proof(x) for x in msgs), [x for x in msgs if not says_not_proof(x)]) + len(msgs) == 20 and all(says_not_proof(x) for x in msgs), [x for x in msgs if not says_not_proof(x)]) check("SMN1 marker control: a message without the sentence fails that test", - not says_not_proof(msgs[0].replace(tr.SMN1_NOT_PROOF, ""))) + not says_not_proof(msgs[0].replace(SMN1_NOT_PROOF, ""))) finally: shutil.rmtree(work) @@ -601,7 +623,7 @@ def rule(rid, *lines, phased=None): marker_checks() print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 + return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 if __name__ == "__main__": From ab5a3c218a3f1668df9c334daa7a1ba92a256180 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:26:59 +0100 Subject: [PATCH 11/53] ci(guard): run the copy-number tests and the cn-engine-calls case with og Bead: pgp-bs9u.24 --- .github/workflows/guard.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index feccd7079..8e9332246 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -222,8 +222,8 @@ jobs: git checkout -- og/src/main.rs ./og.sh check --required - # The tests of the og filters steps 04, 05 and 07 run, and og chip's - # (step 49): unit-tests has + # The tests of the og filters steps 04, 05 and 07 run, og chip's + # (step 49) and og cn's (steps 35, 41 and 46): unit-tests has # no og, so there they skip; here OG_REQUIRED turns a skip into a # failure. The step 05 tests run the real step with bin/og. - name: The og filter tests and step 05 with og @@ -235,7 +235,8 @@ jobs: fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ - tests/test_chip_report.py; do + tests/test_chip_report.py tests/test_copy_number.py tests/test_hba_copy_number.py \ + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in @@ -247,6 +248,15 @@ jobs: done exit "$fail" + # The fake-docker job builds no og, so its cn-engine-calls case says it + # is skipped there; here bin/og is built and the case must run. + - name: The fake-docker case cn-engine-calls (steps 35, 41 and 46 through og cn) with og + working-directory: . + run: | + out=$(scripts/ci/run-fake-docker-suite.sh cn-engine-calls 2>&1) || { echo "$out"; exit 1; } + echo "$out" + if grep -q '^skip: bin/og is not built' <<<"$out"; then echo "ERROR: the case was skipped, with og built"; exit 1; fi + # main.nf runs the same check before any task: a stale og stops # `nextflow run` with the setup command. Without the check, this stub # run completes and the step fails. From 2e3e9d85fa2cb5a31b8d5acb4ba059f693f7cbbc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 12/53] feat(live): replace the findings karyogram with a genome map Figure 1 of a person page now draws the og map document: every chromosome at genome level, and one chromosome opened in place down to about 10 Mb, with the findings lane, runs of homozygosity and the bands. Closed results stay off the map until opened on purpose, with the same switch as the card. Without the document the page shows a table. Bead: pgp-iqon.8 --- assets/live/core.js | 70 +-- assets/live/index.html | 2 + assets/live/live.css | 6 - assets/live/map.css | 57 +++ assets/live/map.js | 1074 ++++++++++++++++++++++++++++++++++++++++ assets/live/person.js | 67 ++- 6 files changed, 1186 insertions(+), 90 deletions(-) create mode 100644 assets/live/map.css create mode 100644 assets/live/map.js diff --git a/assets/live/core.js b/assets/live/core.js index b6e4ff071..0d1db0fe9 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -1,6 +1,6 @@ // core.js: what the live site's pages share (home.html, index.html, finding.html, family.html). Element builders, // a fetch that sends the session and goes to the login on a 401, the pedigree symbol, dates, the -// caveat block, the facts list, the table, and the Journal's level marks and chromosome map. Text only through textContent and createElementNS: +// caveat block, the facts list, the table, and the Journal's level marks. Text only through textContent and createElementNS: // no innerHTML, no inline script, no eval. Everything hangs on window.PGPUI. (function () { 'use strict'; @@ -124,7 +124,7 @@ }); } - // --- the Journal's shared marks: the level glyph and word, and the chromosome map of findings --- + // --- the Journal's shared marks: the level glyph and word --- var LEVEL_WORDS = { act: 'Take to a doctor', medicine: 'Tell your prescriber', review: 'Worth a look', carrier: 'For family planning', trust: 'Data quality', info: 'For information' }; var LEVEL_SHORT = { act: 'doctor', medicine: 'prescriber', review: 'look', carrier: 'family', trust: 'quality', info: 'info' }; @@ -137,68 +137,6 @@ } // A finding's level: a data-quality finding is always a problem with the data, so its mark is red. function findingLevel(lv, words) { return level(lv, words, lv === 'trust' ? 'problem' : undefined); } - // GRCh38: [name, length Mb, centromere Mb]. - var CHR = [['1', 248.96, 123.4], ['2', 242.19, 93.9], ['3', 198.3, 90.9], ['4', 190.21, 50], ['5', 181.54, 48.8], - ['6', 170.81, 59.8], ['7', 159.35, 60.1], ['8', 145.14, 45.2], ['9', 138.39, 43], ['10', 133.8, 39.8], - ['11', 135.09, 53.4], ['12', 133.28, 35.5], ['13', 114.36, 17.7], ['14', 107.04, 17.2], ['15', 101.99, 19], - ['16', 90.34, 36.8], ['17', 83.26, 25.1], ['18', 80.37, 18.5], ['19', 58.62, 26.2], ['20', 64.44, 28.1], - ['21', 46.71, 12], ['22', 50.82, 15], ['X', 156.04, 60.6], ['Y', 57.23, 10.4]]; - var LV_COLOUR = { act: '#d03b3b', medicine: '#ec835a', review: '#fab219', carrier: '#9085e9', trust: '#c3c2b7', info: '#c3c2b7' }; - function glyph(g, lv, x, y) { - var c = LV_COLOUR[lv] || LV_COLOUR.info; - if (lv === 'act') { g.appendChild(svg('rect', { x: x - 4, y: y - 4, width: 8, height: 8, fill: c, transform: 'rotate(45 ' + x + ' ' + y + ')' })); } - else if (lv === 'medicine') { g.appendChild(svg('rect', { x: x - 5.5, y: y - 3, width: 11, height: 6, rx: 3, fill: c })); } - else if (lv === 'review') { g.appendChild(svg('path', { d: 'M' + x + ' ' + (y - 5) + 'L' + (x + 5) + ' ' + (y + 4) + 'L' + (x - 5) + ' ' + (y + 4) + 'Z', fill: c })); } - else if (lv === 'carrier') { - g.appendChild(svg('circle', { cx: x - 2.6, cy: y, r: 3.4, fill: 'none', stroke: c, 'stroke-width': 1.6 })); - g.appendChild(svg('circle', { cx: x + 2.6, cy: y, r: 3.4, fill: 'none', stroke: c, 'stroke-width': 1.6 })); - } else { g.appendChild(svg('circle', { cx: x, cy: y, r: 4, fill: 'none', stroke: c, 'stroke-width': 1.5 })); } - } - // "chr16:2087914" or "chr16:2084000-2096000" -> {chr: '16', mb: 2.08}; null for anything else. - function locus(v) { - var m = /^(?:chr)?([0-9]{1,2}|X|Y):([0-9]+)/.exec(String(v || '')); - return m ? { chr: m[1], mb: Number(m[2]) / 1e6 } : null; - } - // The chromosome map: the 24 chromosomes in a row, each finding a glyph on its chromosome, with its - // name beside it and the same words in a title (and in the list under the figure). - function karyogram(marks, opts) { - opts = opts || {}; - var W = opts.width || 680, H = opts.height || 190, top = 22, bottom = 22; - var slot = (W - 56) / CHR.length, scale = (H - top - bottom) / 249; - var s = svg('svg', { viewBox: '0 0 ' + W + ' ' + H, role: 'img', 'aria-label': opts.aria || 'Where the findings sit, by chromosome' }); - CHR.forEach(function (c, i) { - var x = i * slot + slot / 2 - 3.5; - s.appendChild(svg('rect', { 'class': 'chr', x: x, y: top, width: 7, height: Math.max(1, c[2] * scale - 1.2), rx: 3.5 })); - s.appendChild(svg('rect', { 'class': 'chr', x: x, y: top + c[2] * scale + 1.2, width: 7, height: Math.max(1, (c[1] - c[2]) * scale - 1.2), rx: 3.5 })); - var t = svg('text', { 'class': 'lab', x: x + 3.5, y: H - 6, 'text-anchor': 'middle' }); - t.textContent = c[0]; - s.appendChild(t); - }); - (marks || []).forEach(function (m) { - var i = -1; - CHR.forEach(function (c, j) { if (c[0] === m.chr) { i = j; } }); - if (i < 0) { return; } - var x = i * slot + slot / 2 + 7, y = top + Math.min(m.mb, CHR[i][1]) * scale; - var g = svg('g', { 'class': 'mk', tabindex: '0' }); - var tt = svg('title', {}); - tt.textContent = (LEVEL_WORDS[m.lv] || '') + ': ' + (m.tip || m.label); - g.appendChild(tt); - g.appendChild(svg('line', { x1: x - 10.5, x2: x - 1, y1: y, y2: y, stroke: LV_COLOUR[m.lv] || LV_COLOUR.info, 'stroke-width': 2 })); - glyph(g, m.lv, x + 4, y); - if (m.label) { - var t = svg('text', { x: x + 11, y: y + 3.5, stroke: '#0e0f10', 'stroke-width': 4, 'paint-order': 'stroke', 'stroke-linejoin': 'round' }); - t.textContent = m.label; - g.appendChild(t); - } - if (m.href) { - g.setAttribute('role', 'link'); - g.addEventListener('click', function () { window.location.href = m.href; }); - g.addEventListener('keydown', function (e) { if (e.key === 'Enter') { window.location.href = m.href; } }); - } - s.appendChild(g); - }); - return s; - } // A plain time: "14:22" in UTC, "4 Nov" for a day. var MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; function hm(utc) { var d = new Date(utc); return isFinite(d) ? ('0' + d.getUTCHours()).slice(-2) + ':' + ('0' + d.getUTCMinutes()).slice(-2) : ''; } @@ -209,7 +147,7 @@ } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, - facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, - karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, + facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, + LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, hm: hm, dm: dm, hours: hours }; }()); diff --git a/assets/live/index.html b/assets/live/index.html index b272cd44f..d740c1eda 100644 --- a/assets/live/index.html +++ b/assets/live/index.html @@ -5,8 +5,10 @@ {{run}}: OwnGenome + + {{plugins}} diff --git a/assets/live/live.css b/assets/live/live.css index 5dba9f176..6c0959048 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -321,12 +321,6 @@ ol.w-path li.w-low { border-style: dashed; border-color: var(--car); } .w-count { color: var(--ink-3); font-size: 12px; } /* the chromosome map of the findings (person.js) and the charts plugin */ -.kary { margin-top: 12px; } -.kary svg { width: 100%; height: auto; display: block; } -.kary .chr { fill: rgba(255, 255, 255, .07); } -.kary .lab { fill: var(--ink-3); font-size: 10px; } -.kary .mk { cursor: pointer; } -.kary .mk text { fill: var(--ink-2); font-size: 10.5px; } .chart-slot:empty { display: none; } figure.chart { margin: 0 0 14px; } figure.chart h3 { font: 600 10.5px var(--sans); letter-spacing: .12em; text-transform: uppercase; color: var(--ink-3); margin-bottom: 8px; } diff --git a/assets/live/map.css b/assets/live/map.css new file mode 100644 index 000000000..cc3e42e00 --- /dev/null +++ b/assets/live/map.css @@ -0,0 +1,57 @@ +/* map.css: Figure 1 of a person's page, the genome map (assets/live/map.js). Dark only, the Journal tokens of + live.css. Brass marks where you are, never a level; the levels' colours mark findings only. */ +.gmap { margin-top: 34px; scroll-margin-top: 72px; } +.gm-main { min-width: 0; } +.gm-head { display: grid; gap: 6px; margin-bottom: 10px; } +.gm-titlerow { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; flex-wrap: wrap; } +.gm-title { font: 400 27px/1.15 var(--serif); color: var(--ink); } +.gm-gauge { display: inline-flex; border: 1px solid var(--hair-2); border-radius: 999px; overflow: hidden; } +.gm-gauge button { border: 0; border-radius: 0; background: transparent; color: var(--ink-3); padding: 6px 12px; font-size: 12px; } +.gm-gauge button[aria-pressed="true"] { color: var(--page); background: var(--brass); } +.gm-crumb { color: var(--ink-3); font-size: 12px; } +.gm-crumb button { background: none; border: 0; padding: 0; color: var(--ink-2); font: inherit; text-decoration: underline; + text-decoration-color: var(--brass-dim); text-underline-offset: 3px; } +.gm-crumb [aria-current] { color: var(--ink); } +.gm-chips { display: flex; flex-wrap: wrap; gap: 8px; } +.gm-chips:empty { display: none; } +.gm-chip { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 10px; font: 12.5px/1.4 var(--sans); color: var(--ink-2); + border: 1px dashed var(--hair-2); border-radius: 8px; padding: 6px 10px; max-width: 100%; } +.gm-chip.opened { border-style: solid; border-color: var(--brass-dim); } +.gm-chip button { padding: 5px 10px; font-size: 12px; } +.gm-confirm { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--ink); } +.gm-stage { position: relative; } +.gm-canvas { display: block; width: 100%; touch-action: pan-y; border-radius: 6px; } +.gm-canvas:focus-visible { outline-offset: 4px; } +.gm-tip { position: absolute; z-index: 5; max-width: 260px; pointer-events: none; background: var(--paper-2); border: 1px solid var(--hair-2); + border-radius: 6px; padding: 7px 9px; font: 12px/1.4 var(--sans); color: var(--ink-2); display: grid; gap: 2px; } +.gm-tip b { color: var(--ink); font-weight: 500; } +.gm-toast { position: absolute; left: 50%; top: 8px; transform: translateX(-50%); background: var(--paper-2); border: 1px solid var(--brass-dim); + border-radius: 6px; padding: 7px 12px; font: 12.5px/1.4 var(--sans); color: var(--ink); max-width: 90%; } +/* The marks as links: the path for the keyboard and a screen reader, out of sight until focused. */ +.gm-marks { list-style: none; margin: 0; padding: 0; position: relative; } +.gm-marks li { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; } +.gm-marks li:focus-within { position: static; width: auto; height: auto; clip-path: none; white-space: normal; font-size: 12.5px; margin-top: 6px; } +.gm-fallback .gm-fail { color: var(--ink-2); } +.gm-margin { font-size: 13px; } +.gm-mline { color: var(--ink-3); font-size: 12px; margin: 6px 0 4px; } +.gm-narration { font: 400 15.5px/1.5 var(--serif); color: var(--ink-2); margin-top: 8px; } +.gm-narration b { color: var(--ink); font-weight: 600; } +.gm-narration p { margin: 0 0 10px; } +.gm-narration .dim { font: 12.5px/1.45 var(--sans); } +.gm-more { margin-top: 12px; } +.gm-more > summary { font: 12.5px var(--sans); color: var(--ink-3); cursor: pointer; } +.gm-layers { margin: 10px 0 0; display: grid; gap: 8px; } +.gm-layer dt { font: 600 10.5px/1.3 var(--sans); letter-spacing: .08em; text-transform: uppercase; color: var(--ink-3); } +.gm-layer dd { margin: 2px 0 0; color: var(--ink-2); font-size: 12.5px; line-height: 1.45; } +.gm-layer[data-state="missing"] dd, .gm-layer[data-state="unreadable"] dd { color: var(--ink); } +.gm-caveat summary { color: var(--ink-3); cursor: pointer; font-size: 12px; } +.gm-key { display: flex; flex-wrap: wrap; gap: 6px 12px; margin-top: 14px; font-size: 12px; color: var(--ink-2); } +.gm-key .fig-label { flex-basis: 100%; } +.gm-k-roh, .gm-k-mask { display: inline-flex; align-items: center; gap: 6px; } +.gm-k-roh .sw { width: 16px; height: 3px; background: var(--ink); border-radius: 2px; } +.gm-k-mask .sw { width: 16px; height: 8px; border-radius: 3px; background: repeating-linear-gradient(135deg, #26272c 0 2px, #4a4b50 2px 3px); } +@media (max-width: 700px) { + .gm-title { font-size: 22px; } + .gm-margin { margin-top: 12px; } + .gm-narration { font-size: 14px; } +} diff --git a/assets/live/map.js b/assets/live/map.js new file mode 100644 index 000000000..94ffbaee0 --- /dev/null +++ b/assets/live/map.js @@ -0,0 +1,1074 @@ +// map.js: Figure 1 of a person's page, the genome map. It draws the map document the server builds with +// `og map build` (GET map.json beside the page: gzip-wrapped columnar JSON, rust/crates/og-map/README.md): +// the 24 chromosomes as rows with their bands, the masked spans hatched, each finding with a place as its +// level's glyph, and the runs of homozygosity; a click opens a chromosome in place, where the bands, the +// findings and the runs get lanes of their own under a ruler. Every mark comes from the document, never from +// results.json, and every sentence the margin shows is one the document wrote (text, phrase, empty, note), +// with the counts filled in here. +// +// The veil. The document is the same for every reader: each row a closed result could reveal carries the id +// of that result (veil). Whether it is closed is the page's question (person.js veil()): a closed row is not +// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. The closed +// results are counted in a dashed chip that names no gene and no place, with an Open button that asks once, +// then opens the result through the page's own function, so the map and the card open together. A reload +// closes them again: the open state lives in the page's memory only. +// +// No library and no build step: one canvas, the camera {ci, c, ls, k} (open chromosome, centre bp, natural +// log of the span, 0 genome .. 1 open) eased toward its target each frame, one level() by bp per CSS pixel. +// Text only through textContent: no innerHTML, no style attribute. Everything hangs on window.PGPMap. +(function () { + 'use strict'; + var U = window.PGPUI; + if (!U) { return; } + var el = U.el, add = U.add; + var LV = { act: '#d03b3b', medicine: '#ec835a', review: '#fab219', carrier: '#9085e9', trust: '#c3c2b7', info: '#c3c2b7' }; + var PRI = { act: 0, medicine: 1, review: 2, carrier: 3, trust: 4, info: 5 }; + var STAIN = { gneg: '#26272c', gpos25: '#323337', gpos50: '#3e3f42', gpos75: '#4c4c4d', gpos100: '#5a5955', + gvar: '#25262a', stalk: '#25262a', acen: '#2e2f34' }; + var BRASS = '#c8ab78', INK = '#f0efec', INK2 = '#c3c2b7', INK3 = '#8f8d86', PAGE = '#0e0f10'; + var SANS = 'system-ui, -apple-system, "Segoe UI", sans-serif'; + var MONO = 'ui-monospace, "SF Mono", Menlo, monospace'; + // The chromosome level ends at 16,667 bp per CSS pixel: closer views (region, gene, reads) come later. + var CHROM_BPP = 16667; + // A genome row's least height: 22 CSS px, 14 on a phone (the page there is long enough already). + var ROW_MIN = 22, ROW_PHONE = 14; + var reduced = window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + + function lerp(a, b, t) { return a + (b - a) * t; } + function clamp(v, a, b) { return Math.max(a, Math.min(b, v)); } + function mb(bp, d) { return (bp / 1e6).toFixed(d === undefined ? 1 : d); } + function comma(n) { return String(Math.round(n)).replace(/\B(?=(\d{3})+(?!\d))/g, ','); } + // A sentence of the document: pieces of text, the bold ones in . + function runs(list) { + var p = el('p', {}); + (list || []).forEach(function (x) { add(p, x.b ? el('b', {}, x.t) : document.createTextNode(x.t)); }); + return p; + } + function short(n) { return String(n).replace(/^chr/, ''); } + // A layer's states that put nothing on the map (the document's layer entry); any other state has landed. + var NOT_ON_MAP = { missing: 1, unreadable: 1, not_run: 1, not_in_this_run: 1, running: 1, waiting: 1, failed: 1, skipped: 1 }; + function landed(lay) { return !!lay && !NOT_ON_MAP[lay.state]; } + + // --- the document: gzip -> JSON -> typed columns ------------------------------------------------------ + // Starts are gaps from the previous row (ds) and ends are lengths (l): s, e and the running maximum of e + // are rebuilt here in one pass, so a view [a, b) is a binary search on that maximum. + function columns(o) { + var n = (o.ds || []).length, s = new Float64Array(n), e = new Float64Array(n), me = new Float64Array(n); + var at = 0, m = 0; + for (var i = 0; i < n; i++) { + at += o.ds[i]; + s[i] = at; + e[i] = at + o.l[i]; + m = Math.max(m, e[i]); + me[i] = m; + } + return { n: n, s: s, e: e, me: me, o: o }; + } + // The first row whose running maximum end passes a: every row from there with s < b meets [a, b). + function first(c, a) { + var lo = 0, hi = c.n; + while (lo < hi) { var mid = (lo + hi) >> 1; if (c.me[mid] > a) { hi = mid; } else { lo = mid + 1; } } + return lo; + } + function prepare(d, who) { + var person = (d.people || [])[0] || {}; + var id = who || person.id; + var per = (d.per || {})[id] || {}; + var chroms = ((d.ref && d.ref.chroms) || []).filter(function (c) { return !(c.n === 'chrY' && person.has_y === false); }); + var by = {}; + chroms.forEach(function (c, i) { by[c.n] = i; }); + var bands = {}, roh = {}, bins = {}; + chroms.forEach(function (c) { + var b = d.ref.bands && d.ref.bands[c.n]; + if (b) { bands[c.n] = columns(b); } + if (per.roh && per.roh[c.n]) { roh[c.n] = columns(per.roh[c.n]); } + if (per.roh_bins && per.roh_bins[c.n]) { bins[c.n] = new Float64Array(per.roh_bins[c.n]); } + }); + var f = per.findings || {}, veilIds = (d.dict && d.dict.veil) || []; + var rows = []; + (f.id || []).forEach(function (fid, i) { + if (!(f.chrom[i] in by)) { return; } + rows.push({ id: fid, level: f.level[i], gate: f.gate[i], chrom: f.chrom[i], pos: f.pos[i], s: f.s[i], e: f.e[i], + gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null }); + }); + var gated = {}; + rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); + return { doc: d, id: id, label: person.label || id, chroms: chroms, by: by, bands: bands, roh: roh, bins: bins, + binSize: (per.roh_bins && per.roh_bins.size) || 1e6, findings: rows, gated: gated, veilIds: veilIds, + layers: d.layers || {}, text: d.text || {}, art: (d.dict && d.dict.art) || [] }; + } + + // --- the figure --------------------------------------------------------------------------------------- + // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), + // close(id), fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key + // fetches the document again and keeps the camera. + function create(opts) { + var M = null, key = null, failure = null, t0 = 0; + var fig = el('figure', { 'class': 'row fig1 gmap', id: 'genome-map', 'aria-labelledby': 'gm-title' }); + var main = el('div', { 'class': 'gm-main' }); + var head = el('div', { 'class': 'gm-head' }); + var kicker = el('div', { 'class': 'fig-label' }, 'Figure 1 · Genome map'); + var title = el('h2', { 'class': 'gm-title', id: 'gm-title' }, 'The genome map'); + var gauge = el('div', { 'class': 'gm-gauge', role: 'group', 'aria-label': 'Zoom level' }); + var stopG = el('button', { type: 'button', 'data-stop': 'genome' }, 'Genome'); + var stopC = el('button', { type: 'button', 'data-stop': 'chromosome' }, 'Chromosome'); + add(gauge, stopG, stopC); + var crumb = el('nav', { 'class': 'gm-crumb mono', 'aria-label': 'Where the map is' }); + var chips = el('div', { 'class': 'gm-chips' }); + add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge), crumb, chips); + var stage = el('div', { 'class': 'gm-stage' }); + var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); + var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); + var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); + add(stage, canvas, tip, toast); + // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). + var marks = el('ul', { 'class': 'gm-marks', 'aria-label': 'Findings on the map' }); + var fallback = el('div', { 'class': 'gm-fallback', hidden: 'hidden' }); + add(main, head, stage, marks, fallback); + var margin = el('aside', { 'class': 'gm-margin' }); + var mkick = el('div', { 'class': 'fig-label' }, 'The whole genome'); + var mline = el('div', { 'class': 'gm-mline mono' }); + var narr = el('div', { 'class': 'gm-narration', id: 'gm-narration', 'aria-live': 'polite' }); + var layersBox = el('dl', { 'class': 'gm-layers' }); + var keyBox = el('div', { 'class': 'gm-key', 'aria-label': 'Key' }); + // The layers and the key: open beside the map, folded under it on a phone. + var more = el('details', { 'class': 'gm-more' }); + more.open = !(window.matchMedia && window.matchMedia('(max-width: 700px)').matches); + add(more, el('summary', {}, 'Layers and key'), layersBox, keyBox); + add(margin, mkick, mline, narr, more); + add(fig, main, margin); + + var ctx = canvas.getContext('2d'); + var CW = 0, CH = 0, DPR = 1, phone = false; + var cam = { ci: 0, c: 0, ls: 0, k: 0 }, tgt = { ci: 0, c: 0, ls: 0, k: 0 }; + var hover = { row: -1, hit: null }, hits = [], raf = 0, settleT = 0, hashT = 0, confirming = null, touched = false; + var cache = null, cacheKey = ''; + var hatch = null; + + // --- the veil: what this page keeps closed at this moment --- + function stateOf(id) { return opts.veil(id, !!(M && M.gated[id])); } + // A row shows when its result is not closed and the page shows its finding at this moment (a replay + // before the finding's step ended does not). + function shownRow(r) { return (!r.veil || stateOf(r.veil) !== 'closed') && (!opts.now || opts.now(r.id)); } + function drawn() { return M ? M.findings.filter(shownRow) : []; } + + // --- geometry --- + function chrom() { return M.chroms[cam.ci]; } + function rowH() { var lo = phone ? ROW_PHONE : ROW_MIN; return M ? clamp((CH - 18) / M.chroms.length, lo, 44) : lo; } + function gutG() { return phone ? 30 : 40; } + function pad() { return phone ? 10 : 22; } + function gut() { return lerp(gutG(), phone ? 64 : 128, cam.k); } + function plotW() { return CW - gut() - pad(); } + function minSpan(len) { return Math.min(len, CHROM_BPP * Math.max(200, CW - (phone ? 64 : 128) - pad())); } + function view() { + var span = Math.exp(cam.ls); + return [cam.c - span / 2, cam.c + span / 2, span]; + } + function gx(bp) { return gutG() + bp * (CW - gutG() - pad()) / M.chroms[0].len; } + function ox(bp) { var v = view(); return gut() + (bp - v[0]) / v[2] * plotW(); } + function xAt(bp, open) { return open ? lerp(gx(bp), ox(bp), cam.k) : gx(bp); } + function rowY(i) { return 9 + i * rowH() + rowH() / 2; } + function level() { + if (cam.k < 0.5) { return 'genome'; } + return 'chromosome'; + } + // The lanes of an open chromosome, top to bottom: y of each, stretched to the canvas. + function lanes() { + var free = Math.max(0, CH - 230), st = 1 + Math.min(0.75, free / 400); + var fy = 118, fh = 50 * Math.min(1.2, st), ry = fy + fh + 14, rh = 22 * st; + return { over: 10, ruler: 46, bands: 72, bh: 38, fy: fy, fh: fh, ry: ry, rh: rh }; + } + + // --- the canvas size, the hatch pattern --- + function size() { + var w = Math.max(280, Math.floor(stage.clientWidth || 680)); + phone = w < 520; + var n = M ? M.chroms.length : 24; + var h = 18 + n * (phone ? ROW_PHONE : ROW_MIN); + DPR = Math.min(2, window.devicePixelRatio || 1); + if (w !== CW || h !== CH) { + CW = w; CH = h; + canvas.width = Math.round(w * DPR); + canvas.height = Math.round(h * DPR); + canvas.style.setProperty('width', w + 'px'); + canvas.style.setProperty('height', h + 'px'); + cacheKey = ''; + } + } + function pattern(alpha) { + var p = document.createElement('canvas'); + p.width = p.height = 6; + var c = p.getContext('2d'); + c.strokeStyle = 'rgba(255,255,255,' + alpha + ')'; + c.lineWidth = 1; + c.beginPath(); c.moveTo(-1, 7); c.lineTo(7, -1); c.moveTo(5, 7); c.lineTo(7, 5); c.moveTo(-1, 1); c.lineTo(1, -1); c.stroke(); + return ctx.createPattern(p, 'repeat'); + } + + // --- drawing --- + function rounded(c, x, y, w, h, r) { + r = Math.max(0, Math.min(r, w / 2, h / 2)); + c.moveTo(x + r, y); c.lineTo(x + w - r, y); c.arcTo(x + w, y, x + w, y + r, r); c.lineTo(x + w, y + h - r); + c.arcTo(x + w, y + h, x + w - r, y + h, r); c.lineTo(x + r, y + h); c.arcTo(x, y + h, x, y + h - r, r); + c.lineTo(x, y + r); c.arcTo(x, y, x + r, y, r); + } + // One chromosome's body: two rounded arms pinched at the centromere, filled with its bands, the masked + // spans hatched. X maps a bp to a canvas x. + function body(c, ch, X, yc, h, alpha) { + var cen = ch.cen ? (ch.cen[0] + ch.cen[1]) / 2 : null; + c.save(); + c.globalAlpha = alpha; + c.beginPath(); + var x0 = X(0), x1 = X(ch.len), xm = cen === null ? null : X(cen); + if (xm === null) { rounded(c, x0, yc - h / 2, x1 - x0, h, h / 2); } else { + rounded(c, x0, yc - h / 2, Math.max(1, xm - x0 - 0.6), h, h / 2); + rounded(c, xm + 0.6, yc - h / 2, Math.max(1, x1 - xm - 0.6), h, h / 2); + } + c.clip(); + c.fillStyle = STAIN.gneg; + c.fillRect(x0, yc - h / 2, x1 - x0, h); + var b = M.bands[ch.n]; + if (b) { + for (var i = 0; i < b.n; i++) { + var st = b.o.stain[i]; + if (st === 'gneg') { continue; } + var a = X(b.s[i]), z = X(b.e[i]); + if (z < 0 || a > CW) { continue; } + c.fillStyle = STAIN[st] || STAIN.gneg; + c.fillRect(a, yc - h / 2, Math.max(0.5, z - a), h); + } + } + c.fillStyle = hatch; + (ch.masked || []).forEach(function (m) { var a = X(m[0]), z = X(m[1]); c.fillRect(a, yc - h / 2, Math.max(1, z - a), h); }); + c.restore(); + } + function glyph(c, lv, x, y, r) { + c.save(); + c.fillStyle = LV[lv] || LV.info; + c.strokeStyle = PAGE; + c.lineWidth = 3; + c.beginPath(); + if (lv === 'act') { c.moveTo(x, y - r); c.lineTo(x + r, y); c.lineTo(x, y + r); c.lineTo(x - r, y); c.closePath(); c.stroke(); c.fill(); } + else if (lv === 'medicine') { rounded(c, x - r * 1.3, y - r * 0.7, r * 2.6, r * 1.4, r * 0.7); c.stroke(); c.fill(); } + else if (lv === 'review') { c.moveTo(x, y - r); c.lineTo(x + r, y + r * 0.8); c.lineTo(x - r, y + r * 0.8); c.closePath(); c.stroke(); c.fill(); } + else { + var two = lv === 'carrier', rr = two ? r * 0.75 : r * 0.9; + var at = two ? [x - rr * 0.75, x + rr * 0.75] : [x]; + at.forEach(function (cx) { c.moveTo(cx + rr, y); c.arc(cx, y, rr, 0, Math.PI * 2); }); + c.stroke(); + c.strokeStyle = LV[lv] || LV.info; + c.lineWidth = 1.5; + c.beginPath(); + at.forEach(function (cx) { c.moveTo(cx + rr, y); c.arc(cx, y, rr, 0, Math.PI * 2); }); + c.stroke(); + } + c.restore(); + } + function label(c, text, x, y, font, colour) { + c.font = font; + c.lineJoin = 'round'; + c.strokeStyle = PAGE; + c.lineWidth = 4; + c.strokeText(text, x, y); + c.fillStyle = colour; + c.fillText(text, x, y); + } + var widths = {}; + function measure(font, text) { + var k = font + '\u0000' + text; + if (!(k in widths)) { ctx.font = font; widths[k] = ctx.measureText(text).width; } + return widths[k]; + } + + // The genome rows, drawn once into an offscreen canvas until the data, the open results or the size change. + function genomeRows() { + var k = [CW, CH, DPR, drawn().map(function (r) { return r.id; }).join(',')].join('|'); + if (cache && k === cacheKey) { return cache; } + cache = cache || document.createElement('canvas'); + cache.width = canvas.width; + cache.height = canvas.height; + var c = cache.getContext('2d'); + c.setTransform(DPR, 0, 0, DPR, 0, 0); + c.clearRect(0, 0, CW, CH); + var fs = drawn(); + var occupied = {}; + M.chroms.forEach(function (ch, i) { + var yc = rowY(i); + c.fillStyle = INK3; + c.font = '11px ' + SANS; + c.textAlign = 'right'; + c.textBaseline = 'middle'; + c.fillText(short(ch.n), gutG() - 10, yc); + c.textAlign = 'left'; + body(c, ch, gx, yc, 8, 1); + // Runs of homozygosity under the body: the long ones solid, the rest as 1 Mb coverage bins. + var bins = M.bins[ch.n], ry = yc + 6.5; + if (bins) { + c.fillStyle = INK2; + for (var b = 0; b < bins.length; b++) { + if (!bins[b]) { continue; } + c.globalAlpha = 0.22 * Math.min(1, bins[b] / M.binSize); + c.fillRect(gx(b * M.binSize), ry - 1, Math.max(1, gx((b + 1) * M.binSize) - gx(b * M.binSize)), 2); + } + c.globalAlpha = 1; + } + var rr = M.roh[ch.n]; + if (rr) { + c.strokeStyle = INK; + c.lineWidth = 2.5; + for (var j = 0; j < rr.n; j++) { + if (!rr.o.long[j]) { continue; } + c.beginPath(); c.moveTo(gx(rr.s[j]), ry); c.lineTo(gx(rr.e[j]), ry); c.stroke(); + } + } + occupied[i] = []; + }); + // Findings: glyphs pinned to their row, names to the right by level (act first), never over another + // glyph or name of the row: a name that does not fit is left to the margin and the list. + var byLevel = fs.slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || a.s - b.s; }); + byLevel.forEach(function (r) { + var i = M.by[r.chrom], yc = rowY(i), x = gx((r.s + r.e) / 2); + if (gx(r.e) - gx(r.s) > 3) { + c.fillStyle = LV[r.level] || LV.info; + c.fillRect(gx(r.s), yc + 4.5, gx(r.e) - gx(r.s), 2); + } + glyph(c, r.level, x, yc - 1, phone ? 3.5 : 4.5); + occupied[i].push([x - 6, x + 6]); + }); + byLevel.forEach(function (r) { + var i = M.by[r.chrom], yc = rowY(i), x = gx((r.s + r.e) / 2); + var name = r.gene || ''; + if (!name) { return; } + var font = '500 10.5px ' + SANS, w = measure(font, name), lx = x + 8; + var clash = occupied[i].some(function (o) { return !(lx + w < o[0] || lx > o[1]) && !(o[0] === x - 6 && o[1] === x + 6); }); + if (lx + w > CW - 2 || clash) { return; } + occupied[i].push([lx - 2, lx + w + 4]); + c.textBaseline = 'middle'; + label(c, name, lx, yc - 1, font, INK2); + }); + cacheKey = k; + return cache; + } + + function draw() { + raf = 0; + if (!M) { return; } + var t = performance.now(); + size(); // the page may have changed width since the last frame (a phone turned, a fold opened) + step(); + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.setTransform(DPR, 0, 0, DPR, 0, 0); + hits = []; + if (cam.k < 0.999) { drawGenome(); } + if (cam.k > 0.001) { drawOpen(); } + if (!fig.hasAttribute('data-pgp-map-paint-ms') && t0) { + fig.setAttribute('data-pgp-map-paint-ms', String(Math.round(performance.now() - t0))); + } + if (moving()) { schedule(); } else { settle(); } + return t; + } + function drawGenome() { + var k = cam.k, n = M.chroms.length; + if (k < 0.001) { + if (hover.row >= 0 && level() === 'genome') { + ctx.fillStyle = 'rgba(255,255,255,.035)'; + ctx.fillRect(0, rowY(hover.row) - rowH() / 2, CW, rowH()); + var ch = M.chroms[hover.row], words = 'Open chromosome ' + short(ch.n) + ' ›'; + var font = '500 11px ' + SANS, w = measure(font, words), x = gx(ch.len) + 12; + if (x + w < CW - 4) { ctx.textBaseline = 'middle'; label(ctx, words, x, rowY(hover.row), font, BRASS); } + } + ctx.drawImage(genomeRows(), 0, 0, CW, CH); + M.chroms.forEach(function (c, i) { + hits.push({ kind: 'row', i: i, x0: 0, x1: CW, y0: rowY(i) - rowH() / 2, y1: rowY(i) + rowH() / 2 }); + }); + drawn().forEach(function (r) { + hits.push({ kind: 'finding', r: r, x: gx((r.s + r.e) / 2), y: rowY(M.by[r.chrom]) - 1, rad: 7 }); + }); + return; + } + // The motion: the other rows slide away from the one opening and fade. + var a = Math.pow(1 - k, 1.4); + for (var i = 0; i < n; i++) { + if (i === cam.ci) { continue; } + var dy = k * CH * 0.9 * (0.6 + 0.04 * Math.abs(i - cam.ci)) * (i < cam.ci ? -1 : 1); + ctx.save(); + ctx.globalAlpha = a; + ctx.translate(0, dy); + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillText(short(M.chroms[i].n), gutG() - 10, rowY(i)); + ctx.textAlign = 'left'; + body(ctx, M.chroms[i], gx, rowY(i), 8, 1); + ctx.restore(); + } + } + function niceStep(span, px) { + var raw = span / Math.max(1, px / 90), p = Math.pow(10, Math.floor(Math.log10(raw))); + var m = raw / p; + return (m < 1.5 ? 1 : m < 3.5 ? 2 : m < 7.5 ? 5 : 10) * p; + } + function drawOpen() { + var k = cam.k, ch = chrom(), L = lanes(), v = view(), g = gut(), W = plotW(); + var X = function (bp) { return xAt(bp, true); }; + var yb = lerp(rowY(cam.ci), L.bands + L.bh / 2, k), hb = lerp(8, L.bh, k); + ctx.save(); + // The open row's label moves into the gutter. + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillText(k > 0.5 ? 'Bands' : short(ch.n), lerp(gutG() - 10, g - 12, k), yb); + ctx.restore(); + ctx.save(); + ctx.beginPath(); + ctx.rect(lerp(0, g - 2, k), 0, CW, CH); + ctx.clip(); + body(ctx, ch, X, yb, hb, 1); + ctx.restore(); + ctx.save(); + ctx.globalAlpha = k; + // Overview: the whole chromosome, the view as a brass window, a funnel to the ruler. + var ow = W, oX = function (bp) { return g + bp / ch.len * ow; }; + body(ctx, ch, oX, L.over + 6, 8, 0.8); + ctx.strokeStyle = BRASS; + ctx.lineWidth = 1.5; + var wa = oX(clamp(v[0], 0, ch.len)), wz = oX(clamp(v[1], 0, ch.len)); + ctx.strokeRect(wa, L.over, Math.max(3, wz - wa), 12); + ctx.globalAlpha = k * 0.35; + ctx.beginPath(); ctx.moveTo(wa, L.over + 12); ctx.lineTo(g, L.ruler); ctx.moveTo(wz, L.over + 12); ctx.lineTo(g + W, L.ruler); ctx.stroke(); + ctx.globalAlpha = k; + // The ruler, in Mb. + ctx.strokeStyle = 'rgba(255,255,255,.14)'; + ctx.lineWidth = 1; + ctx.beginPath(); ctx.moveTo(g, L.ruler); ctx.lineTo(g + W, L.ruler); ctx.stroke(); + var stp = niceStep(v[2], W); + ctx.fillStyle = INK3; + ctx.font = '10.5px ' + SANS; + ctx.textAlign = 'center'; + ctx.textBaseline = 'top'; + for (var at = Math.ceil(v[0] / stp) * stp; at <= v[1]; at += stp) { + var x = ox(at); + ctx.beginPath(); ctx.moveTo(x, L.ruler); ctx.lineTo(x, L.ruler + 5); ctx.stroke(); + ctx.fillText(mb(at, stp < 1e6 ? 1 : 0) + ' Mb', x, L.ruler + 8); + } + // Band names where they fit. + var b = M.bands[ch.n]; + if (b) { + ctx.font = '10px ' + SANS; + ctx.textBaseline = 'middle'; + for (var i = first(b, v[0]); i < b.n && b.s[i] < v[1]; i++) { + var a = ox(b.s[i]), z = ox(b.e[i]), nm = b.o.name[i]; + if (nm && z - a > measure('10px ' + SANS, nm) + 6) { + ctx.fillStyle = ['gpos75', 'gpos100'].indexOf(b.o.stain[i]) >= 0 ? INK : INK2; + ctx.fillText(nm, (Math.max(a, g) + Math.min(z, g + W)) / 2, L.bands + L.bh / 2); + } + hits.push({ kind: 'band', x0: a, x1: z, y0: L.bands, y1: L.bands + L.bh, name: nm, s: b.s[i], e: b.e[i] }); + } + } + (ch.masked || []).forEach(function (m) { + hits.push({ kind: 'masked', x0: ox(m[0]), x1: ox(m[1]), y0: L.bands, y1: L.bands + L.bh, s: m[0], e: m[1] }); + }); + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.fillText('Findings', g - 12, L.fy + 16); + ctx.fillText(phone ? 'Runs' : 'Homozygous runs', g - 12, L.ry + L.rh / 2); + ctx.textAlign = 'left'; + ctx.save(); + ctx.beginPath(); ctx.rect(g - 6, 0, W + 12, CH); ctx.clip(); + openFindings(L, v, ch); + openRoh(L, v, ch); + // The reading line: a dashed brass line through every lane at the centre, its place in a pill. + var cx = g + W / 2; + ctx.setLineDash([3, 3]); + ctx.strokeStyle = BRASS; + ctx.globalAlpha = k * 0.8; + ctx.beginPath(); ctx.moveTo(cx, L.ruler + 22); ctx.lineTo(cx, L.ry + L.rh + 6); ctx.stroke(); + ctx.setLineDash([]); + ctx.restore(); + var pill = mb(cam.c, 1) + ' Mb', pw = measure('600 10.5px ' + SANS, pill) + 12; + ctx.fillStyle = BRASS; + rounded(ctx, cx - pw / 2, L.ruler - 20, pw, 15, 7.5); + ctx.beginPath(); rounded(ctx, cx - pw / 2, L.ruler - 20, pw, 15, 7.5); ctx.fill(); + ctx.fillStyle = PAGE; + ctx.font = '600 10.5px ' + SANS; + ctx.textAlign = 'center'; + ctx.fillText(pill, cx, L.ruler - 12.5); + ctx.restore(); + } + function laneWords(c, text, x, y) { + c.fillStyle = INK3; + c.font = 'italic 11.5px ' + SANS; + c.textAlign = 'left'; + c.textBaseline = 'middle'; + c.fillText(text, x, y); + } + function openFindings(L, v, ch) { + var lay = M.layers.findings || {}, g = gut(); + if (!landed(lay)) { laneWords(ctx, layerWords(lay), g + 4, L.fy + 16); return; } + var fs = drawn().filter(function (r) { return r.chrom === ch.n; }); + var inView = fs.filter(function (r) { return r.e > v[0] && r.s < v[1]; }); + if (!inView.length) { + laneWords(ctx, fs.length ? 'No finding in this stretch.' : noFindingHere(), g + 4, L.fy + 16); + return; + } + var used = []; + inView.sort(function (a, b) { return PRI[a.level] - PRI[b.level] || a.s - b.s; }).forEach(function (r) { + var x = ox((r.s + r.e) / 2), y = L.fy + 14; + if (r.e - r.s > v[2] / 2000) { + ctx.fillStyle = LV[r.level] || LV.info; + ctx.fillRect(ox(r.s), y + 9, Math.max(2, ox(r.e) - ox(r.s)), 2); + } + ctx.strokeStyle = LV[r.level] || LV.info; + ctx.globalAlpha = 0.5; + ctx.lineWidth = 1.2; + ctx.beginPath(); ctx.moveTo(x, y + 8); ctx.lineTo(x, L.bands + L.bh); ctx.stroke(); + ctx.globalAlpha = cam.k; + glyph(ctx, r.level, x, y, 6); + hits.push({ kind: 'finding', r: r, x: x, y: y, rad: 9 }); + var name = r.gene || r.title, words = U.LEVEL_WORDS[r.level] || r.level; + var w = Math.max(measure('500 12px ' + SANS, name), measure('10.5px ' + SANS, words)); + var lx = x + 11; + if (lx + w > CW - 4 || used.some(function (u) { return !(lx + w < u[0] || lx > u[1]); })) { return; } + used.push([lx - 4, lx + w + 6]); + ctx.textBaseline = 'middle'; + label(ctx, name, lx, y - 1, '500 12px ' + SANS, INK); + label(ctx, words, lx, y + 13, '10.5px ' + SANS, INK3); + }); + } + function openRoh(L, v, ch) { + var lay = M.layers.roh || {}, g = gut(), y = L.ry + L.rh / 2; + if (!landed(lay)) { laneWords(ctx, layerWords(lay), g + 4, y); return; } + var r = M.roh[ch.n]; + if (!r || !r.n) { laneWords(ctx, 'No run of homozygosity on this chromosome.', g + 4, y); return; } + for (var i = first(r, v[0]); i < r.n && r.s[i] < v[1]; i++) { + var a = ox(r.s[i]), z = ox(r.e[i]), art = M.art[r.o.art[i]] || ''; + if (r.o.long[i]) { + ctx.fillStyle = INK2; + ctx.fillRect(a, y - 3, Math.max(1, z - a), 6); + var t = mb(r.e[i] - r.s[i]) + ' Mb run of homozygosity', font = '11px ' + SANS; + if (measure(font, t) + 8 > z - a) { t = mb(r.e[i] - r.s[i]) + ' Mb'; } + if (measure(font, t) + 8 < z - a) { ctx.textBaseline = 'middle'; label(ctx, t, a + 4, y - 11, font, INK2); } + } else if (art) { + ctx.fillStyle = hatch; + ctx.fillRect(a, y - 3, Math.max(1, z - a), 6); + } else { + ctx.fillStyle = 'rgba(195,194,183,.35)'; + ctx.fillRect(a, y - 0.5, Math.max(1, z - a), 1); + } + hits.push({ kind: 'roh', x0: a, x1: Math.max(a + 3, z), y0: y - 6, y1: y + 6, s: r.s[i], e: r.e[i], art: art, long: r.o.long[i] }); + } + } + function noFindingHere() { + return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; + } + function closedAny() { return M.veilIds.some(function (id) { return stateOf(id) === 'closed'; }); } + function layerWords(lay) { + if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } + if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } + if (lay.state === 'not_run') { return 'Not on the map: step ' + lay.step + ' was not part of this run.'; } + if (lay.state === 'not_in_this_run') { return 'Not on the map: not part of this run.'; } + if (lay.state === 'running' || lay.state === 'waiting') { return 'Not on the map yet: step ' + lay.step + ' has not ended.'; } + if (lay.state === 'failed') { return 'Not on the map: step ' + lay.step + ' failed.'; } + if (lay.state === 'skipped') { return 'Not on the map: step ' + lay.step + ' was skipped.'; } + return lay.note || ''; + } + + // --- the camera --- + function moving() { + return Math.abs(cam.k - tgt.k) > 0.002 || Math.abs(cam.ls - tgt.ls) > 0.002 || Math.abs(cam.c - tgt.c) > Math.exp(cam.ls) * 0.0005; + } + var last = 0; + function step() { + var now = performance.now(), dt = last ? Math.min(64, now - last) : 16; + last = now; + if (reduced || cam.ci !== tgt.ci && cam.k < 0.01) { cam.ci = tgt.ci; } + if (reduced) { cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; return; } + if (cam.ci !== tgt.ci) { + // Another chromosome: close the open one first, then open the new one. + cam.k += (0 - cam.k) * (1 - Math.exp(-dt / 60)); + if (cam.k < 0.01) { cam.k = 0; cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; } + return; + } + cam.c += (tgt.c - cam.c) * (1 - Math.exp(-dt / 75)); + cam.ls += (tgt.ls - cam.ls) * (1 - Math.exp(-dt / 75)); + cam.k += (tgt.k - cam.k) * (1 - Math.exp(-dt / 95)); + if (!moving()) { cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; } + } + function schedule() { if (!raf) { raf = window.requestAnimationFrame(function () { last = last || performance.now(); draw(); }); } } + function clampTgt() { + var ch = M.chroms[tgt.ci], span = clamp(Math.exp(tgt.ls), minSpan(ch.len), ch.len); + tgt.ls = Math.log(span); + tgt.c = clamp(tgt.c, span / 2, ch.len - span / 2); + } + function openChrom(i, c, span, push) { + if (!M || i < 0 || i >= M.chroms.length) { return; } + var ch = M.chroms[i]; + tgt.ci = i; + tgt.k = 1; + tgt.c = c === undefined ? ch.len / 2 : c; + tgt.ls = Math.log(span === undefined ? ch.len : span); + clampTgt(); + if (cam.k < 0.01) { cam.ci = i; cam.c = tgt.c; cam.ls = tgt.ls; } + changed(push); + } + function closeChrom(push) { + tgt.k = 0; + changed(push); + } + function changed(push) { + touched = true; + last = 0; + schedule(); + writeHash(push); + } + + // --- the address: #genome, #chr7, #chr7:117.4-137.4M; never a closed result's id, gene or place --- + function locusWords() { + if (tgt.k < 0.5) { return 'genome'; } + var ch = M.chroms[tgt.ci], span = Math.exp(tgt.ls); + if (span >= ch.len * 0.999) { return ch.n; } + return ch.n + ':' + mb(tgt.c - span / 2) + '-' + mb(tgt.c + span / 2) + 'M'; + } + function writeHash(push) { + if (!M || !window.history || !window.history.replaceState) { return; } + window.clearTimeout(hashT); + var go = function () { + var h = '#' + locusWords(); + if (window.location.hash === h) { return; } + if (push) { window.history.pushState(null, '', h); } else { window.history.replaceState(null, '', h); } + }; + if (push) { go(); } else { hashT = window.setTimeout(go, 250); } + } + // A place in the address: 'genome', 'chr7', 'chr7:117.4-117.7M', 'chr7:120k-140k', 'chr7:117480026-117668665' + // (1-based, inclusive), or the aliases view=genome, view=chromosome&chr=7, chr=7. Returns null when the + // address is not a map place, false when it is one that cannot be read. + function parsePlace(h) { + var m, kv = {}; + if (/^genome$/i.test(h)) { return { k: 0 }; } + if (/^(view|chr)=/.test(h)) { + h.split('&').forEach(function (p) { var x = p.split('='); kv[x[0]] = decodeURIComponent(x[1] || ''); }); + if (kv.view === 'genome') { return { k: 0 }; } + if (!kv.chr) { return kv.view ? false : null; } + h = 'chr' + String(kv.chr).replace(/^chr/i, ''); + } + if (!/^chr/i.test(h)) { return null; } + var name = 'chr' + h.slice(3).split(':')[0].toUpperCase().replace(/^0+/, ''); + if (!(name in M.by)) { return false; } + var ch = M.chroms[M.by[name]], rest = h.indexOf(':') > 0 ? h.slice(h.indexOf(':') + 1) : ''; + if (!rest) { return { k: 1, ci: M.by[name] }; } + if ((m = /^([0-9.]+)-([0-9.]+)([MmKk])$/.exec(rest))) { + var u = /m/i.test(m[3]) ? 1e6 : 1e3, a = Number(m[1]) * u, z = Number(m[2]) * u; + if (!(z > a)) { return false; } + return { k: 1, ci: M.by[name], s: a, e: z }; + } + if ((m = /^([0-9,]+)-([0-9,]+)$/.exec(rest))) { + var s = Number(m[1].replace(/,/g, '')) - 1, e = Number(m[2].replace(/,/g, '')); + if (!(e > s)) { return false; } + return { k: 1, ci: M.by[name], s: s, e: e }; + } + void ch; + return false; + } + function readHash(scroll) { + if (!M) { return; } + var h; + try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return; } + if (!h) { return; } + var p = parsePlace(h); + if (p === null) { return; } + if (p === false) { say('Could not read the place “' + h + '”'); return; } + if (p.k === 0) { tgt.k = 0; } else { + var ch = M.chroms[p.ci]; + openChrom(p.ci, p.s === undefined ? ch.len / 2 : (p.s + p.e) / 2, p.s === undefined ? ch.len : p.e - p.s, false); + window.clearTimeout(hashT); + } + if (scroll) { + cam.ci = tgt.ci; cam.k = tgt.k; cam.c = tgt.c; cam.ls = tgt.ls; + fig.scrollIntoView({ block: 'start' }); + } + schedule(); + } + function say(words) { + toast.textContent = words; + toast.hidden = false; + window.clearTimeout(say.t); + say.t = window.setTimeout(function () { toast.hidden = true; }, 4000); + } + + // --- the margin: settled, not per frame --- + function settle() { + window.clearTimeout(settleT); + settleT = window.setTimeout(describe, 140); + } + function describe() { + if (!M) { return; } + var lv = level(), fs = drawn(), ch = chrom(); + fig.setAttribute('data-level', lv); + fig.setAttribute('data-chrom', lv === 'genome' ? '' : ch.n); + fig.setAttribute('data-view', locusWords()); + stopG.setAttribute('aria-pressed', String(lv === 'genome')); + stopC.setAttribute('aria-pressed', String(lv !== 'genome')); + crumb.textContent = ''; + crumb.hidden = lv === 'genome'; // the gauge already says Genome + var gl = el('button', { type: 'button', 'class': 'gm-crumb-g' }, 'Genome'); + gl.addEventListener('click', function () { closeChrom(true); }); + add(crumb, gl); + var nY = M.chroms.length; + if (lv === 'genome') { + title.textContent = M.label + '’s genome, all ' + nY + ' chromosomes'; + kicker.textContent = 'Figure 1 · Genome map · ' + M.label + '’s genome'; + mkick.textContent = 'The whole genome'; + mline.textContent = ''; + } else { + title.textContent = 'Chromosome ' + short(ch.n); + kicker.textContent = 'Figure 1 · Genome map · ' + M.label; + add(crumb, el('span', { 'class': 'sep', 'aria-hidden': 'true' }, ' › '), el('span', { 'aria-current': 'location' }, ch.n)); + mkick.textContent = 'Under the line'; + mline.textContent = ch.n + ' · ' + mb(cam.c, 2) + ' Mb' + bandAt(ch, cam.c); + } + canvas.setAttribute('aria-label', lv === 'genome' ? 'The genome map: ' + nY + ' chromosomes with ' + fs.length + ' findings drawn' + : 'Chromosome ' + short(ch.n) + ' open on the genome map'); + narr.textContent = ''; + var v = view(); + var shown = lv === 'genome' ? fs : fs.filter(function (r) { return r.chrom === ch.n && r.e > v[0] && r.s < v[1]; }); + var forms = (M.text.findings || {}); + if (lv === 'genome') { + var form = fs.length === 0 ? forms.none : fs.length === 1 ? forms.one : forms.many; + add(narr, runs((form || []).map(function (x) { return { t: x.t.replace('{n}', String(fs.length)), b: x.b }; }))); + } else if (M.text.chrom && M.text.chrom[ch.n]) { + add(narr, runs(M.text.chrom[ch.n])); + } + var lay = M.layers.findings || {}; + if (!landed(lay)) { + add(narr, el('p', {}, layerWords(lay))); + } else if (shown.length && !(phone && lv === 'genome')) { + // On a phone the genome level leaves the list to the sections under it: the page is long already. + var list = shown.slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || M.by[a.chrom] - M.by[b.chrom] || a.s - b.s; }); + var p = el('p', { 'class': 'gm-phrases' }); + var cap = phone ? 3 : 6; + list.slice(0, cap).forEach(function (r, i) { if (i) { add(p, ' '); } add(p, r.phrase); }); + if (list.length > cap) { add(p, ' And ' + (list.length - cap) + ' more on the map.'); } + add(narr, p); + } else if (lv !== 'genome') { + add(narr, el('p', {}, fs.some(function (r) { return r.chrom === ch.n; }) ? 'No finding in this stretch.' : noFindingHere())); + } + if (lv === 'genome') { + add(narr, runs(M.text.genome)); + if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Click a chromosome to open it.')); } + } else { + var r = M.roh[ch.n], n = 0, longest = 0; + if (r) { for (var i = first(r, v[0]); i < r.n && r.s[i] < v[1]; i++) { n++; longest = Math.max(longest, r.e[i] - r.s[i]); } } + if (n && landed(M.layers.roh || {})) { + add(narr, el('p', {}, 'Runs of homozygosity in view: ' + n + ', the longest ' + mb(longest) + ' Mb.')); + } + if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Drag to move along it, scroll to zoom, Esc for the whole genome.')); } + } + drawLayers(); + drawMarks(); + } + function bandAt(ch, bp) { + var b = M.bands[ch.n]; + if (!b) { return ''; } + for (var i = first(b, bp); i < b.n && b.s[i] <= bp; i++) { + if (b.e[i] > bp) { return ' · band ' + short(ch.n) + b.o.name[i]; } + } + return ''; + } + function drawLayers() { + layersBox.textContent = ''; + [['findings', 'Findings'], ['roh', 'Runs of homozygosity']].forEach(function (x) { + var lay = M.layers[x[0]]; + var row = el('div', { 'class': 'gm-layer', 'data-layer': x[0], 'data-state': lay ? lay.state : 'missing' }); + var dd = el('dd', {}); + if (!lay) { add(dd, 'Not on the map: this server’s map has no ' + x[1].toLowerCase() + ' layer.'); } + else if (!landed(lay)) { add(dd, layerWords(lay)); } + else { + var n = x[0] === 'findings' ? drawn().length : lay.count; + add(dd, n === 0 && lay.empty ? lay.empty : n + ' on the map. '); + if (lay.note && n !== 0) { add(dd, lay.note); } + if (lay.caveat) { + var d = el('details', { 'class': 'gm-caveat' }); + add(d, el('summary', {}, 'Caveat'), el('p', {}, lay.caveat)); + add(dd, d); + } + } + add(row, el('dt', {}, x[1] + (lay && lay.step ? ' · step ' + lay.step : '')), dd); + add(layersBox, row); + }); + } + function drawMarks() { + marks.textContent = ''; + var lv = level(), ch = chrom(), v = view(); + drawn().filter(function (r) { return lv === 'genome' || (r.chrom === ch.n && r.e > v[0] && r.s < v[1]); }) + .sort(function (a, b) { return M.by[a.chrom] - M.by[b.chrom] || a.s - b.s; }).forEach(function (r) { + var a = el('a', { href: r.href, 'data-id': r.id, 'data-level': r.level, 'data-chrom': r.chrom, + 'data-s': String(r.s), 'data-e': String(r.e) }, (U.LEVEL_WORDS[r.level] || r.level) + ': ' + r.title + ', ' + r.chrom); + add(marks, add(el('li', {}), a)); + }); + } + function drawKey() { + keyBox.textContent = ''; + add(keyBox, el('div', { 'class': 'fig-label' }, 'Key')); + ['act', 'medicine', 'review', 'carrier', 'info'].forEach(function (lv) { add(keyBox, U.level(lv)); }); + add(keyBox, add(el('span', { 'class': 'gm-k-roh' }), el('span', { 'class': 'sw', 'aria-hidden': 'true' }), 'Run of homozygosity, 5 Mb or more')); + add(keyBox, add(el('span', { 'class': 'gm-k-mask' }), el('span', { 'class': 'sw', 'aria-hidden': 'true' }), 'Hatched: centromere or heterochromatin, few reads map there')); + } + + // --- the closed results: one dashed chip per kind, and the opened ones --- + function drawChips() { + chips.textContent = ''; + if (!M) { return; } + var acts = [], gates = [], opened = []; + M.veilIds.forEach(function (id) { + var s = stateOf(id); + if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } + }); + if (acts.length) { + var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); + add(c, el('span', {}, (acts.length === 1 ? '1 result closed' : acts.length + ' results closed') + ' · Not drawn until opened on purpose')); + add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); })); + add(chips, c); + } + gates.forEach(function (id) { + var g = M.gated[id], name = g.gene || 'This result'; + var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); + add(c, el('span', {}, name + ' closed · Not drawn until opened on purpose')); + add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); })); + add(chips, c); + }); + opened.forEach(function (id) { + var r = M.findings.filter(function (x) { return x.id === id; })[0]; + var c = el('div', { 'class': 'gm-chip opened', 'data-opened': '1' }); + add(c, el('span', {}, 'Opened: ' + (r ? r.title : 'a result to act on'))); + add(c, button('Close it again', function () { confirming = null; opts.close(id); })); + add(chips, c); + }); + } + function confirmBox(id, gated) { + var box = el('span', { 'class': 'gm-confirm' }); + add(box, el('span', {}, gated ? 'It is kept behind its warning on every page. Open it here and in its section?' + : 'A result to act on in adult life is usually not looked at before the child can choose. Open it here and in its section?')); + add(box, button('Show the result', function () { confirming = null; opts.open(id); }, 'gm-yes')); + add(box, button('Keep it closed', function () { confirming = null; drawChips(); }, 'gm-no')); + return box; + } + function button(text, fn, cls) { + var b = el('button', { type: 'button', 'class': cls || '' }, text); + b.addEventListener('click', fn); + return b; + } + + // --- input --- + function at(e) { var r = canvas.getBoundingClientRect(); return [e.clientX - r.left, e.clientY - r.top]; } + function hitAt(x, y) { + var best = null; + for (var i = hits.length - 1; i >= 0; i--) { + var h = hits[i]; + if (h.kind === 'finding') { if (Math.abs(h.x - x) <= h.rad && Math.abs(h.y - y) <= h.rad) { return h; } } + else if (x >= h.x0 && x <= h.x1 && y >= h.y0 && y <= h.y1) { + if (!best || (h.kind === 'masked' && best.kind === 'band') || (h.kind === 'roh')) { best = h; } + } + } + return best; + } + function tipFor(h) { + var ch = M.chroms[cam.ci]; + if (h.kind === 'finding') { return [h.r.phrase, 'Click for the finding’s page']; } + if (h.kind === 'band') { return ['Band ' + short(ch.n) + (h.name || ''), ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e)]; } + if (h.kind === 'masked') { + return ['Not readable here · ' + ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e), + 'Centromere, heterochromatin or acrocentric arm: few reads map, so layers are cut here. Not missing data.']; + } + if (h.kind === 'roh') { + return [mb(h.e - h.s) + ' Mb run of homozygosity' + (h.art ? ' (' + h.art + ': not a real run)' : ''), ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e)]; + } + if (h.kind === 'row') { return null; } + return null; + } + // A hidden tooltip keeps no words: it may have named a result that is closed now. + function hideTip() { tip.hidden = true; tip.textContent = ''; } + function showTip(h, x, y) { + var t = h && tipFor(h); + if (!t) { hideTip(); return; } + tip.textContent = ''; + add(tip, el('b', {}, t[0])); + if (t[1]) { add(tip, el('span', {}, t[1])); } + tip.hidden = false; + tip.style.setProperty('left', Math.min(x + 12, Math.max(0, CW - 260)) + 'px'); + tip.style.setProperty('top', (y + 14) + 'px'); + } + var drag = null; + canvas.addEventListener('pointermove', function (e) { + if (!M) { return; } + var p = at(e); + if (drag && level() !== 'genome') { + var v = view(); + tgt.c = drag.c - (p[0] - drag.x) / plotW() * v[2]; + clampTgt(); + cam.c = tgt.c; + drag.moved = drag.moved || Math.abs(p[0] - drag.x) > 3; + changed(false); + return; + } + var h = hitAt(p[0], p[1]); + var row = level() === 'genome' && h ? (h.kind === 'row' ? h.i : h.kind === 'finding' ? M.by[h.r.chrom] : -1) : -1; + if (row !== hover.row) { hover.row = row; schedule(); } + hover.hit = h; + canvas.style.setProperty('cursor', h && (h.kind === 'finding' || h.kind === 'row') ? 'pointer' : level() === 'genome' ? 'default' : 'grab'); + showTip(h, p[0], p[1]); + }); + canvas.addEventListener('pointerleave', function () { hover.row = -1; hideTip(); schedule(); }); + canvas.addEventListener('pointerdown', function (e) { + if (!M || level() === 'genome' || e.button !== 0) { return; } + drag = { x: at(e)[0], c: tgt.c, moved: false }; + if (canvas.setPointerCapture) { canvas.setPointerCapture(e.pointerId); } + }); + canvas.addEventListener('pointerup', function (e) { + var was = drag; + drag = null; + if (!M || (was && was.moved)) { if (was && was.moved) { writeHash(false); } return; } + var p = at(e), h = hitAt(p[0], p[1]); + if (h && h.kind === 'finding') { window.location.href = h.r.href; return; } + if (level() === 'genome' && h && h.kind === 'row') { openChrom(h.i, undefined, undefined, true); } + }); + canvas.addEventListener('wheel', function (e) { + if (!M || level() === 'genome') { return; } + e.preventDefault(); + var p = at(e), v = view(), ch = chrom(); + var bp = v[0] + (p[0] - gut()) / plotW() * v[2]; + var f = Math.exp(clamp(e.deltaY, -200, 200) * 0.0025); + if (f > 1 && Math.exp(tgt.ls) >= ch.len * 0.999) { closeChrom(true); return; } + var span = clamp(Math.exp(tgt.ls) * f, minSpan(ch.len), ch.len); + tgt.c = bp - (bp - tgt.c) * span / Math.exp(tgt.ls); + tgt.ls = Math.log(span); + clampTgt(); + changed(false); + }, { passive: false }); + canvas.addEventListener('keydown', function (e) { + if (!M || e.altKey || e.ctrlKey || e.metaKey) { return; } + var open = level() !== 'genome', span = Math.exp(tgt.ls); + var done = true; + if (e.key === 'Escape' || e.key === '0') { if (open) { closeChrom(true); } else { done = false; } } + else if (e.key === '[' || e.key === ']') { + var i = (open ? tgt.ci : (e.key === ']' ? -1 : M.chroms.length)) + (e.key === ']' ? 1 : -1); + openChrom((i + M.chroms.length) % M.chroms.length, undefined, undefined, true); + } else if (open && (e.key === '+' || e.key === '=' || e.key === '-' || e.key === '−')) { + tgt.ls = Math.log(span * (e.key === '-' || e.key === '−' ? 1.6 : 1 / 1.6)); + clampTgt(); + changed(false); + } else if (open && (e.key === 'ArrowLeft' || e.key === 'ArrowRight')) { + tgt.c += (e.key === 'ArrowLeft' ? -0.2 : 0.2) * span; + clampTgt(); + changed(false); + } else if (!open && e.key === 'Enter') { openChrom(tgt.ci, undefined, undefined, true); } + else { done = false; } + if (done) { e.preventDefault(); } + }); + stopG.addEventListener('click', function () { closeChrom(true); }); + stopC.addEventListener('click', function () { + if (level() !== 'genome') { openChrom(tgt.ci, undefined, undefined, true); return; } + // From the genome: the chromosome of the first finding by level, else chromosome 1. + var f = drawn().slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || M.by[a.chrom] - M.by[b.chrom]; })[0]; + openChrom(f ? M.by[f.chrom] : 0, undefined, undefined, true); + }); + window.addEventListener('hashchange', function () { readHash(false); }); + window.addEventListener('popstate', function () { readHash(false); }); + if (window.ResizeObserver) { new window.ResizeObserver(function () { if (M) { size(); clampTgt(); schedule(); } }).observe(stage); } + + // --- loading --- + function fail(why, words) { + failure = why; + fig.setAttribute('data-pgp-map', why); + stage.hidden = true; + marks.hidden = true; + fallback.hidden = false; + fallback.textContent = ''; + add(fallback, el('p', { 'class': 'gm-fail' }, words)); + var rows = opts.fallback ? opts.fallback() : []; + if (rows.length) { + add(fallback, U.table({ columns: ['Level', 'Finding', 'Where'], rows: rows }, 12)); + } else { + add(fallback, el('p', { 'class': 'dim' }, 'No finding with a place on the map is open on this page; every finding is listed in its section.')); + } + narr.textContent = ''; + layersBox.textContent = ''; + } + function load(k) { + key = k; + if (typeof window.DecompressionStream === 'undefined' || !window.Response || !window.ReadableStream) { + fail('nodecompress', 'This browser cannot open the map file. The findings with a place are listed here instead.'); + return; + } + fig.setAttribute('data-pgp-map', 'loading'); + fetch(opts.url, { credentials: 'same-origin', cache: 'no-store' }).then(function (r) { + if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(window.location.pathname); return null; } + if (!r.ok) { + return r.text().then(function (t) { throw new Error((t || '').trim() || ('the server answered ' + r.status)); }, function () { throw new Error('the server answered ' + r.status); }); + } + var t1 = performance.now(); + return new Response(r.body.pipeThrough(new window.DecompressionStream('gzip'))).text().then(function (txt) { + var p0 = performance.now(); + var d = JSON.parse(txt); + var m = prepare(d); + var p1 = performance.now(); + return { m: m, parse: p1 - p0, arrived: t1 }; + }); + }).then(function (x) { + if (!x || k !== key) { return; } + var fresh = !M; + M = x.m; + failure = null; + stage.hidden = false; + marks.hidden = false; + fallback.hidden = true; + hatch = hatch || pattern(0.16); + fig.setAttribute('data-pgp-map-parse-ms', String(Math.round(x.parse))); + fig.removeAttribute('data-pgp-map-paint-ms'); + t0 = x.arrived; + size(); + if (fresh) { + tgt.ci = cam.ci = 0; + tgt.c = cam.c = M.chroms[0].len / 2; + tgt.ls = cam.ls = Math.log(M.chroms[0].len); + } + cacheKey = ''; + drawKey(); + drawChips(); + draw(); + describe(); + fig.setAttribute('data-pgp-map', 'drawn'); + if (fresh) { readHash(true); } + }).catch(function (err) { + if (k !== key) { return; } + fail('failed', 'The map could not be loaded: ' + String((err && err.message) || err).split('\n')[0] + + ' The findings with a place are listed here instead.'); + }); + } + + return { + node: fig, + // The page drew again: maybe new results (a new key fetches the document again), maybe a result + // opened or closed (the veil is asked again). + refresh: function (k) { + if (k !== key) { load(k); } + if (failure) { + fail(failure, fallback.firstChild ? fallback.firstChild.textContent : ''); + return; + } + if (!M) { return; } + cacheKey = ''; + hideTip(); // a tooltip of a result closed since leaves the page + drawChips(); + schedule(); + describe(); + } + }; + } + + window.PGPMap = { create: create, prepare: prepare }; +}()); diff --git a/assets/live/person.js b/assets/live/person.js index 6dcd7490f..83ad5a151 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -1,7 +1,7 @@ // person.js: a person's genome as a report, the Journal design (docs/design/journal/), on their page // (assets/live/index.html), from results.json (bin/live_results.py, served by // bin/owngenome_live_results.py) and the run's states (live.js). The page reads top to bottom: the lead -// paragraph built from the findings, the counts by level, the chromosome map of the findings, then ten +// paragraph built from the findings, the counts by level, the genome map of the findings (map.js), then ten // numbered sections: 1 To act on, 2 Medicines, 3 For your family, 4 Worth a look, 5 Can I trust this?, // 6 Origins, 7 Traits and scores, 8 Assessed, no finding, 9 What it cannot see, 10 Sources and files. // A run still going is the same report being written: each section carries its stamp (written, being @@ -21,6 +21,7 @@ // 'none': in no family; 'pending' or 'failed': in one whose roles are not known (yet); 'known'. var familyStatus = 'none'; var shown = {}; // veiled findings the reader opened, by id + var genomeMap = null, resultsKey = null; // Figure 1 (map.js), and the results it was built for var panel = document.getElementById('results'); var rail = document.getElementById('rail'); var NOT_ALL_CLEAR = 'No finding in what these steps assessed.'; @@ -428,8 +429,10 @@ add(text, lineList(lines)); if (c.hidden) { // APOE and the like: behind their warning, opened on purpose. - var d = el('details', { 'class': 'behind' }); + // Its fold is the map's APOE switch too (map.js): opening it here opens APOE on the map, and back. + var d = el('details', { 'class': 'behind', 'data-gate': 'apoe' }); add(d, el('summary', {}, c.hidden.summary), el('p', {}, c.hidden.text)); + d.addEventListener('toggle', function () { if (d.open !== !!shown.apoe) { setShown('apoe', d.open); } }); add(text, d); } if (c.explore && c.explore.length) { @@ -527,6 +530,37 @@ return !!(t && tagClosed(t)); } function openFinding(f) { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } } + // The genome map's veil (map.js): 'closed' while this page keeps the result closed, 'opened' once it was + // opened on purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is + // closed on every page, an adult's too, until opened; a tag naming no finding of these results stays closed. + function mapVeil(id, gated) { + if (gated) { return shown[id] ? 'opened' : 'closed'; } + if (!closedPage()) { return 'open'; } + var f = findingById(id); + if (!f || veiled(f)) { return 'closed'; } + return f.level === 'act' || findingTag(f) ? 'opened' : 'open'; + } + // One switch for a result, from the map's chip or the card's "Show the result" (openFinding): opening it in + // one place opens it in both. Closing it closes the findings that were opened with it. + function setShown(id, on) { + var f = findingById(id); + if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { + delete shown[id]; + ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); + } + redraw(true); + } + // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws + // the run's end results from its first frame). A row with no finding of these results (the map's own APOE + // row) follows its card's fold instead. + function mapNow(id) { var f = findingById(id); var c = f && cardById(f.card); return !f || !!(c && gate(c) === 'show'); } + // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings + // with a place, as a table, each through this page's veil first. + function mapFallback() { + return findingsNow().filter(function (f) { return f.viewer && !veiled(f); }).map(function (f) { + return [U.LEVEL_WORDS[f.level] || f.level, f.title, f.viewer]; + }); + } function findingArticle(f, big) { var row = el('div', { 'class': 'row' }); var art = el('div', { 'class': 'finding', id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); @@ -843,22 +877,16 @@ add(side, note('A research pipeline', el('span', {}, 'Not a clinical test: anything to act on is confirmed by a certified laboratory first.'))); add(row, main, side); add(slot, row); - // Figure 1: where the findings sit. - var marks = []; - fs.forEach(function (f) { - var l = U.locus(f.viewer); - if (l && !veiled(f)) { marks.push({ chr: l.chr, mb: l.mb, lv: f.level, label: f.gene || '', tip: f.title, href: findingHref(f) }); } - }); - var fig = el('figure', { 'class': 'row fig1' }); - var inner = el('div', {}); - add(inner, el('div', { 'class': 'fig-label' }, 'Figure 1 · Where the findings sit')); - var k = el('div', { 'class': 'kary', id: 'kary' }); - add(k, U.karyogram(marks)); - add(inner, k); - add(inner, el('figcaption', {}, marks.length ? 'Each mark is one finding with a position, on its chromosome. Shape and colour give the level, as in the row above; every finding is also listed in its section.' - : 'No finding with a position yet. The findings without one (medicine genes, repeat counts, copy numbers) are listed in their sections.')); - add(fig, inner, el('aside', {})); - add(slot, fig); + // Figure 1: the genome map (map.js), one figure kept across redraws so its camera stays where the + // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. + if (window.PGPMap) { + if (!genomeMap) { + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, open: function (id) { setShown(id, true); }, + close: function (id) { setShown(id, false); }, fallback: mapFallback }); + } + add(slot, genomeMap.node); + genomeMap.refresh(resultsKey); + } } // --- the table of contents, the phone tab bar, the cover ------------------------------------------ @@ -1106,6 +1134,7 @@ if (key !== fetchedFor) { return; } if (!(r && (r.chapters || r.note))) { fetchedFor = null; return; } results = r; + resultsKey = key; redraw(true); route(); }).catch(function () { if (key === fetchedFor) { fetchedFor = null; } }); @@ -1145,6 +1174,8 @@ drawLead(); drawSections(); restoreFolds(); + // A gated result's fold follows its switch, whatever fold state the reader left (setShown). + Array.prototype.forEach.call(panel.querySelectorAll('details[data-gate]'), function (d) { d.open = !!shown[d.getAttribute('data-gate')]; }); window.scrollTo(0, y); document.getElementById('progress-q').textContent = running() ? 'Is it moving, when will it end, is anything wrong? One line per step.' : 'One line per step: when it ran, and what it found.'; From 39749a0b3072354407db70bb498836911785f970 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 13/53] ci(docs): build og so the demo site carries the map documents The demo build needs og to write map.json for each run. Docs now builds og, checks that each demo run has its map, and uploads screenshots of the map as the map-screens artifact. Bead: pgp-iqon.8 --- .github/workflows/docs.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fe99ccddf..87f0a1229 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,6 +31,9 @@ on: - 'bin/**' - 'tests/demo/**' - 'tests/fixtures/live/**' + # The demo's Figure 1, the genome map, is built by og (og map build) from these. + - 'rust/**' + - 'tests/fixtures/map/**' # No paths filter on pull requests, on purpose: a check skipped by a path # filter stays Pending and blocks the merge once it is a required check # (repos.md step 5 makes green checks required). The build takes seconds. @@ -92,10 +95,41 @@ jobs: # (tests/demo/build_live_demo.py; standard library only). It is a static # folder beside the pages, not a page of the nav, so it goes in after the # site check. + # The demo's Figure 1 is the genome map, whose documents og writes (og map build). The map reads no + # BAM, so a native build without the bam feature is all it needs: no htslib, no container, and the + # cache keeps it to seconds. + - name: Install the toolchain rust/rust-toolchain.toml pins + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust/rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + + - name: Build og for the demo's map + working-directory: rust + run: cargo build --release --locked -p og --no-default-features + - name: Add the live-site demo run: | - python3 tests/demo/build_live_demo.py --out site/demo + python3 tests/demo/build_live_demo.py --out site/demo --og rust/target/release/og test -s site/demo/runs/DEMO-001/frames.json + for r in DEMO-001 DEMO-002 DEMO-003; do test -s "site/demo/runs/${r}/map.json"; done + + # The map at 1440 x 900 and 390 x 844, dark: genome, chromosome and a closed page, kept as the + # artifact map-screens (tests/test_live_demo.py --map-screens; Chrome is on the runner). + - name: Screenshots of the genome map + run: python3 tests/test_live_demo.py --map-screens "$RUNNER_TEMP/map-screens" site/demo + + - name: Upload the map screenshots + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: map-screens + path: ${{ runner.temp }}/map-screens/*.png + if-no-files-found: error + retention-days: 14 # Push to main only: workflow_dispatch can be run on any branch, and the # push filter above does not apply to it. Same line as docs-pages-helm.yml. From 77c4d0c0a11eafb43133e1ddddc0155314102e22 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 14/53] docs(live): describe Figure 1, the genome map Bead: pgp-iqon.8 --- docs/live-site.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/live-site.md b/docs/live-site.md index 11b784bc2..025b3f34c 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -10,7 +10,7 @@ https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the inve ![The person page of DEMO-001 at the end of the demo run: the name with its family and run state, the contents of the ten sections with a count per level, the lead paragraph (no finding for a doctor, two medicine results, six worth a look), the five-level ledger, the data-quality line and the start of the chromosome map of the findings](images/demo-live-site.png){ loading=lazy } -The pages and their plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The cards come from the `results.json` the server would give at the end of the run; each shows once the replay reaches the end of its step. The home page, the charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome viewer does not: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo. +The pages and their plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The cards come from the `results.json` the server would give at the end of the run; each shows once the replay reaches the end of its step. The home page, the charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome map of each person page is the document `og map build` writes for that run, built once when the demo is. The genome viewer does not work: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo. What differs from your own site: @@ -48,7 +48,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th **Home** (`/`). A date line, then one sentence on where everyone stands (how many reports are finished or being written, and whether any finding needs a doctor), and beside it the findings that need attention across everyone, one line per person and level, each opening its finding page. A run still going comes first, as "Being written", with its steps done and the sections already written. A run with no end that has written nothing for a long time reads "Stopped" instead, with the day it was last heard from, and comes after the runs still going: twice the longest step's usual hours (27 h, as DeepVariant's 13.5 h can pass with no message) and never under 6 h. It reads "Being written" again as soon as something new arrives (a message, a trace row, a status row). Then each family: a small pedigree in the members' colours and one tile per person, with the first lines of their report's lead and their counts by level. A family member with no run on this server is listed, dashed. Last, every run as a log: state, steps, hours, when it ended. Without scripts the page is a plain list of runs. -**A person** (`/runs//`). The name, the sex the reads show against the declared one, the person's place in the family and the run state. Down the left, the contents: the ten sections with how many findings each level has, then the step timeline and the family. The page opens on the lead: a paragraph built from the findings (data-quality problems first, then the findings for a doctor named one by one, the medicine genes, the carrier genes, what is worth a look, and the mean depth; with no finding for a doctor it says so only when a clinical screen ran, and otherwise that the clinical screens have no result yet), the ledger of the five levels with a count each, the data-quality line, and Figure 1, the chromosomes with each finding at its place. Then the sections: +**A person** (`/runs//`). The name, the sex the reads show against the declared one, the person's place in the family and the run state. Down the left, the contents: the ten sections with how many findings each level has, then the step timeline and the family. The page opens on the lead: a paragraph built from the findings (data-quality problems first, then the findings for a doctor named one by one, the medicine genes, the carrier genes, what is worth a look, and the mean depth; with no finding for a doctor it says so only when a clinical screen ran, and otherwise that the clinical screens have no result yet), the ledger of the five levels with a count each, the data-quality line, and Figure 1, the genome map. Then the sections: 1. **To act on**, 2. **Medicines**, 3. **For your family**, 4. **Worth a look**, 5. **Can I trust this?**: each finding of the level, with what it is, what to do, and in the margin its evidence, its caveat and its link to the report; then the cards of the report's sections that belong there. 6. **Origins** and 7. **Traits and scores**: the ancestry and trait cards. @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -70,6 +70,8 @@ While the run goes, the page still opens on the report being written. A short ba The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. +**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. Every sentence in the margin is one the map document carries; the page only fills in the counts. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. + **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. **A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. From f1449f78e36161bce1e7b1e7d168220a1ac78907 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:30:09 +0100 Subject: [PATCH 15/53] test(e2e): run og cn where the cases called the Python engine The e2e cases that printed regions or wrote tables with copy_number.py, hba_copy_number.py or traits_report.py --marker now call og cn, and the LPA case cuts the module's combined count-and-marker call. The case paths list bin/og for every case that runs the copy-number steps, and the deleted scripts leave the table. couple_report's two table messages and the comments name og cn. Bead: pgp-bs9u.24 --- bin/bloodtype_report.py | 4 +- bin/collect_summary.py | 10 +-- bin/couple_report.py | 4 +- scripts/ci/build-cn-intervals.py | 2 +- scripts/ci/e2e-case-paths.tsv | 82 +++++++++---------- tests/e2e/bloodtype-1.sh | 2 +- tests/e2e/cn-guard-1-mapq0.sh | 2 +- tests/e2e/cn-intervals-2-hg002.sh | 2 +- tests/e2e/cyp21a2-short-reads-1-hg002.sh | 2 +- tests/e2e/genotype-rules-2-lpa-null.sh | 55 ++++++++----- tests/e2e/genotype-rules-2-smn1-marker.sh | 4 +- .../hba-short-read-accuracy-1-nf-report.sh | 12 +-- tests/e2e/lpa-1.sh | 2 +- tests/e2e/paralog-catalog-1-gba.sh | 4 +- .../e2e/paralog-checks-and-doc-fixes-1-hba.sh | 4 +- 15 files changed, 103 insertions(+), 88 deletions(-) diff --git a/bin/bloodtype_report.py b/bin/bloodtype_report.py index a16874c84..73eaf4236 100755 --- a/bin/bloodtype_report.py +++ b/bin/bloodtype_report.py @@ -26,8 +26,8 @@ --depth is samtools bedcov -Q 20 -j over the table's sites and the RHD row of bin/cn_loci.tsv (scripts/41-bloodtype.sh and the BLOODTYPE_DEPTH process make -it from the BAM; `copy_number.py bed --locus RHD` prints that row's regions, -and bin/copy_number.py turns their depth into the copy number). It gives: +it from the BAM; `og cn bed --locus RHD` prints that row's regions, and the +catalog code of bin/copy_number.py turns their depth into the copy number). It gives: - RhD. The common RhD-negative allele (RHD*01N.01) deletes the whole RHD gene, which a VCF does not show. RHD depth over the depth of its two flanks gives the copy number (0, 1, 2); exons 3 to 7 without RHD reads in a gene that is diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 4f59a0f80..9b528b73f 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -132,7 +132,7 @@ "Variants of uncertain significance (not interpreted)", ] -# Added to NOT_ASSESSED unless step 35's depth count (bin/hba_copy_number.py, +# Added to NOT_ASSESSED unless step 35's depth count (og cn call --locus HBA, # short reads) or step 35b (Paraphase, PacBio HiFi, both genes) made a call. HBA_NOT_ASSESSED = ("HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count " "(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call") @@ -914,13 +914,13 @@ def sec_parascopy(d, s): "smn1_marker": read_marker_table(os.path.join(d, "paralogs", f"{s}_smn1_marker.tsv"), "35", "SMN1_27134"), "rows": len(rows), - # HBA1/HBA2 from depth (bin/hba_copy_number.py); None when + # HBA1/HBA2 from depth (og cn call --locus HBA); None when # step 35 wrote no HBA table (a run from before the check) "hba": read_hba(hp) if os.path.isfile(hp) else None, # the interval rows of bin/cn_loci.tsv (recurrent CNV regions, # carrier deletions, AZF); None without step 35's table "cn_intervals": read_cn_intervals(ip) if os.path.isfile(ip) else None, - # CYP21A2 from depth and PSV sites (bin/copy_number.py, the + # CYP21A2 from depth and PSV sites (og cn call, the # CYP21A2 row): copies, module total, finding; None without it "cyp21a2": read_hba(cp) if os.path.isfile(cp) else None, # the gene-dosage rows (HP, LCE3BC, CCL3L1): total copies from @@ -935,7 +935,7 @@ def sec_parascopy(d, s): def read_cn_intervals(path): - """Step 35's interval table (bin/copy_number.py intervals): the rows with + """Step 35's interval table (og cn intervals): the rows with a deletion or a duplication, and a count of the rows by status.""" rows = read_tsv(path) cols = ("locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change") @@ -2807,7 +2807,7 @@ def sec_archaic(d, s): def sec_lpa(d, s): - """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, bin/lpa_kiv2.py), with + """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, og cn call --locus LPA_KIV2), with step 42's two LPA risk variants when the traits table is there, and whether either points to a genetically high Lp(a).""" p = first_existing(d, [f"lpa/{s}_lpa_kiv2.tsv"]) diff --git a/bin/couple_report.py b/bin/couple_report.py index 9cd5d3c45..b41a24d44 100755 --- a/bin/couple_report.py +++ b/bin/couple_report.py @@ -442,7 +442,7 @@ def cyp21a2_finding(path, genes): copy left.""" t = cyp21a2_table(path) if t.get("status") not in ("called", "not assessed"): - die(["%s: not step 35's CYP21A2 table (bin/copy_number.py call --locus CYP21A2)" % path]) + die(["%s: not step 35's CYP21A2 table (og cn call --locus CYP21A2)" % path]) if t["status"] != "called": return [], "CYP21A2 not assessed (%s)" % t.get("reason", ".") what = "CYP21A2 %s copies, RCCX module total %s of %s" % (t.get("cyp21a2_copies", "."), t.get("module_total", "."), @@ -489,7 +489,7 @@ def read_cn_intervals(path): lines = [ln.rstrip("\n").split("\t") for ln in f if ln.strip()] if (not lines or lines[0][:6] != ["locus", "name", "chrom", "start", "end", "status"] or not {"copies", "expected", "change", "reason"} <= set(lines[0])): - die(["%s: not step 35's interval table (bin/copy_number.py intervals)" % path]) + die(["%s: not step 35's interval table (og cn intervals)" % path]) head = lines[0] return {r[0]: dict(zip(head, r)) for r in lines[1:] if len(r) == len(head)} diff --git a/scripts/ci/build-cn-intervals.py b/scripts/ci/build-cn-intervals.py index 6c186bf7d..5982debdf 100755 --- a/scripts/ci/build-cn-intervals.py +++ b/scripts/ci/build-cn-intervals.py @@ -2,7 +2,7 @@ """build-cn-intervals.py: write the interval rows of bin/cn_loci.tsv. The interval rows are the copy-number checks step 35 runs with one -`samtools bedcov` pass (bin/copy_number.py intervals): +`samtools bedcov` pass (og cn intervals): - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CLINGEN_RECURRENT_CNV_URL and _SHA256 in diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 2b3f5054e..4a8353b84 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -60,9 +60,9 @@ bash-step-14-imputation scripts/14-imputation-prep.sh scripts/ci/fake-docker/lib bash-step-19-delly scripts/19-delly.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -bloodtype-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +bloodtype-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh -callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh +callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/export_consumer.py scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz @@ -73,90 +73,90 @@ clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/ clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -cn-engine-1-parity assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -cn-intervals-1-catalog bin/cn_loci.tsv scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh -cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-engine-1-parity assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh +cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/comphet_phase.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/export_consumer.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/exomiser_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/exomiser_report.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/rfc1_motif.py bin/strchive_convert.py scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh -followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz +followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv bin/traits_report.py modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/genotype_rules_indels.py bin/traits_report.py -genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh -genotype-rules-2-smn1-marker bin/collect_summary.py bin/copy_number.py bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh +genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh +genotype-rules-2-smn1-marker bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/export_consumer.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/export_consumer.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/family_segments.py bin/ibd_report.py scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/mei_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/mocha_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/mosaic_sex_loss.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh -nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/traits_report.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/export_consumer.py bin/family_report.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/y_tree_place.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/export_consumer.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/clip_check.py bin/collect_summary.py bin/cpsr_sample_id bin/render_report.py bin/results_index.py bin/traits_report.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og bin/strchive_convert.py modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/export_consumer.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -pgx-1-depth-check bin/cyp2d6_depth_check.py scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh -pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/comphet_phase.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/export_consumer.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +pgx-1-depth-check bin/cyp2d6_depth_check.py bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh +pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -platelet-antigens-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +platelet-antigens-1 assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_loci.tsv bin/copy_number.py bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py bin/traits_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-1-sample-qc bin/collect_summary.py bin/mosaic_sex_loss.py bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/export_consumer.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/clip_check.py bin/comphet_phase.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh -reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/traits_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh reports-and-pgx-correctness-revel scripts/30-vcfanno.sh scripts/ci/fake-docker/lib.sh -run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cpsr_sample_id bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cpsr_sample_id bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/rfc1_motif.py bin/strchive_convert.py modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_delta.py bin/strchive_convert.py bin/traits_report.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -169,9 +169,9 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/viral_report.py modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/vntyper_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index 3d90a68c4..984840f2f 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -112,7 +112,7 @@ check_eq "Kidd: GIAB's c.838G>A 1/1" "$(cell "$H" Kidd phenotype)" "Jk(a-b+)" check_eq "ABO confidence (every site has reads)" "$(cell "$H" ABO confidence)" "high" check_eq "the evidence file has a row per site and per RhD region (the RHD row of bin/cn_loci.tsv)" \ "$(awk 'END {print NR - 1}' "${G}/${H}/bloodtype/${H}_bloodtype_evidence.tsv" 2>/dev/null)" \ - "$(( $(grep -v '^#' "$MARKERS" | awk 'END {print NR - 1}') + $(python3 "${REPO}/bin/copy_number.py" bed --locus RHD | wc -l) ))" + "$(( $(grep -v '^#' "$MARKERS" | awk 'END {print NR - 1}') + $("${REPO}/bin/og" cn bed --locus RHD | wc -l) ))" # --- planted ------------------------------------------------------------------------- P="${SAMPLE}btp" diff --git a/tests/e2e/cn-guard-1-mapq0.sh b/tests/e2e/cn-guard-1-mapq0.sh index e1c5ca2a4..422773fdf 100755 --- a/tests/e2e/cn-guard-1-mapq0.sh +++ b/tests/e2e/cn-guard-1-mapq0.sh @@ -20,7 +20,7 @@ SRC="https://storage.googleapis.com/deepvariant/case-study-testdata/HG002.novase P=HG002cnguard W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" -python3 "${REPO}/bin/copy_number.py" bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" check_ge "segments of the CYP21A2 row to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/cn-intervals-2-hg002.sh b/tests/e2e/cn-intervals-2-hg002.sh index 371861a09..bb2cf3bb5 100755 --- a/tests/e2e/cn-intervals-2-hg002.sh +++ b/tests/e2e/cn-intervals-2-hg002.sh @@ -23,7 +23,7 @@ W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" ROWS=(AZFa AZFb AZFc_DAZ 22q11.2_DGS_proximal_A-B) args=(); for r in "${ROWS[@]}"; do args+=(--locus "$r"); done -python3 "${REPO}/bin/copy_number.py" bed "${args[@]}" | sort -k1,1 -k2,2n -u > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed "${args[@]}" | sort -k1,1 -k2,2n -u > "${W}/fetch.bed" check_ge "segments of the four rows to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/cyp21a2-short-reads-1-hg002.sh b/tests/e2e/cyp21a2-short-reads-1-hg002.sh index d7e2cf8b2..13a26e226 100755 --- a/tests/e2e/cyp21a2-short-reads-1-hg002.sh +++ b/tests/e2e/cyp21a2-short-reads-1-hg002.sh @@ -21,7 +21,7 @@ SRC="https://storage.googleapis.com/deepvariant/case-study-testdata/HG002.novase P=HG002cyp W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" -python3 "${REPO}/bin/copy_number.py" bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" +"${REPO}/bin/og" cn bed --locus CYP21A2 | sort -k1,1 -k2,2n > "${W}/fetch.bed" check_ge "segments of the CYP21A2 row to fetch" "$(wc -l < "${W}/fetch.bed" | tr -d ' ')" 10 retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } diff --git a/tests/e2e/genotype-rules-2-lpa-null.sh b/tests/e2e/genotype-rules-2-lpa-null.sh index c287305ce..f92c2bc11 100755 --- a/tests/e2e/genotype-rules-2-lpa-null.sh +++ b/tests/e2e/genotype-rules-2-lpa-null.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Step 46's KIV-2 4925G>A reading (bin/lpa_kiv2.py null-allele, MARKERS in -# bin/traits_report.py) on real reads: +# Step 46's KIV-2 4925G>A reading (og cn call --marker LPA_KIV2_4925, the +# marker of MARKERS in bin/traits_report.py) on real reads: # - dbSNP orientation, online: the record pinned for the marker is what # dbSNP's GRCh38 VCF of the build versions.env pins holds for rs1484919544 # (same position and REF, the pinned ALT among dbSNP's); control: the same @@ -19,10 +19,10 @@ # dropped: 'not seen' # The reading goes to lpa/_lpa_kiv2_4925.tsv; step 46's own table # keeps its rows. -# - the LPA module's pileup and 4925G>A lines, cut from its main.nf, write the -# same pileup and table in their images, byte for byte, on the planted BAM; -# with a 4925G>A reader that crashes, its block writes a table that says -# not assessed and leaves the count; +# - the LPA module's sites, pileup and count lines, cut from its main.nf, +# write the same pileup and tables in their images, byte for byte, on the +# planted BAM; with a 4925G>A reader that crashes after the count, its +# block writes a table that says not assessed and keeps the count; # - steps 24 and generate-report.sh on the planted and the reference samples: # both reports print the reading under the count with its caveat # ('present' with the reads and copies, 'not seen'). @@ -159,14 +159,15 @@ MOD="${REPO}/modules/local/lpa/main.nf" M="${T}/module" rm -rf "$M"; mkdir -p "$M" unesc() { sed -e 's/\\\\/\\/g' -e 's/\\\$/$/g' -e "s|\${bam}|${G}/${P}/aligned/${P}_sorted.bam|g" -e "s|\${meta.id}|${P}|g" \ - -e "s|\${pileup}|${P}_lpa_kiv2_4925_pileup.tsv|g"; } -grep -E '^ *traits_report\.py --marker-regions LPA_KIV2_4925 > lpa_sites\.txt$' "$MOD" | unesc > "${M}/sites.sh" + -e "s|\${pileup}|${P}_lpa_kiv2_4925_pileup.tsv|g" -e "s|\${depth}|${P}_lpa_kiv2_depth.tsv|g" \ + -e "s|\${depth_q1}|${P}_lpa_kiv2_depth_q1.tsv|g"; } +grep -E '^ *og cn marker-regions LPA_KIV2_4925 > lpa_sites\.txt$' "$MOD" | unesc > "${M}/sites.sh" sed -n '/^ *( while read -r r; do samtools mpileup /,/|| echo /p' "$MOD" | unesc > "${M}/pileup.sh" -sed -n '/^ *if ! traits_report\.py --marker LPA_KIV2_4925 --pileup/,/^ *fi$/p' "$MOD" | unesc > "${M}/null.sh" -check_eq "the module's sites, pileup and 4925G>A lines were cut from main.nf" \ - "$(cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" | wc -l | tr -d ' ')" 9 +sed -n '/^ *rc=0$/,/^ *fi$/p' "$MOD" | unesc > "${M}/null.sh" +check_eq "the module's sites, pileup and count lines were cut from main.nf" \ + "$(cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" | wc -l | tr -d ' ')" 12 cat "${M}/sites.sh" "${M}/pileup.sh" "${M}/null.sh" -cp "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "$M/" +cp "${G}/${P}/lpa/${P}_lpa_kiv2_depth.tsv" "${G}/${P}/lpa/${P}_lpa_kiv2_depth_q1.tsv" "$M/" mod() { docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "${REPO}/bin:/pgp-bin:ro" \ -w "$M" "$1" sh -c 'PATH="/pgp-bin:$PATH"; set -e; . "./$1"' _ "$2"; } mod "$PYTHON_IMAGE" sites.sh && mod "$SAMTOOLS_IMAGE" pileup.sh && mod "$PYTHON_IMAGE" null.sh 2>&1 | tail -n 3 @@ -174,16 +175,30 @@ check "the module writes step 46's pileup, byte for byte" \ cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2_4925_pileup.tsv" "${M}/${P}_lpa_kiv2_4925_pileup.tsv" check "the module writes step 46's 4925G>A table, byte for byte" \ cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2_4925.tsv" "${M}/${P}_lpa_kiv2_4925.tsv" -check "the module leaves step 46's table byte for byte" cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/${P}_lpa_kiv2.tsv" +check "the module writes step 46's count table, byte for byte" cmp -s "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/${P}_lpa_kiv2.tsv" -# The module's fallback, in its image: a 4925G>A reader that crashes leaves the -# count and a table that says not assessed (scripts/46-lpa.sh writes the same -# table; tests/test_lpa_fallback.sh checks that). Control: the planted table -# above was 'present'. +# The module's fallback, in its image: an og whose 4925G>A reader crashes after +# it wrote the count (the planted og runs the real one without --marker, then +# exits 1) keeps the count and a table that says not assessed +# (scripts/46-lpa.sh writes the same table; tests/test_lpa_fallback.sh checks +# that). Control: the planted table above was 'present'. mkdir -p "${M}/broken" -printf '#!/bin/sh\necho "planted crash" >&2\nexit 1\n' > "${M}/broken/traits_report.py" -chmod +x "${M}/broken/traits_report.py" -cp "${G}/${P}/lpa/${P}_lpa_kiv2.tsv" "${M}/count_before.tsv" +cat > "${M}/broken/og" <<'OG' +#!/bin/sh +n=$# +while [ "$n" -gt 0 ]; do + case "$1" in + --marker|--marker-pileup|--marker-out) shift 2; n=$((n - 2)); continue ;; + esac + set -- "$@" "$1"; shift; n=$((n - 1)) +done +/pgp-bin/og "$@" || exit +echo "planted crash" >&2 +exit 1 +OG +chmod +x "${M}/broken/og" +mv "${M}/${P}_lpa_kiv2.tsv" "${M}/count_before.tsv" +rm -f "${M}/${P}_lpa_kiv2_4925.tsv" docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -v "${REPO}/bin:/pgp-bin:ro" \ -w "$M" "$PYTHON_IMAGE" sh -c "PATH=\"${M}/broken:/pgp-bin:\$PATH\"; set -e; . ./null.sh" 2>&1 | tail -n 2 check_eq "the module with a crashing 4925G>A reader: not assessed" \ diff --git a/tests/e2e/genotype-rules-2-smn1-marker.sh b/tests/e2e/genotype-rules-2-smn1-marker.sh index a5804dbea..38cc4b746 100755 --- a/tests/e2e/genotype-rules-2-smn1-marker.sh +++ b/tests/e2e/genotype-rules-2-smn1-marker.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Step 35's SMN1 g.27134T>G reading (bin/traits_report.py --marker -# SMN1_27134) on real reads: +# Step 35's SMN1 g.27134T>G reading (og cn marker-read SMN1_27134) on +# real reads: # - dbSNP orientation, online: the record pinned for the marker is what # dbSNP's GRCh38 VCF of the build versions.env pins holds for rs143838139; # control: the same compare with REF and ALT swapped fails; diff --git a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh index f9eee7a47..ef465eec9 100755 --- a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh +++ b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # The Nextflow HTML report and step 35's HBA1/HBA2 table -# (_hba_copy_number.tsv, bin/hba_copy_number.py), which the bash report +# (_hba_copy_number.tsv, og cn call --locus HBA), which the bash report # already reads beside the SMN table. # 1. HTML_REPORT, run for real in its image through a two-line pipeline, on # HG002's VCF with an SMN table and an HBA table that -# bin/hba_copy_number.py writes from made-up depths (four genes). It +# og cn writes from made-up depths (four genes). It # must place the HBA table in paralogs/ (a name it has no place for # stops the task), the report must carry the HBA line with the count, # and the not-assessed list must leave HBA1/HBA2 out. Negative control: @@ -28,12 +28,12 @@ mkdir -p "$T" printf 'chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n' \ > "${T}/${SAMPLE}_smn_copy_number.tsv" # Made-up depths: 30x everywhere, a quarter of the gene depth at MAPQ 0 -python3 "${REPO}/bin/hba_copy_number.py" bed \ +"${REPO}/bin/og" cn bed --locus HBA \ | awk -F'\t' -v OFS='\t' '{print $0, ($3 - $2) * 30}' > "${T}/bedcov.tsv" -python3 "${REPO}/bin/hba_copy_number.py" bed \ +"${REPO}/bin/og" cn bed --locus HBA \ | awk -F'\t' -v OFS='\t' '{print $0, ($3 - $2) * ($4 ~ /^HBA/ ? 22.5 : 30)}' > "${T}/bedcov_q1.tsv" -python3 "${REPO}/bin/hba_copy_number.py" check --bedcov "${T}/bedcov.tsv" --bedcov-mapq1 "${T}/bedcov_q1.tsv" \ - --out "${T}/${SAMPLE}_hba_copy_number.tsv" +"${REPO}/bin/og" cn call --locus HBA --bedcov "${T}/bedcov.tsv" --bedcov-mapq1 "${T}/bedcov_q1.tsv" \ + --table "${T}/${SAMPLE}_hba_copy_number.tsv" cat "${T}/${SAMPLE}_hba_copy_number.tsv" check_eq "the made-up HBA table counts four" \ "$(awk -F'\t' '$1 == "alpha_globin_genes" {print $2}' "${T}/${SAMPLE}_hba_copy_number.tsv")" 4 diff --git a/tests/e2e/lpa-1.sh b/tests/e2e/lpa-1.sh index 2a01370da..e7dead184 100755 --- a/tests/e2e/lpa-1.sh +++ b/tests/e2e/lpa-1.sh @@ -11,7 +11,7 @@ # HG002 is published that this case could compare with, so the value goes # to the notes and the checks are on its arithmetic: # - the same reads with half of those over the KIV-2 block: N + 2 halves -# (bin/lpa_kiv2.py: N = 14 x M / D - 2); +# (og cn call --locus LPA_KIV2: N = 14 x M / D - 2); # - the same reads at an eighth of the depth: "too little depth"; # - nextflow run main.nf --tools lpa,html_report on the real reads publishes # the same three files as the bash step, byte for byte, and the HTML diff --git a/tests/e2e/paralog-catalog-1-gba.sh b/tests/e2e/paralog-catalog-1-gba.sh index 25f112717..cd548aef5 100755 --- a/tests/e2e/paralog-catalog-1-gba.sh +++ b/tests/e2e/paralog-catalog-1-gba.sh @@ -127,7 +127,7 @@ M="${GENOME_DIR}/${P}/module" rm -rf "$M"; mkdir -p "$M" SPECS=$(awk -F'\t' '/^#/ || !NF {next} !h {for (i = 1; i <= NF; i++) c[$i] = i; h = 1; next} {printf "%s%s|%s|%s", sep, $c["model"], $c["region"], $c["table"]; sep = " "}' "$LOCI") -# bin/ is on PATH in Nextflow; here copy_number.py runs from it mounted, as step 35 runs it. +# bin/ is on PATH in Nextflow; here og runs from it mounted, as step 35 runs it. sed -n '/# Paralog loci (as scripts\/35-paralogs.sh)/,/# end of the paralog loci/p' "$MOD" \ | sed -e 's/\\\\/\\/g' -e 's/\\\$/$/g' \ -e "s|\${parascopy_data}|/genome/reference/parascopy-${PARASCOPY_DATA_VERSION}|g" \ @@ -138,7 +138,7 @@ sed -n '/# Paralog loci (as scripts\/35-paralogs.sh)/,/# end of the paralog loci -e "s|\${background}|-b /genome/${P}/parascopy_windows.bed --no-gc|g" \ -e "s|\${task.cpus}|2|g" \ -e "s/\${loci}/${SPECS}/g" \ - -e "s|^\( *\)copy_number.py |\1python3 /pgp-bin/copy_number.py |" > "${M}/block.sh" + -e "s|^\( *\)og cn |\1/pgp-bin/og cn |" > "${M}/block.sh" check_ge "the module's loop was cut from main.nf" "$(wc -l < "${M}/block.sh" | tr -d ' ')" 25 check "the cut loop has no Nextflow variable left" \ sh -c "! grep -E '[$][{](parascopy_data|params[.]|bam|meta[.]|reference|background|task[.]|loci)' '${M}/block.sh'" diff --git a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh index e6659f6c6..7e203c434 100755 --- a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh +++ b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Step 35's HBA1/HBA2 count from read depth (bin/hba_copy_number.py) on +# Step 35's HBA1/HBA2 count from read depth (og cn call --locus HBA) on # synthetic reads, with a planted one-gene deletion. The fixture has no reads # at the alpha-globin cluster, so the case writes its own: 150 bp reads cut # from the fixture reference every 5 bp over chr16:95,000-235,000 (30x), @@ -64,7 +64,7 @@ fa, header, sample, out = sys.argv[1:] seq = "".join(l.strip() for l in open(fa) if not l.startswith(">")).upper() start0 = 95000 # 0-based start of chr16:95001-235000 rg = next((l.split("\t")[1][3:] for l in open(header) if l.startswith("@RG")), None) -hba2 = (172875, 173710) # bin/hba_copy_number.py's HBA2, 0-based +hba2 = (172875, 173710) # the HBA row's HBA2 (og cn bed --locus HBA), 0-based L, step, n = 150, 5, 0 comp = str.maketrans("ACGTN", "TGCAN") seen = {} From 180b21697eace0257605ea941507e72a6ee41896 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:32:13 +0100 Subject: [PATCH 16/53] docs(cn): og cn is the copy-number engine; name its deliberate fixes Bead: pgp-bs9u.24 --- CHANGELOG.md | 10 ++++++++++ docs/35-paralogs.md | 10 +++++----- docs/46-lpa.md | 8 ++++---- docs/cn-loci.md | 15 ++++++++------- docs/lessons-learned.md | 9 +++++++++ docs/rust.md | 3 ++- rust/README.md | 1 + 7 files changed, 39 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a197c3414..151765145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Copy number from depth (steps 32, 35, 41 and 46) now runs in the Rust `og` binary: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read`. They replace `bin/hba_copy_number.py` and `bin/lpa_kiv2.py`, which are gone, the commands of `bin/copy_number.py` (it keeps the catalog reader that step 41's report imports) and the marker reader of `bin/traits_report.py` (it keeps `MARKERS` for the summary). Every table, file name and option a step uses stays the same; the parity cases in `tests/parity` hold every output that changed on purpose: + - Step 46: the KIV-2 4925G>A reading is made in the same call as the count and judges the aligner on the count's own MAPQ 0 share. Before, it read the share the count table prints, rounded to three places, so at a share just under 0.20 the count said not assessed while the marker still read `present`. Both now say not assessed. + - Step 35: the SMN1 marker message takes the SMN1 psCN from the first row of the SMN table that covers exon 7, the rule of the Paraphase comparison. It took the last such row, so a table with two rows over exon 7 made the two read different counts. + - Steps 35 and 46: a status the HBA or LPA table layout did not name (a catalog edit could give one) now writes `not assessed: `. The HBA table wrote `called` with no count, and the LPA table stopped with an error. + - A catalog row with a `more=` reading and no numbered reading is refused when the catalog loads, naming the row. Before, a call that reached it failed without naming the row. + - `og cn call --sex` reads a gene row on chrX or chrY against the sample's count there. The Python call had no sex, so such a row could only read `sex_unknown`. No row of the shipped catalog is a gene row on chrX or chrY. + - The tables are written to a temporary file and renamed, so a failed run leaves no half-written table. + - `copy_number.py call --json` is gone with no replacement: no step passed it. + - Error messages about a wrong regions table say to make it with `og cn bed`. + - Step 49 (clonal haematopoiesis) now prepares the CHIP gene models and writes its two tables with the Rust `og` binary: `og chip prepare`, `og chip report` and `og chip check-table` replace the commands of `bin/chip_report.py`, which keeps only the detection limit and the column names the reports import. On Ensembl 116's full GFF3, prepare takes 1.7 s and 6 MB instead of 15 s and 20 MB. The two tables, `chip.gff3` and `chip.bed` are the same, byte for byte. What changes: a missing input file or a position that is not a whole number stops with an `ERROR:` line and exit 1, not a Python traceback, and each output is written whole or not at all. Step 49 now needs `og` (`scripts/setup.sh --og`). - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. diff --git a/docs/35-paralogs.md b/docs/35-paralogs.md index 295da9f1d..84f228e0d 100644 --- a/docs/35-paralogs.md +++ b/docs/35-paralogs.md @@ -62,11 +62,11 @@ With Nextflow: `--tools ...,parascopy --parascopy_data ${GENOME_DIR}/reference/p 1. `parascopy depth`: background read depth of the sample, over Parascopy's GRCh38 windows (or `PARASCOPY_DEPTH_BED`). 2. For each row of the catalog: `samtools view -c` counts the reads over the row's region, then `parascopy cn-using models_GRCh38_1KGP//` gives the copy number of the locus with those model parameters. A row whose model is not installed, whose region has no read (or whose reads `samtools` could not count), or on which Parascopy fails or writes no result file is not assessed: its table holds one row that says so, with the reason, and the other rows still run. Before any of this the step stops on a catalog row it cannot use: a malformed model, region or table name, a table name used twice, or the table names `hba` and `paraphase`, which other tables use. 3. Writes one row per region of each locus profile to `${SAMPLE}__copy_number.tsv` (`${SAMPLE}_smn_copy_number.tsv` for the SMN1 row). -4. `samtools bedcov` (in the Parascopy image) sums the depth over HBA2, HBA1 and two flanks, once over every read and once over reads of mapping quality 1 or more (`-Q 1`), and `bin/copy_number.py call --locus HBA` (the copy-number engine, on the HBA row of `bin/cn_loci.tsv`; see [Copy number from depth](cn-loci.md)) writes `${SAMPLE}_hba_copy_number.tsv`, laid out by `bin/hba_copy_number.py`. A failure here prints a warning and writes a table that says not assessed; the SMN result stands. +4. `samtools bedcov` (in the Parascopy image) sums the depth over HBA2, HBA1 and two flanks, once over every read and once over reads of mapping quality 1 or more (`-Q 1`), and `og cn call --locus HBA` (the copy-number engine, on the HBA row of `bin/cn_loci.tsv`; see [Copy number from depth](cn-loci.md)) writes `${SAMPLE}_hba_copy_number.tsv`. A failure here prints a warning and writes a table that says not assessed; the SMN result stands. 5. The gene-dosage rows (HP, LCE3BC, CCL3L1) the same way, into `${SAMPLE}_cn_dosage.tsv` ([the gene-dosage rows](cn-loci.md#the-gene-dosage-rows)). -6. `samtools bedcov` over the CYP21A2 row's segments (two passes, all reads and mapping quality 1 or more, as HBA), `samtools mpileup` without a reference at its PSV sites, then `bin/copy_number.py call --locus CYP21A2 --bedcov-mapq1 ... --pileup ...` writes `${SAMPLE}_cyp21a2_psv.tsv`. A failure prints a warning and writes a table that says not assessed (`check_failed`). +6. `samtools bedcov` over the CYP21A2 row's segments (two passes, all reads and mapping quality 1 or more, as HBA), `samtools mpileup` without a reference at its PSV sites, then `og cn call --locus CYP21A2 --bedcov-mapq1 ... --pileup ...` writes `${SAMPLE}_cyp21a2_psv.tsv`. A failure prints a warning and writes a table that says not assessed (`check_failed`). 7. The interval rows into `${SAMPLE}_cn_intervals.tsv` ([the interval rows](cn-loci.md#the-interval-rows)). -8. SMN1 g.27134T>G beside the SMN1 psCN (before the HBA count): `samtools mpileup` without a reference at its SMN1 and SMN2 positions, then `bin/traits_report.py --marker SMN1_27134` with the SMN table writes `${SAMPLE}_smn1_marker.tsv` ([the SMN1 silent-carrier marker](#the-smn1-silent-carrier-marker)). A failure prints a warning and writes a table that says not assessed. +8. SMN1 g.27134T>G beside the SMN1 psCN (before the HBA count): `samtools mpileup` without a reference at its SMN1 and SMN2 positions, then `og cn marker-read SMN1_27134` with the SMN table writes `${SAMPLE}_smn1_marker.tsv` ([the SMN1 silent-carrier marker](#the-smn1-silent-carrier-marker)). A failure prints a warning and writes a table that says not assessed. ## Output @@ -96,7 +96,7 @@ SMN2 holds the same 21 bases 875,420 bases upstream, so the reads cannot tell an | `too little depth` | under 10 reads over the two positions | | `not assessed` | the pileup could not be read, or the check failed | -The message reads the marker beside the SMN1 psCN at exon 7 (filter `PASS` and quality 20 or more, the rule of the report's SMN1 line): with two SMN1 copies it says whether the marker raises or lowers the chance of 2+0; with another count, or a psCN that is not resolved, it says the marker refines a two-copy result only. Every message ends with "This marker does not prove 2+0, and its absence does not rule 2+0 out." Both reports and the live person page print the marker right under the SMN1 copies: its state (`present` or `uncertain` with the reads that carry G and the copies, `not seen` with the reads, `too little depth`, or `not assessed`), the reader's message, and the caveat that the marker raises the chance that two SMN1 copies sit on one chromosome, does not prove 2+0, and that a carrier test in a clinical laboratory confirms it. Without the table (a run from before this reader) or with a table that has no known status, the line says `not assessed` with the reason. The live card also lists the SMN1 copies and the marker state as facts. The Nextflow `HTML_REPORT` gets the table from the `PARASCOPY` task the same way. +The message reads the marker beside the SMN1 psCN at exon 7 (filter `PASS` and quality 20 or more, from the first row of the SMN table whose region or homologous regions hold exon 7: the rule of the report's SMN1 line and of the Paraphase comparison): with two SMN1 copies it says whether the marker raises or lowers the chance of 2+0; with another count, or a psCN that is not resolved, it says the marker refines a two-copy result only. Every message ends with "This marker does not prove 2+0, and its absence does not rule 2+0 out." Both reports and the live person page print the marker right under the SMN1 copies: its state (`present` or `uncertain` with the reads that carry G and the copies, `not seen` with the reads, `too little depth`, or `not assessed`), the reader's message, and the caveat that the marker raises the chance that two SMN1 copies sit on one chromosome, does not prove 2+0, and that a carrier test in a clinical laboratory confirms it. Without the table (a run from before this reader) or with a table that has no known status, the line says `not assessed` with the reason. The live card also lists the SMN1 copies and the marker state as facts. The Nextflow `HTML_REPORT` gets the table from the `PARASCOPY` task the same way. ## Runtime @@ -237,7 +237,7 @@ The ratio of a site is its estimate over the median of the sites, so one copy co - `tests/test_paralog_loci.py` checks the model list (167 genes in `assets/parascopy_models_v1.7.txt`, each with one main region) and every catalog row (a model on the list, a GRCh38 region inside the model's main region, a known tag and cross-check, a table name used once, the SMN1 row), and that planted faults are refused: a model not on the list (`GBA1.gz`, and `CYP21A2.gz`, which the data has no model for), a bad region, a region outside its model, a repeated table name, a catalog without the SMN1 row. It checks one owner per locus: no catalog model overlaps a depth-engine row of `bin/cn_loci.tsv` but RHCE (without that exemption RHCE is refused, and a planted model over HBA is refused), and every catalog gene `assets/callability_genes.tsv` lists has the owner `parascopy` there (a planted `.` is refused). The catalog sets owners in that file but adds no rows to it: its rows come from GIAB's lists, gene body by gene body, and a model can cover only part of a gene. So 17 catalog genes are not there: seven pseudogenes (CELP, GBAP1, NCF1B, NCF1C, OTOAP1, PMS2CL, STRCP1), which it leaves out as not protein-coding, and ten genes to which none of its reasons applies (ABCC6, CFH, CFHR4, CHRNA7, DEFA3, KRT86, NEB, PIK3CA, PLG, VWF). This section reads them all. The test holds that list and fails when it changes. It also runs the step's own lines that write a not-assessed SMN table and checks that the report, the Paraphase side-by-side and the couple screen each read it as not assessed, with the reason, and that a catalog table without `agCN_filter` or `agCN_qual` reads as unreadable, not as not resolved. - `tests/fake-docker/paralog-catalog-loop.sh` runs the step on a catalog of three rows against a fake Docker: three `parascopy cn-using` calls with the right model files, in order, one table each; another population's files with `PARASCOPY_POPULATION`; no call and a not-assessed table for a region without reads, a model that is not installed and a Parascopy run that fails, while the other rows run; a rerun with fewer rows removes the tables of the dropped rows and keeps the HBA and Paraphase tables; a bad row, or a region off chr1-22, X and Y or with its start past its end, stops the step before any container. - The e2e case `paralog-catalog-1-gba` adds a GBA row (`GBA.gz`) to a copy of the catalog and runs the step on the fixture's slice plus GIAB's HG002 reads over GBA1 and GBAP1, sampled to about 30x: the GBA table must hold an integer agCN with its quality. On the fixture's slice alone, which has no read at GBA1, the same row must be not assessed. The case also checks that the committed models are the names in the installed `list.txt` of each population, that the SMN table is byte for byte what one `parascopy cn-using` of `SMN1.gz` writes, as the step did before the catalog, and that the loop of the Nextflow module, cut from the file, writes the same two tables in the same image. -- `tests/test_copy_number.py` checks the HBA row and that these depth tables give, through the engine, the same tables as the code before it. `tests/test_hba_copy_number.py` gives `bin/hba_copy_number.py` made-up depth tables: four genes at even depth, three with HBA2 at half depth, two, a triplication, an estimate between two whole numbers, low flanks, flanks that disagree, and no read of mapping quality 0 over the genes (the depths of the novoalign BAM, and a clean three), which must be not assessed. +- `tests/test_copy_number.py` checks the HBA row and that these depth tables give, through the engine, the same tables as the code before it. `tests/test_hba_copy_number.py` gives `og cn call --locus HBA` made-up depth tables: four genes at even depth, three with HBA2 at half depth, two, a triplication, an estimate between two whole numbers, low flanks, flanks that disagree, and no read of mapping quality 0 over the genes (the depths of the novoalign BAM, and a clean three), which must be not assessed. - The e2e case `paralog-checks-and-doc-fixes-1-hba` writes synthetic 30x reads over chr16:95,001-235,000 for two samples, one with every second read over HBA2 left out (a planted one-gene loss), runs step 35 on each and checks four and three genes. A synthetic read whose bases occur more than once in that stretch gets mapping quality 0, as an aligner gives it. The HBA block of `modules/local/parascopy/main.nf`, cut from the file, then runs on the same BAMs in the same image and must write the same table. Last, the block runs on HG002's real Illumina reads at the cluster, three ways: GIAB's 60x novoalign BAM sampled to about 30x must be not assessed, naming the missing mapping quality 0 reads; Google's 35x NovaSeq BWA-MEM BAM must count four; and the same reads realigned with `minimap2 -a -x sr` to chr16:1-300,000 must count four. The truth is four, from Paraphase on HG002's HiFi reads. The run's notes record each estimate, the four depths and the share of mapping quality 0. - The e2e case `paralog-rows-1` runs the whole catalog on the fixture's slice plus GIAB's HG002 reads over every catalog locus on the fixture's chromosomes, sampled to about 30x, timed (it fails past 30 minutes): every row's table must hold an integer agCN or say not assessed with the reason, and a header-only table fails that check. It then runs the step with a catalog that has no SMN1 row and checks that both reports still show the HBA count and the other loci. `tests/test_collect_summary.py` holds the same case on made-up tables, with the loci read in the catalog's order. - `tests/test_copy_number.py` section 10 runs the CYP21A2 row on made-up bedcov and mpileup tables (`tests/fixtures/cn/CYP21A2_*`): unmodified, 2 copies and no finding; a planted chimeric deletion, 1 copy with the module total 3; a planted In2G conversion on one copy, a ratio of 0.5 at In2G and a possible conversion there. The same depths with every site even give no finding. The deletion's tables with no MAPQ 0 read (a share under 0.02) read not assessed (`low_mapq0_share`), and with a share of 0.021 or HG002's 0.1 still give the deletion. The CYP21A2 block of step 35, cut out, writes those tables, a not-assessed one when mpileup fails and a `low_mapq0_share` one when its MAPQ 1 pass equals the first, and the couple screen and the report read them. diff --git a/docs/46-lpa.md b/docs/46-lpa.md index b1ead3ceb..ab6bb4010 100644 --- a/docs/46-lpa.md +++ b/docs/46-lpa.md @@ -22,7 +22,7 @@ A VCF cannot show the repeat count or the 4925G>A variant: the six KIV-2 units i ## Tool - samtools `bedcov -j` for the read depth over the exons, `mpileup` for the reads at the 4925G>A sites -- `bin/copy_number.py` (standard library Python), the copy-number engine, on the LPA_KIV2 row of `bin/cn_loci.tsv` for the count, its interval and the reading; `bin/lpa_kiv2.py` lays out the table. See [Copy number from depth](cn-loci.md) +- `og cn` (OwnGenome's Rust binary, [docs/rust.md](rust.md)), the copy-number engine, on the LPA_KIV2 row of `bin/cn_loci.tsv` for the count, its interval, the reading and the table, and in the same call the 4925G>A reading. See [Copy number from depth](cn-loci.md) ## Docker Image @@ -48,7 +48,7 @@ With `run-all.sh`: `TOOLS=...,lpa`. It is opt-in and needs no data, so a run lis The method is the one published short-read estimators use: depth over the KIV-2 repeat against depth over single-copy sequence (Zekavat et al., Nat Commun 2018, PMID 29973585; Behera et al., BMC Med Genomics 2024, PMID 39449055). Behera et al. give the KIV-2 region in GRCh38 as chr6:160,611,568-160,646,868, six units, and its reads reach into the last exon of KIV-1 and the first exon of KIV-3, which resemble KIV-2. -The regions are exons of the MANE Select transcript ENST00000316300.10 (Ensembl, checked 2026-10-25), numbered from its first exon. They are the LPA_KIV2 row of `bin/cn_loci.tsv`, and `bin/copy_number.py bed --locus LPA_KIV2` prints them: +The regions are exons of the MANE Select transcript ENST00000316300.10 (Ensembl, checked 2026-10-25), numbered from its first exon. They are the LPA_KIV2 row of `bin/cn_loci.tsv`, and `og cn bed --locus LPA_KIV2` prints them: | Group | Exons | What they are | |---|---|---| @@ -84,7 +84,7 @@ The count is not made when: ## How the 4925G>A variant is read -The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds that base at the end of each of its six KIV-2 units (exons 5, 7, ..., 15; `NM_005577.4:c.733G>A` at the first, the one dbSNP names as rs1484919544) and at the near-identical last exon of KIV-1 (exon 3). The 21 bases around it are the same at all seven, so a read from any copy can land on any of them. `samtools mpileup` without a reference reads every read at the seven positions, every mapping quality, duplicates left out, bases of quality 13 or more, and `bin/traits_report.py --marker LPA_KIV2_4925` adds them up into its own table (`bin/lpa_kiv2.py` lays it out). The share of reads with T is a fraction of the copies, not a genotype. With a count, the copies that carry it are that fraction times N + 2 (the KIV-2 repeats plus one KIV-1 per chromosome). +The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds that base at the end of each of its six KIV-2 units (exons 5, 7, ..., 15; `NM_005577.4:c.733G>A` at the first, the one dbSNP names as rs1484919544) and at the near-identical last exon of KIV-1 (exon 3). The 21 bases around it are the same at all seven, so a read from any copy can land on any of them. `samtools mpileup` without a reference reads every read at the seven positions, every mapping quality, duplicates left out, bases of quality 13 or more, and `og cn call --locus LPA_KIV2 ... --marker LPA_KIV2_4925` adds them up into its own table, in the same call that makes the count. The share of reads with T is a fraction of the copies, not a genotype. With a count, the copies that carry it are that fraction times N + 2 (the KIV-2 repeats plus one KIV-1 per chromosome). | `status` | When | |---|---| @@ -92,7 +92,7 @@ The variant is the last base of the second KIV-2 exon, G>A on LPA's minus strand | `not seen` | 2 reads with T or fewer (one or two can be sequencing errors) | | `uncertain` | between the two | | `too little depth` | under 100 reads over the seven positions together | -| `not assessed` | the pileup could not be read (also when `samtools mpileup` failed; the count still runs), or the count found that the aligner left out the reads of MAPQ 0, which are the reads a carrying repeat gives | +| `not assessed` | the pileup could not be read (also when `samtools mpileup` failed; the count still runs), or the count found that the aligner left out the reads of MAPQ 0, which are the reads a carrying repeat gives. The marker reads the count's own MAPQ 0 share, not the rounded value in the count table, so the two never disagree at the 20% line | What a short-read fraction cannot say: which chromosome or which repeat carries the variant, so whether it sits on the small or the large isoform. A variant in one repeat among many gives few reads, so `not seen` is not proof of absence at low depth. diff --git a/docs/cn-loci.md b/docs/cn-loci.md index cada3a10b..0c3452763 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -1,6 +1,6 @@ # Copy Number from Depth (the locus catalog) -Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one engine, `bin/copy_number.py`, and one table, `bin/cn_loci.tsv`, with one row per locus. A new locus is a new row, not a new script. +Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)), and one table, `bin/cn_loci.tsv`, with one row per locus. A new locus is a new row, not a new script. The table sits in `bin/` because `bin/` is the one repository folder that every container mounts: the bash steps mount it as `/pgp-bin`, and Nextflow puts it on every task's PATH. @@ -28,7 +28,7 @@ Then the row's `call` column decides: - `psv:D`: the module total of a gene and its pseudogene as `round:D` does, then the reads at the row's PSV sites ([the CYP21A2 row](#the-cyp21a2-row)); - `class:LOW,HIGH`: low, typical or high from the 95% interval (LPA KIV-2, 30 and 45). With `interval` `cv`, that interval comes from the spread of the per-segment depths: relative standard error sqrt(cv_target^2 / n_target + cv_control^2 / n_control), applied on the log scale. It covers sampling noise only. -`copy_number.py call --json` writes the result as one JSON block per locus: status, reason, estimate, copies, interval, class, the depths and the reading. +`og cn call --table OUT` writes the result as the step's table (status, the estimate, the copies or class, the depths and the reading), and prints one summary line. ## The columns @@ -81,7 +81,7 @@ CYP21A2 and its pseudogene CYP21A1P lie in the duplicated RCCX module, about 98% The call has two parts: 1. **The module total**: `scale` 2 times the two targets' depths added up over the flank depth, a whole number within 0.4, else not assessed (`between_counts`). Two modules per chromosome give 4. The tolerance is wider than HBA's 0.3 because the depth here runs high: HG002's 35x reads give 4.22, and a deletion planted in the same reads 3.32. -2. **The PSV sites**: `copy_number.py bed --psv --locus CYP21A2` prints each site on CYP21A2 and on CYP21A1P, and `samtools mpileup -A -B -q 0 -Q 13 -d 100000` without a reference counts the bases there (every read whatever its mapping quality, bases of quality 13 or more; read starts, ends and indels are skipped). A read with CYP21A2's base counts for CYP21A2 at either position, since a converted copy's reads may land on the pseudogene. A site with 30 such reads or more gets `estimate = module total x CYP21A2 reads / (CYP21A2 + CYP21A1P reads)` and its ratio to the median of the sites. +2. **The PSV sites**: `og cn bed --psv --locus CYP21A2` prints each site on CYP21A2 and on CYP21A1P, and `samtools mpileup -A -B -q 0 -Q 13 -d 100000` without a reference counts the bases there (every read whatever its mapping quality, bases of quality 13 or more; read starts, ends and indels are skipped). A read with CYP21A2's base counts for CYP21A2 at either position, since a converted copy's reads may land on the pseudogene. A site with 30 such reads or more gets `estimate = module total x CYP21A2 reads / (CYP21A2 + CYP21A1P reads)` and its ratio to the median of the sites. The sites, each CYP21A2's base in the reference with CYP21A1P's base at the matching position (dbSNP positions, the CYP21A1P bases read from the reference, checked on 2026-11-04): In2G c.293-13A/C>G (rs6467, chr6:32,039,081; CYP21A2 carries A or C there, CYP21A1P G), I172N c.518T>A (rs6475), V237E c.713T>A (rs12530380, the exon 6 cluster), V281L c.844G>T (rs6471), Q318X c.955C>T (rs7755898) and R356W c.1069C>T (rs7769409). @@ -89,7 +89,7 @@ A site is low when its estimate is more than 0.6 copies under the larger of the ### The interval rows -`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `copy_number.py bed --kind interval`, one `samtools bedcov` over every segment of every row, then `copy_number.py intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. +`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. - **The ClinGen recurrent regions**: every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0, pinned by `CLINGEN_RECURRENT_CNV_URL` and its sha256 in `versions.env`). The file's breakpoint and LCR rows are the segmental duplications that bound the regions. They are cut out of the targets, as are GIAB v3.6 segmental duplications, assembly gaps, satellites and the regions of every pinned Parascopy model (CHRNA7 in 15q13.3 stays with Parascopy). Targets are cut to pieces of at most 500 kb, and their depth is pooled. Controls are two 100 kb windows, one on each side of the cluster of ClinGen rows the region belongs to, clear of all of that and of the targets of the hand rows below (so no row's control sits on another row's target); a side with no clear window within 3 Mb (a chromosome end, as next to 22q11.2 and 15q11.2) is left out. The BED spells its chrX rows `chrx`; the script normalises them, and the engine refuses any contig name that is not a GRCh38 chromosome. The classification of a CNV found here follows the ACMG and ClinGen technical standards (Riggs 2020, PMID 31690835); a call from this step is a lead for a clinical test, not a classification. - **Two more ClinGen curations**: the 1p36 terminal region (ISCA-37434, haploinsufficiency score 3) and the Xq28 region that holds MECP2 (ISCA-46304, triplosensitivity score 3: the MECP2 duplication), from ClinGen's region curation list, read on 2026-11-04. @@ -119,7 +119,7 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. 1. Check that no pinned caller owns the gene: Parascopy (`assets/parascopy_models_v1.7.txt`), pypgx, Paraphase. 2. Choose single-copy controls near the gene, outside any common deletion or duplication there and outside segmental duplications. 3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to `scripts/ci/build-cn-intervals.py` and rerun the script. -4. Run it from a step: `copy_number.py bed --locus NAME`, `samtools bedcov` with the row's passes, then `copy_number.py call --locus NAME --bedcov ... --json ...`. +4. Run it from a step: `og cn bed --locus NAME`, `samtools bedcov` with the row's passes, then `og cn call --locus NAME --bedcov ... --table ...`. 5. Add made-up bedcov tables for the row to `tests/test_copy_number.py`, at every count it can call and at each guard. ## How it is tested @@ -128,7 +128,8 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. - The e2e case `cn-intervals-1-catalog` rebuilds the interval rows from the pinned downloads and fails on any difference with the committed catalog. `cn-intervals-2-hg002` runs the interval blocks of step 35 and of the PARASCOPY module on HG002's real reads at the AZF genes, their controls and 22q11.2: a male with one copy of AZFa and AZFb, AZFc_DAZ not assessed (its estimate, between 3.3 and 3.9, is no whole count of the four DAZ copies, so the row says `between_counts` rather than a call), no change at 22q11.2 A-B, no call on a row off chrY and chr22 (they have no read; the chr22 rows share A-B's control window, so they are left out of that check), and the AZF rows not applicable when the same reads are read as female. - The e2e case `followups-cn-1-couple-report` runs step 39, COUPLE_CARRIER and HTML_REPORT on a made-up couple, plus a stub run of `main.nf`, and checks that the interval table and the CNVpytor calls reach the couple report and `summary.json`. Each wiring line has a red control: the case runs again without it and checks that the rows are gone. -- `tests/test_copy_number.py` checks every row and refuses broken ones. For the three gene-dosage rows it calls made-up depths at 0 to 4 copies, cuts the dosage block out of step 35 and runs it on the made-up bedcov tables in `tests/fixtures/cn/` (HP 3, LCE3BC 1, CCL3L1 5 copies), checks that a row shifted by 1 kb no longer matches its table, and that the PARASCOPY module runs the same list. It runs the owner check with planted AMY1C, C4B and DEFB4 rows. It feeds made-up bedcov tables at 0, 1, 2, 3 and 4 copies and a MAPQ 0 share over the ceiling. It checks that a control with no read stops the run with `BROKEN_CONTROL`. It also checks that the depth tables of `tests/test_hba_copy_number.py` and `tests/test_lpa_kiv2.py` give, through the engine, the same tables the code before the engine wrote. -- The fake-docker case `cn-engine-calls` runs steps 35, 41 and 46 with made-up depths. Each step calls `copy_number.py` once for its regions, step 35 and 46 once more for the call, and none calls the old per-gene scripts. A control exon with no read fails step 46 with `BROKEN_CONTROL`. +- `tests/test_copy_number.py` checks every row and refuses broken ones. For the three gene-dosage rows it calls made-up depths at 0 to 4 copies, cuts the dosage block out of step 35 and runs it on the made-up bedcov tables in `tests/fixtures/cn/` (HP 3, LCE3BC 1, CCL3L1 5 copies), checks that a row shifted by 1 kb no longer matches its table, and that the PARASCOPY module runs the same list. It runs the owner check with planted AMY1C, C4B and DEFB4 rows. It feeds made-up bedcov tables at 0, 1, 2, 3 and 4 copies and a MAPQ 0 share over the ceiling. It checks that a control with no read stops the run with `BROKEN_CONTROL`. With `OG` set (Guard's rust job sets it) it also checks that the depth tables of `tests/test_hba_copy_number.py` and `tests/test_lpa_kiv2.py` give, through `og cn`, the same tables the Python code before it wrote, and that `og cn` refuses the broken rows with the same message as the catalog reader. +- The fake-docker case `cn-engine-calls` runs steps 35, 41 and 46 with made-up depths. Each step calls `og cn` once for its regions, step 35 and 46 once more for the call, step 46 reads the KIV-2 4925G>A marker in the same call, and no step calls a Python copy-number script. A control exon with no read fails step 46 with `BROKEN_CONTROL`. The case needs `og`, so Guard's rust job runs it after the build. - `tests/test_copy_number.py` section 10 checks the CYP21A2 row: its six sites, the pileup parser, and the fixtures `tests/fixtures/cn/CYP21A2_{normal,deletion,conversion}` (2 copies and no finding; 1 copy with the module total 3; an In2G ratio of 0.5 and a possible conversion), with the conversion's depths read with every site even as the red control (no finding). The e2e case `cyp21a2-short-reads-1-hg002` runs the step's and the module's blocks on HG002's reads, unmodified and with a planted conversion and a planted deletion. The same section feeds the deletion's tables with MAPQ 0 shares of 0, 0.019, 0.021 and 0.1 (not assessed twice, then the deletion), and `cn-guard-1-mapq0` runs both blocks on HG002's reads with every MAPQ 0 read removed: not assessed (`low_mapq0_share`), never a deletion. +- `tests/parity` runs `og cn` against the Python engine it replaced on every command and the tables of the three per-gene loci (`scripts/ci/oracle-diff.sh`, [docs/rust.md](rust.md)); the deliberate fixes are its goldens. - The e2e case `cn-engine-1-parity` runs steps 41 and 46 and the HBA block of step 35 on HG002's real reads at the three loci, at full depth and at an eighth of it. The code before the engine is fetched by its commit, 6056452, so the comparison still means something after this change is on main. The LPA and HBA tables and the RhD rows of step 41's table and evidence file must be byte for byte what it wrote. diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index cb1adbf61..01d01d7b2 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -627,6 +627,15 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted - **Why:** HBA had the guard (a second `samtools bedcov -Q 1` pass and `mapq0_min`); the CYP21A2 row was added later without it. On HG002's 35x reads, 9.9% of the depth over the two targets comes from MAPQ 0 reads, with BWA-MEM and with minimap2 `-ax sr` alike, so a share near 0 means the aligner dropped them. - **Fix:** the row runs both bedcov passes with `mapq0_min` 0.02. Under that share it is not assessed (`low_mapq0_share`), and the table's `message` row, which both reports print, says the BAM has few or no multi-mapping reads there. `tests/test_copy_number.py` and the e2e case `cn-guard-1-mapq0` check it. +### The KIV-2 count and its 4925G>A reading disagreed at a MAPQ 0 share of 0.1996 +- **Failed:** step 46's count judged the MAPQ 0 share on the raw value, and the 4925G>A reader judged it on the value the count table printed, rounded to three places. At a share of 0.1996 the count said not assessed (under 0.20) while the marker read 0.200 and said present, a reading from the very reads the count had refused. +- **Why:** the marker ran as a second program that could only see the count's table, not the count's numbers. +- **Fix:** `og cn call --marker LPA_KIV2_4925` reads the marker in the same call as the count and passes it the raw share, so both judge the same number. The parity case `marker-lpa-share1996-kiv-present` holds the fixed output. + +### Two rows of the SMN table over exon 7 gave two different SMN1 counts +- **Failed:** the SMN1 marker took the last row of step 35's SMN table that held exon 7, and the Paraphase comparison took the first. A table with two such rows made the marker message and the comparison read different psCN values. +- **Fix:** one rule in `og cn` (`rust/crates/og-cn/src/smn.rs`): the first row whose region or homologous regions hold exon 7 gives the psCN. The parity case `smn-smn-present-smn-two-rows` holds the fixed output. + ## Downloads (2026-11) ### One bad transfer of a large file failed the whole setup diff --git a/docs/rust.md b/docs/rust.md index 3319bf7c2..bcfa04565 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -18,12 +18,13 @@ rust/ crates/og-filters/ the VCF filters of steps 04, 05 and 07 crates/og-steps/ the step helpers: constraint-join crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table + crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read og/ the binary: og [args] bin/og the shim tasks call; runs bin/.og/og-linux- (og-darwin- on a Mac) tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader step 41's report imports) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2`, `zlib-rs`, `memchr`, `smallvec` and `rust-htslib` so far, declared in `rust/Cargo.toml` and locked in `Cargo.lock`). diff --git a/rust/README.md b/rust/README.md index fb08ae3d2..cf5c44834 100644 --- a/rust/README.md +++ b/rust/README.md @@ -9,6 +9,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-vcf` | VCF records read in place: INFO kept in order, GT and VEP's CSQ fields, the header passed through | | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | +| `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | | `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | From 6d3a80356e5101da3a4a539a6179248bea3dfd0f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:37:19 +0100 Subject: [PATCH 17/53] test(cn): planted faults, the red control (reverted in the next commit) The HBA table says "call" for "called", and the SMN rule takes the last row over exon 7. Guard's rust job and the Rust workflow must go red. Bead: pgp-bs9u.24 --- rust/crates/og-cn/src/layout.rs | 2 +- rust/crates/og-cn/src/smn.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-cn/src/layout.rs b/rust/crates/og-cn/src/layout.rs index cfc683669..9059e3f77 100644 --- a/rust/crates/og-cn/src/layout.rs +++ b/rust/crates/og-cn/src/layout.rs @@ -121,7 +121,7 @@ fn hba(row: &Locus, b: &Block) -> Table { let share = b.mapq0_share; let est = b.estimate; let (status, genes, message) = match (b.status, b.reason) { - (Status::Called, _) => ("called", d(b.copies), b.reading.clone()), + (Status::Called, _) => ("call", d(b.copies), b.reading.clone()), (_, Some(Reason::LowControlDepth)) => ( "not assessed", ".".into(), diff --git a/rust/crates/og-cn/src/smn.rs b/rust/crates/og-cn/src/smn.rs index fba00a73b..613c905b3 100644 --- a/rust/crates/og-cn/src/smn.rs +++ b/rust/crates/og-cn/src/smn.rs @@ -115,7 +115,8 @@ pub fn read(path: Option<&str>, chrom: &str, marker_copies: &[i64]) -> Result Date: Wed, 11 Nov 2026 18:37:39 +0100 Subject: [PATCH 18/53] perf(family): read the joint VCF once and check it is sorted Step 38 and the Nextflow family workflow now run og family plan, prepare, check, report and segments, ported from family_report.py and family_segments.py. The report reads the joint VCF once, compresses each child's inheritance table on its own thread at gzip level 6, and smooths the chromosome map in O(R log R) with the same tie-breaks. A joint VCF that is not sorted now stops it with exit 2. family_segments.py is gone. family_report.py keeps only the names its importers use, and og dev remnants checks that against tests/parity/remnants.tsv. The new og-pedigree crate holds the pedigree rules. --- .github/workflows/guard.yml | 40 +- CHANGELOG.md | 8 + bin/family_report.py | 891 +------------ bin/family_segments.py | 453 ------- bin/owngenome_live_family.py | 2 +- docs/multi-sample.md | 4 +- docs/rust.md | 7 +- main.nf | 2 +- modules/local/glnexus/main.nf | 8 +- modules/local/somalier/main.nf | 4 +- modules/local/whatshap_ped/main.nf | 6 +- rust/Cargo.lock | 15 + rust/Cargo.toml | 3 +- rust/README.md | 1 + rust/crates/og-dev/src/lib.rs | 7 +- rust/crates/og-dev/src/parity.rs | 15 + rust/crates/og-dev/src/parity/self_test.rs | 5 + rust/crates/og-dev/src/remnants.rs | 441 +++++++ rust/crates/og-pedigree/Cargo.toml | 15 + rust/crates/og-pedigree/src/lib.rs | 849 +++++++++++++ rust/crates/og-pedigree/src/tests.rs | 122 ++ rust/crates/og-steps/Cargo.toml | 8 +- rust/crates/og-steps/src/family/args.rs | 229 ++++ rust/crates/og-steps/src/family/check.rs | 190 +++ rust/crates/og-steps/src/family/mod.rs | 94 ++ rust/crates/og-steps/src/family/plan.rs | 350 ++++++ rust/crates/og-steps/src/family/prepare.rs | 238 ++++ rust/crates/og-steps/src/family/report.rs | 1022 +++++++++++++++ rust/crates/og-steps/src/family/segments.rs | 1110 +++++++++++++++++ .../og-steps/src/family/segments/tests.rs | 110 ++ rust/crates/og-steps/src/family/vcf.rs | 272 ++++ rust/crates/og-steps/src/lib.rs | 3 + rust/og/src/main.rs | 3 +- scripts/38-family.sh | 24 +- scripts/39-couple.sh | 4 +- scripts/ci/e2e-case-paths.tsv | 18 +- scripts/run-all.sh | 12 +- tests/e2e/family-joint-1.sh | 2 +- tests/fake-docker/run-all-families.sh | 24 +- tests/fixtures/families/README.md | 4 +- tests/parity/README.md | 2 +- tests/parity/cases.tsv | 74 ++ tests/parity/gen/family-3gen.sh | 37 + tests/parity/gen/family-bench.sh | 29 + tests/parity/gen/family-oracle.sh | 28 + .../golden/family/plan-missing-file/exit | 1 + .../golden/family/plan-missing-file/stderr | 1 + .../out/kid_inheritance.tsv.gz.absent | 0 .../out/kid_inheritance.tsv.gz.absent | 0 .../parity/golden/family/report-unsorted/exit | 1 + .../out/compound_het.tsv.absent | 0 .../report-unsorted/out/de_novo.tsv.absent | 0 .../out/family_report.md.absent | 0 .../out/kid_dnapainter.csv.absent | 0 .../out/kid_inheritance.tsv.gz.absent | 0 .../out/kid_segments.svg.absent | 0 .../out/kid_segments.tsv.absent | 0 .../golden/family/report-unsorted/stderr | 1 + .../golden/family/report-unsorted/stdout | 0 tests/parity/inputs/family/child_a.g.vcf | 3 + tests/parity/inputs/family/dad.g.vcf | 3 + tests/parity/inputs/family/daughter.vcf | 7 + tests/parity/inputs/family/duo.vcf | 7 + tests/parity/inputs/family/fam-allmissing.tsv | 4 + .../parity/inputs/family/fam-famissample.tsv | 3 + tests/parity/inputs/family/fam-famname.tsv | 2 + .../parity/inputs/family/fam-parentother.tsv | 3 + tests/parity/inputs/family/fam-partial.tsv | 3 + .../parity/inputs/family/fam-partnerother.tsv | 3 + tests/parity/inputs/family/fam-twofam.tsv | 4 + tests/parity/inputs/family/father_a.g.vcf | 3 + tests/parity/inputs/family/hg003.g.vcf | 3 + tests/parity/inputs/family/kid.g.vcf | 3 + tests/parity/inputs/family/kid_clinical.vcf | 8 + tests/parity/inputs/family/low.vcf | 5 + .../parity/inputs/family/members-3gen-duo.tsv | 6 + tests/parity/inputs/family/members-3gen.tsv | 6 + tests/parity/inputs/family/members-bench.tsv | 6 + .../parity/inputs/family/members-daughter.tsv | 5 + tests/parity/inputs/family/members-duo.tsv | 5 + tests/parity/inputs/family/members-swap.tsv | 5 + tests/parity/inputs/family/members-trio.tsv | 5 + tests/parity/inputs/family/mother_a.g.vcf | 3 + tests/parity/inputs/family/mum.g.vcf | 3 + tests/parity/inputs/family/mum_ann.vcf | 4 + tests/parity/inputs/family/nochrom.g.vcf | 2 + tests/parity/inputs/family/nochrom.vcf | 1 + tests/parity/inputs/family/nocol.vcf | 6 + tests/parity/inputs/family/pairs-duo.tsv | 2 + tests/parity/inputs/family/pairs-few.tsv | 4 + tests/parity/inputs/family/pairs-lost.tsv | 3 + tests/parity/inputs/family/pairs-ok.tsv | 4 + tests/parity/inputs/family/pairs-sib.tsv | 4 + tests/parity/inputs/family/pairs-warn.tsv | 4 + tests/parity/inputs/family/par.bed | 4 + tests/parity/inputs/family/ped-badname.tsv | 5 + tests/parity/inputs/family/ped-badsex.tsv | 4 + tests/parity/inputs/family/ped-crlf.tsv | 7 + tests/parity/inputs/family/ped-empty.tsv | 0 .../parity/inputs/family/ped-femalefather.tsv | 4 + tests/parity/inputs/family/ped-loop.tsv | 4 + tests/parity/inputs/family/ped-nohead.tsv | 1 + tests/parity/inputs/family/ped-ownparent.tsv | 4 + tests/parity/inputs/family/ped-partners.tsv | 4 + tests/parity/inputs/family/ped-samefm.tsv | 4 + tests/parity/inputs/family/ped-secondrow.tsv | 5 + tests/parity/inputs/family/ped-toomany.tsv | 5 + tests/parity/inputs/family/ped-unknowncol.tsv | 2 + .../parity/inputs/family/pedigree.example.tsv | 4 + tests/parity/inputs/family/phased.vcf | 7 + tests/parity/inputs/family/rel-duo.tsv | 2 + tests/parity/inputs/family/rel-empty.tsv | 1 + tests/parity/inputs/family/rel.tsv | 4 + tests/parity/inputs/family/trio.tsv | 4 + tests/parity/inputs/family/trio.vcf | 27 + tests/parity/inputs/family/two_families.tsv | 7 + tests/parity/inputs/family/unsorted.vcf | 8 + tests/parity/oracle.ref | 1 + tests/parity/remnants.tsv | 6 + tests/test_family_report.py | 36 +- tests/test_family_segments.py | 29 +- 121 files changed, 5692 insertions(+), 1428 deletions(-) delete mode 100755 bin/family_segments.py create mode 100644 rust/crates/og-dev/src/remnants.rs create mode 100644 rust/crates/og-pedigree/Cargo.toml create mode 100644 rust/crates/og-pedigree/src/lib.rs create mode 100644 rust/crates/og-pedigree/src/tests.rs create mode 100644 rust/crates/og-steps/src/family/args.rs create mode 100644 rust/crates/og-steps/src/family/check.rs create mode 100644 rust/crates/og-steps/src/family/mod.rs create mode 100644 rust/crates/og-steps/src/family/plan.rs create mode 100644 rust/crates/og-steps/src/family/prepare.rs create mode 100644 rust/crates/og-steps/src/family/report.rs create mode 100644 rust/crates/og-steps/src/family/segments.rs create mode 100644 rust/crates/og-steps/src/family/segments/tests.rs create mode 100644 rust/crates/og-steps/src/family/vcf.rs create mode 100644 tests/parity/gen/family-3gen.sh create mode 100644 tests/parity/gen/family-bench.sh create mode 100644 tests/parity/gen/family-oracle.sh create mode 100644 tests/parity/golden/family/plan-missing-file/exit create mode 100644 tests/parity/golden/family/plan-missing-file/stderr create mode 100644 tests/parity/golden/family/report-no-column/out/kid_inheritance.tsv.gz.absent create mode 100644 tests/parity/golden/family/report-nochrom/out/kid_inheritance.tsv.gz.absent create mode 100644 tests/parity/golden/family/report-unsorted/exit create mode 100644 tests/parity/golden/family/report-unsorted/out/compound_het.tsv.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/de_novo.tsv.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/family_report.md.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/kid_dnapainter.csv.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/kid_inheritance.tsv.gz.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/kid_segments.svg.absent create mode 100644 tests/parity/golden/family/report-unsorted/out/kid_segments.tsv.absent create mode 100644 tests/parity/golden/family/report-unsorted/stderr create mode 100644 tests/parity/golden/family/report-unsorted/stdout create mode 100644 tests/parity/inputs/family/child_a.g.vcf create mode 100644 tests/parity/inputs/family/dad.g.vcf create mode 100644 tests/parity/inputs/family/daughter.vcf create mode 100644 tests/parity/inputs/family/duo.vcf create mode 100644 tests/parity/inputs/family/fam-allmissing.tsv create mode 100644 tests/parity/inputs/family/fam-famissample.tsv create mode 100644 tests/parity/inputs/family/fam-famname.tsv create mode 100644 tests/parity/inputs/family/fam-parentother.tsv create mode 100644 tests/parity/inputs/family/fam-partial.tsv create mode 100644 tests/parity/inputs/family/fam-partnerother.tsv create mode 100644 tests/parity/inputs/family/fam-twofam.tsv create mode 100644 tests/parity/inputs/family/father_a.g.vcf create mode 100644 tests/parity/inputs/family/hg003.g.vcf create mode 100644 tests/parity/inputs/family/kid.g.vcf create mode 100644 tests/parity/inputs/family/kid_clinical.vcf create mode 100644 tests/parity/inputs/family/low.vcf create mode 100644 tests/parity/inputs/family/members-3gen-duo.tsv create mode 100644 tests/parity/inputs/family/members-3gen.tsv create mode 100644 tests/parity/inputs/family/members-bench.tsv create mode 100644 tests/parity/inputs/family/members-daughter.tsv create mode 100644 tests/parity/inputs/family/members-duo.tsv create mode 100644 tests/parity/inputs/family/members-swap.tsv create mode 100644 tests/parity/inputs/family/members-trio.tsv create mode 100644 tests/parity/inputs/family/mother_a.g.vcf create mode 100644 tests/parity/inputs/family/mum.g.vcf create mode 100644 tests/parity/inputs/family/mum_ann.vcf create mode 100644 tests/parity/inputs/family/nochrom.g.vcf create mode 100644 tests/parity/inputs/family/nochrom.vcf create mode 100644 tests/parity/inputs/family/nocol.vcf create mode 100644 tests/parity/inputs/family/pairs-duo.tsv create mode 100644 tests/parity/inputs/family/pairs-few.tsv create mode 100644 tests/parity/inputs/family/pairs-lost.tsv create mode 100644 tests/parity/inputs/family/pairs-ok.tsv create mode 100644 tests/parity/inputs/family/pairs-sib.tsv create mode 100644 tests/parity/inputs/family/pairs-warn.tsv create mode 100644 tests/parity/inputs/family/par.bed create mode 100644 tests/parity/inputs/family/ped-badname.tsv create mode 100644 tests/parity/inputs/family/ped-badsex.tsv create mode 100644 tests/parity/inputs/family/ped-crlf.tsv create mode 100644 tests/parity/inputs/family/ped-empty.tsv create mode 100644 tests/parity/inputs/family/ped-femalefather.tsv create mode 100644 tests/parity/inputs/family/ped-loop.tsv create mode 100644 tests/parity/inputs/family/ped-nohead.tsv create mode 100644 tests/parity/inputs/family/ped-ownparent.tsv create mode 100644 tests/parity/inputs/family/ped-partners.tsv create mode 100644 tests/parity/inputs/family/ped-samefm.tsv create mode 100644 tests/parity/inputs/family/ped-secondrow.tsv create mode 100644 tests/parity/inputs/family/ped-toomany.tsv create mode 100644 tests/parity/inputs/family/ped-unknowncol.tsv create mode 100644 tests/parity/inputs/family/pedigree.example.tsv create mode 100644 tests/parity/inputs/family/phased.vcf create mode 100644 tests/parity/inputs/family/rel-duo.tsv create mode 100644 tests/parity/inputs/family/rel-empty.tsv create mode 100644 tests/parity/inputs/family/rel.tsv create mode 100644 tests/parity/inputs/family/trio.tsv create mode 100644 tests/parity/inputs/family/trio.vcf create mode 100644 tests/parity/inputs/family/two_families.tsv create mode 100644 tests/parity/inputs/family/unsorted.vcf create mode 100644 tests/parity/remnants.tsv diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index feccd7079..0f2416466 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -64,10 +64,35 @@ jobs: fake-docker: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # generate-report.sh runs og run delta, and the run-state case runs og + # run state, so the cases need og in bin/.og, stamped with this + # checkout's rust/ tree so run-all.sh's stale-binary check passes. One + # host build is enough: the fake docker runs og on the host. + - name: Install the toolchain rust/rust-toolchain.toml pins + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + key: fake-docker + + - name: Build og into bin/.og + working-directory: rust + run: | + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og + mkdir -p ../bin/.og + cp target/release/og "../bin/.og/og-linux-$(uname -m)" + ./og.sh check + ../bin/og version + - name: Self-test (a swallowed empty image, an unknown option and a failing case must be caught) run: scripts/ci/run-fake-docker-suite.sh --self-test @@ -222,8 +247,8 @@ jobs: git checkout -- og/src/main.rs ./og.sh check --required - # The tests of the og filters steps 04, 05 and 07 run, and og chip's - # (step 49): unit-tests has + # The tests of the og filters steps 04, 05 and 07 run, og chip's + # (step 49) and og family's (step 38): unit-tests has # no og, so there they skip; here OG_REQUIRED turns a skip into a # failure. The step 05 tests run the real step with bin/og. - name: The og filter tests and step 05 with og @@ -235,7 +260,8 @@ jobs: fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ - tests/test_chip_report.py; do + tests/test_chip_report.py \ + tests/test_family_report.py tests/test_family_segments.py; do rc=0 echo "::group::${t}" case "$t" in @@ -247,6 +273,12 @@ jobs: done exit "$fail" + # A Python file kept only for what other Python imports from it holds + # those names and nothing else (tests/parity/remnants.tsv). + - name: og dev remnants + working-directory: . + run: bin/og dev remnants + # main.nf runs the same check before any task: a stale og stops # `nextflow run` with the setup command. Without the check, this stub # run completes and the step fails. diff --git a/CHANGELOG.md b/CHANGELOG.md index a197c3414..0f739c6f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 38 (family analysis) and the Nextflow family workflow now run in the Rust `og` binary: `og family plan`, `prepare`, `check`, `report` and `segments` replace `bin/family_report.py` and `bin/family_segments.py`. `family_segments.py` is gone. `family_report.py` keeps only the pedigree, kinship and PAR rules that step 39, the IBD report, the pedigree drawing and the SV steps still import, and `og dev remnants` fails if anything else comes back. `og family report` reads the joint VCF once (twice at the same time with `--phased`) and compresses each child's inheritance table on its own thread. On a synthetic five-person family of 1,000,000 sites it takes 0.9 s and 26 MB instead of 24 s and 85 MB, and at 5,000,000 sites 4.3 s and 93 MB instead of 161 s and 307 MB. Every table, map and report is the same as before, by content. What changes: + - A joint VCF that is not sorted (a position that goes back, or a chromosome that comes back) now stops `og family report` and `og family segments` with exit 2 and names the two records. The Python split the chromosome map at the wrong places and said nothing. + - The inheritance tables are gzip level 6, not 9. They decompress to the same bytes. + - Smoothing the chromosome map takes O(R log R) time instead of O(R²) in the number of runs, with the same merges in the same order. + - A missing input file, a genotype or position that is not a number, or a record before the `#CHROM` line now stops with an `ERROR:` line and exit 2, not a Python traceback. + - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. + - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). + - Step 49 (clonal haematopoiesis) now prepares the CHIP gene models and writes its two tables with the Rust `og` binary: `og chip prepare`, `og chip report` and `og chip check-table` replace the commands of `bin/chip_report.py`, which keeps only the detection limit and the column names the reports import. On Ensembl 116's full GFF3, prepare takes 1.7 s and 6 MB instead of 15 s and 20 MB. The two tables, `chip.gff3` and `chip.bed` are the same, byte for byte. What changes: a missing input file or a position that is not a whole number stops with an `ERROR:` line and exit 1, not a Python traceback, and each output is written whole or not at all. Step 49 now needs `og` (`scripts/setup.sh --og`). - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. diff --git a/bin/family_report.py b/bin/family_report.py index dc611c40d..044940c32 100755 --- a/bin/family_report.py +++ b/bin/family_report.py @@ -1,94 +1,25 @@ #!/usr/bin/env python3 -"""family_report.py: the family analysis of step 38 (scripts/38-family.sh) and -of the Nextflow FAMILY workflow (workflows/family.nf). Both run prepare, check -and report, in this order; plan says beforehand which family analyses run. -Standard library only, so it runs in the plain python image. - - A families file is a pedigree (assets/pedigree.example.tsv: sample, sex, - father, mother, partner, tab separated) with an optional leading family - column (assets/families.example.tsv), so one file can declare several - families. A file without the column is one family, read as before. - - family_report.py plan --families F [--family NAME] [--sample S [--sex SEX]] - [--have-gvcf ID ...] [--have-clinvar ID ...] [--ibd-panel yes|no] - [--write-pedigree DIR] [--which] [--out family_plan.tsv] - Checks the file and picks one family: NAME, else the family of S, - else the file's only one. With --sex, S's sex in the file must agree. - --which prints the family's name (nothing when S is in none) and - stops. Otherwise writes, for each analysis of steps 38 and 39 - (joint_call, trio_phasing, inheritance_map, shared_segments, and - couple_carrier per couple), whether it runs, is skipped or is waiting - for a member's own run, and why (family, analysis, members, decision, - reason); --write-pedigree writes DIR//pedigree.tsv, the - family's rows without the family column. Exits 2 on a problem, naming - the file and line. - - family_report.py prepare --pedigree P.tsv [--family NAME] --gvcf ID=PATH ... [--declared-sex ID=SEX ...] - [--have-clinvar ID ...] [--ibd-panel yes|no] [--pedigree-out PED] --out-dir D - Reads and checks the pedigree, or one family of a families file, and - checks that each gVCF names its own sample. Writes - D/members.tsv (every person of the pedigree, sequenced or not), - D/family.ped (somalier relate's pedigree: the sequenced members), - D/trios.ped (WhatsHap's: the children whose two parents are - sequenced; empty when there is none) and D/family_plan.tsv (plan's - table for these members). Exits 2 on a problem. - - family_report.py check --members D/members.tsv --pairs family.pairs.tsv --out relationships.tsv - For every pair of sequenced members, compares the relatedness the - pedigree implies with the one somalier relate measured. Exits 1 when - a pair disagrees: a sample swap, or a wrong line in the pedigree. - - family_report.py report --members D/members.tsv --relationships R.tsv --vcf FAMILY.vcf.gz - [--phased FAMILY.phased.vcf.gz] [--candidates ID=VCF ...] - [--par-bed par_grch38.bed] --out-dir D - Per child with a sequenced parent: the allele each parent passed on at - every site of the joint VCF where the genotypes tell - (_inheritance.tsv.gz), de novo candidates when both parents - are sequenced (de_novo.tsv), and pairs of candidate variants in one - gene in trans or in cis (compound_het.tsv), and the chromosome map of - bin/family_segments.py (_segments.tsv, _dnapainter.csv, - _segments.svg). family_report.md says it in words, pair by - pair and child by child. +# remnant: kept for couple_report, ibd_report, pedigree_svg, annotsv_postfilter, cnvpytor_to_vcf; deleted in lane R4 +"""family_report.py: what is left of the family analysis of step 38 once it +moved to `og family` (rust/crates/og-steps/src/family). Only the pedigree, +relatedness and PAR rules that other Python tools still import are kept +here; `og dev remnants` (tests/parity/remnants.tsv) checks that nothing +else comes back. Standard library only. """ -import argparse -import gzip -import itertools -import os import re import sys COLUMNS = ("family", "sample", "sex", "father", "mother", "partner") -# The pedigree's own columns: a families file is a pedigree with a leading -# family column (assets/families.example.tsv), and each family's rows are -# written out as a pedigree of their own, without it. -PED_COLUMNS = COLUMNS[1:] -# The family of a file without the family column. -DEFAULT_FAMILY = "family" -# The analyses plan decides, in order: step 38's four, then step 39's (one row -# per couple). -ANALYSES = ("joint_call", "trio_phasing", "inheritance_map", "shared_segments", "couple_carrier") -PLAN_COLUMNS = ("family", "analysis", "members", "decision", "reason") MISSING = {"", ".", "0", "-9"} SEX = {"male": "male", "m": "male", "1": "male", "female": "female", "f": "female", "2": "female", "unknown": None, "": None, ".": None, "0": None, "-9": None} NAME = re.compile(r"^[A-Za-z0-9._-]+$") -# somalier relate needs this many sites both samples have a genotype at -# before its relatedness is read as a verdict. -MIN_SITES = 1000 # Relatedness (twice the kinship coefficient) bins, as KING draws them: the # geometric middle between two degrees. BINS = ((0.707, "same person"), (0.354, "first degree"), (0.177, "second degree"), (0.0884, "third degree")) -# A parent and child share an allele at every site: the share of sites where -# they are opposite homozygotes (IBS0) stays near 0 (genotype errors only). -PARENT_CHILD_MAX_IBS0 = 0.01 -# De novo candidates: every one of the three must have these. -MIN_GQ = 20 -MIN_DP = 10 -HET_AB = (0.25, 0.75) -PRIMARY = {"chr%s" % c for c in list(range(1, 23)) + ["X", "Y"]} def die(lines, code=2): @@ -97,33 +28,10 @@ def die(lines, code=2): sys.exit(code) -def open_text(path): - with open(path, "rb") as f: - magic = f.read(2) - return gzip.open(path, "rt") if magic == b"\x1f\x8b" else open(path) - - -class GzOut: - """A gzip file whose bytes depend on its content only (no name, mtime 0).""" - - def __init__(self, path): - self.raw = open(path, "wb") - self.gz = gzip.GzipFile(filename="", mode="wb", fileobj=self.raw, mtime=0) - - def write(self, b): - self.gz.write(b) - - def close(self): - self.gz.close() - self.raw.close() - - def missing(v): return v is None or v.strip() in MISSING -# --- pedigree --------------------------------------------------------------- - def read_pedigree(path): """Rows of the pedigree TSV as dicts, and the problems found. The first non-blank line is the header (a leading '#' is dropped); later lines @@ -249,238 +157,10 @@ def build_members(rows, declared, path, source="the samplesheet"): return members, errors -# --- families file ------------------------------------------------------------ - def bad_name(v): return not NAME.match(v) or v.startswith("-") -def split_families(rows, path): - """{family: rows} of a families file, and the problems found. A file - without the family column is one family, keyed None. The family column - is filled on every row or on none; a person is in one family; the - parents and partner a row names are in its own family; a family name - follows the sample-name rule and is no person's name (it names the - family's folder beside the samples' folders).""" - has = any("family" in r for r in rows) - if not has: - return ({None: rows} if rows else {}), [] - errors, fams, where_of = [], {}, {} - filled = [r for r in rows if not missing(r.get("family"))] - for r in rows: - fam = r.get("family", "").strip() - where = "%s line %d" % (path, r["line"]) - if missing(fam): - if filled: - errors.append("%s: no family name, but other rows have one (line %d: '%s'); fill the family " - "column on every row, or leave it out of the file" - % (where, filled[0]["line"], filled[0]["family"].strip())) - continue - if bad_name(fam): - errors.append("%s: family name '%s' may hold only letters, digits, '.', '_' and '-' and may not start " - "with '-' (it names the family's folder)" % (where, fam)) - continue - s = r.get("sample", "").strip() - if s and not missing(s): - if s in where_of and where_of[s][0] != fam: - errors.append("%s: sample '%s' is in family '%s' here and in family '%s' on line %d; a person " - "belongs to one family" % (where, s, fam, where_of[s][0], where_of[s][1])) - continue - where_of.setdefault(s, (fam, r["line"])) - fams.setdefault(fam, []).append(r) - if not filled: - return ({None: rows} if rows else {}), errors - people = {} - for r in rows: - for k in ("sample", "father", "mother", "partner"): - v = r.get(k, "").strip() - if v and not missing(v): - people.setdefault(v, r["line"]) - for fam in sorted(fams): - if fam in people: - errors.append("%s: the family name '%s' is also a person's name (line %d); its folder would be the " - "sample's. Give the family another name" % (path, fam, people[fam])) - for fam, rs in fams.items(): - for r in rs: - s = r.get("sample", "").strip() - for k in ("father", "mother", "partner"): - v = r.get(k, "").strip() - if v and not missing(v) and v in where_of and where_of[v][0] != fam: - errors.append("%s line %d: the %s of '%s', '%s', is in family '%s' (line %d), '%s' in family " - "'%s'; a family lists its own parents and partners" - % (path, r["line"], k, s, v, where_of[v][0], where_of[v][1], s, fam)) - return fams, errors - - -def family_rows(path, family=None, sample=None): - """(name, rows) of one family of a families file (or a pedigree): the - family named, else the one --sample is in, else the file's only family. - name is DEFAULT_FAMILY (or the --family given) for a file without the - family column; rows is [] when sample is in no family. Exits 2 on a - problem.""" - rows, errors = read_pedigree(path) - if errors: - die(errors) - fams, errors = split_families(rows, path) - if errors: - die(errors) - if None in fams: - rs, name = fams[None], family or DEFAULT_FAMILY - if bad_name(name): - die(["family name '%s' may hold only letters, digits, '.', '_' and '-' and may not start with '-' " - "(it names the family's folder)" % name]) - for r in rs: - if name in (r.get(k, "").strip() for k in ("sample", "father", "mother", "partner")): - die(["%s line %d: the family name '%s' is also a person's name; its folder would be the " - "sample's. Give the family another name" % (path, r["line"], name)]) - if sample and not family and not any(r.get("sample", "").strip() == sample for r in rs): - return name, [] - return name, rs - names = sorted(fams) - if family: - if family not in fams: - die(["%s has no family '%s'; its families are: %s" % (path, family, ", ".join(names) or "none")]) - return family, fams[family] - if sample: - for name in names: - if any(r.get("sample", "").strip() == sample for r in fams[name]): - return name, fams[name] - return None, [] - if len(names) > 1: - die(["%s holds %d families (%s); name the one to run (--family, or --family_name in Nextflow)" - % (path, len(names), ", ".join(names))]) - return (names[0], fams[names[0]]) if names else (DEFAULT_FAMILY, []) - - -def write_pedigree(path, rows): - """A family's rows as a pedigree of its own (the pedigree columns, no - family column), read back the same way.""" - with open(path, "w") as f: - f.write("\t".join(PED_COLUMNS) + "\n") - for r in rows: - f.write("\t".join(r.get(k, "").strip() or "." for k in PED_COLUMNS) + "\n") - - -def plan_family(name, members, have_gvcf, have_clinvar, ibd_panel): - """The rows of family_plan.tsv: for each analysis of steps 38 and 39, - whether it runs, is skipped, or waits for a member's own run, and why, - in words. members is build_members' table; have_gvcf and have_clinvar - are the samples with a gVCF and with step 06's ClinVar hits.""" - rows_of = sorted(s for s, m in members.items() if m["row"]) - seq = [s for s in rows_of if s in have_gvcf] - lacking = [s for s in rows_of if s not in have_gvcf] - - def no_gvcf(ss): - return "; ".join("%s has no gVCF yet: run run-all.sh %s first" % (s, s) for s in ss) - - out = [] - - def row(analysis, who, decision, reason): - out.append({"family": name, "analysis": analysis, "members": ",".join(who) or ".", - "decision": decision, "reason": reason}) - - if len(rows_of) < 2: - joint = ("skipped", "the family has %d sample with a row (%s); the family analysis needs two or more" - % (len(rows_of), ", ".join(rows_of) or "none")) - elif len(seq) < 2: - joint = ("waiting", no_gvcf(lacking)) - else: - why = "GLnexus joint call, somalier relationship check and the family report of %s" % ", ".join(seq) - if lacking: - why += "; left out: %s (run step 38 again after its run)" % no_gvcf(lacking) - joint = ("runs", why) - row("joint_call", seq if joint[0] == "runs" else rows_of, *joint) - - trios = [s for s in seq if members[s]["father"] in seq and members[s]["mother"] in seq] - duos = [s for s in seq if s not in trios and (members[s]["father"] in seq or members[s]["mother"] in seq)] - if joint[0] != "runs": - row("trio_phasing", rows_of, *joint) - row("inheritance_map", rows_of, *joint) - row("shared_segments", rows_of, *joint) - else: - if trios: - row("trio_phasing", trios, "runs", "WhatsHap phases %s by inheritance from both parents" % ", ".join(trios)) - else: - row("trio_phasing", duos, "skipped", "no child has both parents sequenced%s; phasing by inheritance " - "needs a trio" % ((" (%s has one)" % ", ".join(duos)) if duos else "")) - kids = trios + duos - if kids: - row("inheritance_map", sorted(kids), "runs", "the allele each sequenced parent passed on, and the " - "chromosome map, of %s" % ", ".join(sorted(kids))) - else: - row("inheritance_map", [], "skipped", "no child has a sequenced parent") - if ibd_panel: - row("shared_segments", seq, "runs", "Beagle phases the joint call against step 26b's panel and hap-ibd " - "lists the DNA each pair shares") - else: - row("shared_segments", seq, "skipped", "no phasing panel: install it with setup.sh --ancestry-fine-panel") - - couples = sorted({tuple(sorted((s, members[s]["partner"]))) for s in rows_of - if members[s]["partner"] and members[members[s]["partner"]]["row"]}) - if not couples: - row("couple_carrier", [], "skipped", "no two samples of the family are marked as partners") - # Step 39 reads each partner's ClinVar hits (and Delly, EH, SMN when present), - # never a gVCF: a partner run from a VCF alone still gets the screen. - for a, b in couples: - lack = [s for s in (a, b) if s not in have_clinvar] - if lack: - row("couple_carrier", [a, b], "waiting", "; ".join( - "%s has no ClinVar hits yet (step 06 writes clinvar/%s_clinvar_hits.vcf): run run-all.sh %s first" - % (s, s, s) for s in lack)) - else: - row("couple_carrier", [a, b], "runs", "carrier screen of %s and %s" % (a, b)) - return out - - -def write_plan(path, rows): - with open(path, "w") as f: - f.write("\t".join(PLAN_COLUMNS) + "\n") - for r in rows: - f.write("\t".join(r[k] for k in PLAN_COLUMNS) + "\n") - - -def cmd_plan(a): - sex = (a.sex or "").strip().lower() - if sex and sex not in ("male", "female"): - die(["--sex %s: the sex is male or female" % a.sex]) - if sex and not a.sample: - die(["--sex needs --sample"]) - name, rows = family_rows(a.families, a.family, a.sample) - if a.sample and not rows: - if a.which: - return - die(["%s is in no family of %s" % (a.sample, a.families)]) - declared = {a.sample: sex} if sex else {} - members, errors = build_members(rows, declared, a.families, "the sex given to run-all.sh") - if errors: - die(errors) - if a.sample and rows and a.sample not in members: - die(["%s: '%s' is not in family '%s'" % (a.families, a.sample, name)]) - if a.which: - print(name) - return - if a.write_pedigree: - d = os.path.join(a.write_pedigree, name) - os.makedirs(d, exist_ok=True) - write_pedigree(os.path.join(d, "pedigree.tsv"), rows) - plan = plan_family(name, members, set(a.have_gvcf or []), set(a.have_clinvar or []), a.ibd_panel == "yes") - if a.out: - write_plan(a.out, plan) - for r in plan: - print("%s %-16s %-8s %s" % (r["family"], r["analysis"], r["decision"], r["reason"])) - - -def gvcf_samples(path): - """Sample columns of a VCF's #CHROM line (reads the header only).""" - with open_text(path) as f: - for line in f: - if line.startswith("#CHROM"): - return line.rstrip("\r\n").split("\t")[9:] - if not line.startswith("#"): - break - return None - - def kinship_table(members): """phi(a, b): the kinship coefficient of two members, from the pedigree. Founders (no parent known) are unrelated; an unknown parent is a founder.""" @@ -534,162 +214,6 @@ def degree(r): return "unrelated" -def read_members(path): - members = {} - with open(path) as f: - head = f.readline().rstrip("\n").split("\t") - for line in f: - r = dict(zip(head, line.rstrip("\n").split("\t"))) - members[r["sample"]] = { - "sex": r["sex"] or None, "father": r["father"] or None, "mother": r["mother"] or None, - "partner": r["partner"] or None, "row": r["in_pedigree"] == "yes", - "sequenced": r["sequenced"] == "yes"} - return members - - -def cmd_prepare(a): - declared = {} - for kv in a.declared_sex or []: - k, _, v = kv.partition("=") - v = v.strip().lower() - if v: - if v not in ("male", "female"): - die(["--declared-sex %s: the sex is male or female" % kv]) - declared[k] = v - name, rows = family_rows(a.pedigree, a.family) - members, errors = build_members(rows, declared, a.pedigree) - gvcfs = {} - for kv in a.gvcf or []: - k, _, v = kv.partition("=") - gvcfs[k] = v - seq = sorted(s for s in gvcfs if s in members and members[s]["row"]) - if len(seq) < 2: - errors.append("the pedigree names %d of the processed samples (%s); the family analysis needs two " - "or more, each with a row in %s and a gVCF" - % (len(seq), ", ".join(seq) or "none", a.pedigree)) - for s in seq: - names = gvcf_samples(gvcfs[s]) - if names != [s]: - errors.append("the gVCF of '%s' (%s) holds the sample%s %s, not '%s'. The joint call names each " - "column after the gVCF's own sample. Rename it: echo %s > name.txt; bcftools reheader " - "-s name.txt -o %s.g.vcf.gz %s; bcftools index -t %s.g.vcf.gz" - % (s, os.path.basename(gvcfs[s]), "s" if names and len(names) > 1 else "", - ", ".join(names) if names else "none (no #CHROM line)", s, s, s, - os.path.basename(gvcfs[s]), s)) - if errors: - die(errors) - for s in seq: - members[s]["sequenced"] = True - phi = kinship_table(members) - os.makedirs(a.out_dir, exist_ok=True) - with open(os.path.join(a.out_dir, "members.tsv"), "w") as f: - f.write("sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n") - for s in sorted(members): - m = members[s] - f.write("\t".join([s, m["sex"] or "", m["father"] or "", m["mother"] or "", m["partner"] or "", - "yes" if m["row"] else "no", "yes" if m.get("sequenced") else "no"]) + "\n") - code = {"male": "1", "female": "2"} - with open(os.path.join(a.out_dir, "family.ped"), "w") as f: - for s in seq: - m = members[s] - f.write("\t".join(["family", s, m["father"] if m["father"] in seq else "-9", - m["mother"] if m["mother"] in seq else "-9", - code.get(m["sex"], "-9"), "-9"]) + "\n") - trios = [s for s in seq if members[s]["father"] in seq and members[s]["mother"] in seq] - with open(os.path.join(a.out_dir, "trios.ped"), "w") as f: - for s in trios: - m = members[s] - f.write("\t".join(["family", s, m["father"], m["mother"], code.get(m["sex"], "0"), "0"]) + "\n") - if a.pedigree_out: - write_pedigree(a.pedigree_out, rows) - # The same decision plan writes before the step runs (run-all.sh) - write_plan(os.path.join(a.out_dir, "family_plan.tsv"), - plan_family(name, members, set(seq), set(a.have_clinvar or []), a.ibd_panel == "yes")) - print("Family: %d people in the pedigree, %d sequenced (%s)" % (len(members), len(seq), ", ".join(seq))) - for x, y in itertools.combinations(seq, 2): - e = 2 * phi(x, y) - print(" %s and %s: declared %s (expected relatedness %.3f)" - % (x, y, declared_relationship(x, y, members, e), e)) - for s in seq: - ps = [p for p in (members[s]["father"], members[s]["mother"]) if p in seq] - if ps: - print(" child %s: %s sequenced" % (s, " and ".join(ps))) - print("Trios WhatsHap phases: %s" % (", ".join(trios) or "none (no child has both parents sequenced)")) - - -# --- relationship check ------------------------------------------------------- - -def read_pairs(path): - pairs = {} - with open(path) as f: - head = f.readline().rstrip("\n").lstrip("#").split("\t") - for line in f: - r = dict(zip(head, line.rstrip("\n").split("\t"))) - pairs[frozenset((r["sample_a"], r["sample_b"]))] = r - return pairs - - -def verdict(declared, expected, rel, ibs0_rate, n): - """(verdict, reason) for one pair.""" - if n < MIN_SITES: - return "not_checked", ("somalier compared the two at %d sites, fewer than %d: too few to tell " - "(a sliced or targeted sample, or the wrong sites file)" % (n, MIN_SITES)) - want, got = degree(expected), degree(rel) - if expected >= 0.177: - if got != want: - return "mismatch", "declared %s (%s), but somalier measures %s" % (declared, want, got) - if declared == "parent-child" and ibs0_rate > PARENT_CHILD_MAX_IBS0: - return "mismatch", ("first degree, but opposite homozygotes at %.1f%% of sites: a parent and " - "child share an allele at every site (siblings do not)" % (100 * ibs0_rate)) - return "ok", "%s, as declared" % got - if rel >= 0.354: - return "mismatch", "declared %s, but somalier measures %s" % (declared, got) - if got != want and rel > expected: - return "warn", "declared %s, somalier measures %s: closer than the pedigree says" % (declared, got) - return "ok", "%s, as declared" % got - - -def cmd_check(a): - members = read_members(a.members) - pairs = read_pairs(a.pairs) - phi = kinship_table(members) - seq = sorted(s for s, m in members.items() if m["sequenced"]) - out, bad, errors = [], [], [] - for x, y in itertools.combinations(seq, 2): - r = pairs.get(frozenset((x, y))) - if r is None: - errors.append("somalier relate has no row for %s and %s; its samples are: %s" - % (x, y, ", ".join(sorted({s for p in pairs for s in p})))) - continue - e = 2 * phi(x, y) - d = declared_relationship(x, y, members, e) - rel, ibs0, ibs2, n = float(r["relatedness"]), int(r["ibs0"]), int(r["ibs2"]), int(r["n"]) - rate = ibs0 / n if n else 0.0 - v, why = verdict(d, e, rel, rate, n) - out.append([x, y, d, "%.3f" % e, "%.3f" % rel, str(ibs0), "%.4f" % rate, str(ibs2), str(n), v, why]) - if v == "mismatch": - bad.append("%s and %s: %s" % (x, y, why)) - elif v != "ok": - print("WARNING: %s and %s: %s" % (x, y, why), file=sys.stderr) - if errors: - die(errors) - with open(a.out, "w") as f: - f.write("sample_a\tsample_b\tdeclared\texpected_relatedness\trelatedness\tibs0\tibs0_rate\tibs2\tsites" - "\tverdict\treason\n") - for row in out: - f.write("\t".join(row) + "\n") - for row in out: - print("%s and %s: %s; relatedness %s (expected %s), %s sites: %s" - % (row[0], row[1], row[2], row[4], row[3], row[8], row[9])) - if bad: - die(["the data disagree with the pedigree:"] + [" " + b for b in bad] + - ["A sample swap (at the lab or in the samplesheet), or a wrong line in the pedigree. " - "Check the pedigree and the samplesheet; somalier's own tables are beside relationships.tsv."], - code=1) - - -# --- report ------------------------------------------------------------------- - def read_par(path): par = {} if path: @@ -704,406 +228,3 @@ def read_par(path): def in_par(par, chrom, pos): # BED: 0-based start, end exclusive; pos is 1-based return any(s < pos <= e for s, e in par.get(chrom, ())) - - -def parse_gt(s): - """(alleles, phased) of a GT string; alleles is None when any is missing.""" - if not s or s == ".": - return None, False - phased = "|" in s - parts = re.split(r"[/|]", s) - if any(p == "." or p == "" for p in parts): - return None, phased - return tuple(int(p) for p in parts), phased - - -def fmt_field(fmt, vals, key): - try: - v = vals[fmt.index(key)] - except (ValueError, IndexError): - return None - return None if v in (".", "") else v - - -def to_int(v): - try: - return int(v) - except (TypeError, ValueError): - return None - - -def candidate_genes(path): - """{(chrom, pos, ref, alt): genes} of a candidate VCF: VEP's CSQ SYMBOL, - else snpEff's ANN gene, else INFO/GENE.""" - out, symbol, ann = {}, None, False - with open_text(path) as f: - for line in f: - if line.startswith("##INFO= symbol and f2[symbol]: - genes.add(f2[symbol]) - elif ann and "ANN" in info: - for entry in info["ANN"].split(","): - f2 = entry.split("|") - if len(f2) > 3 and f2[3]: - genes.add(f2[3]) - elif info.get("GENE"): - genes.update(g for g in info["GENE"].split(",") if g) - if genes: - for alt in c[4].split(","): - out[(c[0], int(c[1]), c[3], alt)] = genes - return out - - -def resolve(child, f_alleles, m_alleles): - """The (from_father, from_mother) allele assignments a diploid child - genotype allows; a parent not sequenced (None) allows any allele.""" - a, b = child - opts = set() - for x, y in ((a, b), (b, a)): - if (f_alleles is None or x in f_alleles) and (m_alleles is None or y in m_alleles): - opts.add((x, y)) - return opts - - -class Child: - def __init__(self, name, m, members, out_dir): - self.name = name - self.sex = m["sex"] - self.father, self.mother = m["father"], m["mother"] - self.f_seq = bool(self.father and members[self.father]["sequenced"]) - self.m_seq = bool(self.mother and members[self.mother]["sequenced"]) - self.path = os.path.join(out_dir, "%s_inheritance.tsv.gz" % name) - self.fh = GzOut(self.path) - self.fh.write(b"chrom\tpos\tref\talt\tchild_gt\tfather_gt\tmother_gt\tfrom_father\tfrom_mother\thow\n") - self.n = {"sites": 0, "father": 0, "mother": 0, "both": 0, "ambiguous": 0, "inconsistent": 0, - "skipped_sex": 0} - self.de_novo = [] - self.cands = [] - # WhatsHap: per (chrom, PS) block, sites where the first haplotype - # holds the father's allele, and sites where it holds the mother's - self.blocks = {} - self.phased_hets = 0 - - -def cmd_report(a): - members = read_members(a.members) - rels = [] - with open(a.relationships) as f: - head = f.readline().rstrip("\n").split("\t") - rels = [dict(zip(head, l.rstrip("\n").split("\t"))) for l in f] - par = read_par(a.par_bed) - os.makedirs(a.out_dir, exist_ok=True) - seq = sorted(s for s, m in members.items() if m["sequenced"]) - children = [Child(s, members[s], members, a.out_dir) for s in seq - if any(p and members[p]["sequenced"] for p in (members[s]["father"], members[s]["mother"]))] - cands = {} - for kv in a.candidates or []: - k, _, v = kv.partition("=") - if k in members: - cands[k] = candidate_genes(v) - - vcf = a.phased or a.vcf - samples, idx = None, {} - with open_text(vcf) as f: - for line in f: - if line.startswith("##"): - continue - if line.startswith("#CHROM"): - samples = line.rstrip("\n").split("\t")[9:] - idx = {s: 9 + i for i, s in enumerate(samples)} - lost = [s for s in seq if s not in idx] - if lost: - die(["%s has no column for %s (its columns: %s)" % (vcf, ", ".join(lost), ", ".join(samples))]) - continue - c = line.rstrip("\n").split("\t") - chrom, pos = c[0], int(c[1]) - if chrom not in PRIMARY: - continue - alleles = [c[3]] + c[4].split(",") - fmt = c[8].split(":") - gi = fmt.index("GT") if "GT" in fmt else None - if gi is None: - continue - calls = {} - - def call(s): - if s not in calls: - vals = c[idx[s]].split(":") - calls[s] = (parse_gt(vals[gi] if gi < len(vals) else "."), vals) - return calls[s] - - sexchr = chrom in ("chrX", "chrY") and not in_par(par, chrom, pos) - for ch in children: - (cg, cphased), cvals = call(ch.name) - if cg is None: - continue - fg = call(ch.father)[0][0] if ch.f_seq else None - mg = call(ch.mother)[0][0] if ch.m_seq else None - fset = set(fg) if fg is not None else None - mset = set(mg) if mg is not None else None - # a sequenced parent with no call here tells nothing - f_known, m_known = fset is not None, mset is not None - if not f_known and not m_known: - continue - how, frm_f, frm_m = None, None, None - if sexchr: - if ch.sex not in ("male", "female") or (chrom == "chrY" and ch.sex == "female"): - ch.n["skipped_sex"] += 1 - continue - if ch.sex == "male": - if len(set(cg)) != 1: - ch.n["skipped_sex"] += 1 # a heterozygous call on a single X or Y - continue - allele = cg[0] - src = mset if chrom == "chrX" else fset - if (chrom == "chrX" and not m_known) or (chrom == "chrY" and not f_known): - continue - if allele in src: - how = "genotype" - if chrom == "chrX": - frm_m = allele - else: - frm_f = allele - else: - how = "inconsistent" - if how is None: - if len(cg) != 2: - continue - opts = resolve(cg, fset, mset) - if not opts: - how = "inconsistent" - elif len(opts) == 1: - how = "genotype" - frm_f, frm_m = next(iter(opts)) - else: - how = "ambiguous" - ch.n["sites"] += 1 - if how == "genotype": - if frm_f is not None and frm_m is not None: - ch.n["both"] += 1 - if frm_f is not None: - ch.n["father"] += 1 - if frm_m is not None: - ch.n["mother"] += 1 - else: - ch.n[how] += 1 - gt = lambda s, known: (call(s)[1][gi] if gi < len(call(s)[1]) else ".") if known else "." - ch.fh.write(("\t".join([chrom, c[1], c[3], c[4], cvals[gi], gt(ch.father, ch.f_seq), - gt(ch.mother, ch.m_seq), - alleles[frm_f] if frm_f is not None else ".", - alleles[frm_m] if frm_m is not None else ".", how]) + "\n").encode()) - het = len(set(cg)) == 2 and 0 in cg - # WhatsHap's phase of the child against the genotype origin - if a.phased and cphased and het: - ch.phased_hets += 1 - ps = fmt_field(fmt, cvals, "PS") or "." - if how == "genotype" and frm_f is not None: - b = ch.blocks.setdefault((chrom, ps), [0, 0]) - b[0 if cg[0] == frm_f else 1] += 1 - # de novo: an allele the child carries that neither parent has - if how == "inconsistent" and f_known and m_known: - # on a male's single X only the mother's alleles count, on his Y the father's; - # a female's two X come one from each parent, so both parents' alleles count - if sexchr and chrom == "chrX": - seen = mset if ch.sex == "male" else fset | mset - else: - seen = fset if sexchr else fset | mset - novel = sorted(set(cg) - seen - {0}) - if novel: - ch.de_novo.append(de_novo_row(ch, c, fmt, cvals, call, gi, alleles, novel, sexchr)) - # compound heterozygote candidates - if het and ch.name in cands: - alt = alleles[cg[1] if cg[0] == 0 else cg[0]] - genes = cands[ch.name].get((chrom, pos, c[3], alt)) - if genes: - ai = alleles.index(alt) - origin = "father" if frm_f == ai else "mother" if frm_m == ai else None - ch.cands.append((genes, "%s:%d:%s>%s" % (chrom, pos, c[3], alt), origin)) - if samples is None: - die(["%s has no #CHROM line" % vcf]) - - with open(os.path.join(a.out_dir, "de_novo.tsv"), "w") as f: - f.write("child\tchrom\tpos\tref\talt\tchild_gt\tfather_gt\tmother_gt\tchild_dp\tchild_gq\tchild_ab" - "\tfather_dp\tfather_gq\tmother_dp\tmother_gq\tquality\n") - for ch in children: - for row in ch.de_novo: - f.write("\t".join(row) + "\n") - comp = [] - for ch in children: - by_gene = {} - for genes, var, origin in ch.cands: - for g in genes: - by_gene.setdefault(g, []).append((var, origin)) - for g in sorted(by_gene): - for (v1, o1), (v2, o2) in itertools.combinations(by_gene[g], 2): - phase = "unknown" if not (o1 and o2) else ("in cis" if o1 == o2 else "in trans") - comp.append([ch.name, g, v1, v2, o1 or ".", o2 or ".", phase]) - with open(os.path.join(a.out_dir, "compound_het.tsv"), "w") as f: - f.write("child\tgene\tvariant_a\tvariant_b\ta_from\tb_from\tphase\n") - for row in comp: - f.write("\t".join(row) + "\n") - for ch in children: - ch.fh.close() - # The chromosome map: which stretches each child got from each parent - # (bin/family_segments.py), from the joint VCF's genotypes - import family_segments - seg = family_segments.run(members, a.vcf, par, a.out_dir) - write_markdown(a, members, rels, children, comp, cands, seg) - - -def de_novo_row(ch, c, fmt, cvals, call, gi, alleles, novel, sexchr): - fv, mv = call(ch.father)[1], call(ch.mother)[1] - dp = [to_int(fmt_field(fmt, v, "DP")) for v in (cvals, fv, mv)] - gq = [to_int(fmt_field(fmt, v, "GQ")) for v in (cvals, fv, mv)] - ab = None - ad = fmt_field(fmt, cvals, "AD") - if ad: - n = [to_int(x) or 0 for x in ad.split(",")] - tot = sum(n) - if tot: - ab = n[novel[0]] / tot if novel[0] < len(n) else None - het = len(set(parse_gt(cvals[gi])[0])) == 2 - good = all(x is not None and x >= MIN_GQ for x in gq) and all(x is not None and x >= MIN_DP for x in dp) - if het and (ab is None or not HET_AB[0] <= ab <= HET_AB[1]): - good = False - s = lambda x: "." if x is None else str(x) - return [ch.name, c[0], c[1], c[3], ",".join(alleles[i] for i in novel), cvals[gi], - fv[gi] if gi < len(fv) else ".", mv[gi] if gi < len(mv) else ".", - s(dp[0]), s(gq[0]), "." if ab is None else "%.2f" % ab, s(dp[1]), s(gq[1]), s(dp[2]), s(gq[2]), - "pass" if good else "low"] - - -def pct(k, n): - return "%.1f%%" % (100.0 * k / n) if n else "-" - - -def write_markdown(a, members, rels, children, comp, cands, seg=()): - L = ["# Family analysis", ""] - seq = sorted(s for s, m in members.items() if m["sequenced"]) - others = sorted(s for s, m in members.items() if not m["sequenced"]) - L.append("Sequenced: %s. In the pedigree, not sequenced: %s." % (", ".join(seq), ", ".join(others) or "none")) - L += ["", "## Relationships", "", - "Relatedness is twice the kinship coefficient: about the share of the genome the two have " - "inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, " - "0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the " - "expected value.", "", - "| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict |", - "|---|---|---|---|---|---|---|"] - for r in rels: - L.append("| %s and %s | %s | %s | %s | %s | %s | %s: %s |" - % (r["sample_a"], r["sample_b"], r["declared"], r["expected_relatedness"], r["relatedness"], - pct(float(r["ibs0_rate"]), 1), r["sites"], r["verdict"], r["reason"])) - for ch in children: - n = ch.n - L += ["", "## %s" % ch.name, ""] - fa = "%s (%s)" % (ch.father, "sequenced" if ch.f_seq else "not sequenced") if ch.father else "not named" - mo = "%s (%s)" % (ch.mother, "sequenced" if ch.m_seq else "not sequenced") if ch.mother else "not named" - L.append("Father: %s. Mother: %s." % (fa, mo)) - L += ["", "Sites of the joint VCF where %s has a genotype and a sequenced parent does too: %d. " - "`%s` lists them." % (ch.name, n["sites"], os.path.basename(ch.path)), ""] - L.append("- the allele from the father is known at %d (%s)" % (n["father"], pct(n["father"], n["sites"]))) - L.append("- the allele from the mother is known at %d (%s)" % (n["mother"], pct(n["mother"], n["sites"]))) - L.append("- both parents heterozygous like the child, so no origin from the genotypes: %d (%s)" - % (n["ambiguous"], pct(n["ambiguous"], n["sites"]))) - L.append("- inconsistent with the declared parents (a genotype error, a deletion, or a new " - "mutation): %d (%s)" % (n["inconsistent"], pct(n["inconsistent"], n["sites"]))) - if n["skipped_sex"]: - L.append("- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a " - "heterozygous call on a single X or Y): %d" % n["skipped_sex"]) - for who, seq_, k in (("father", ch.f_seq, "father"), ("mother", ch.m_seq, "mother")): - if not seq_: - L += ["", "The %s is not sequenced. At %d of these sites (%s) the genotypes give the allele " - "the %s passed on to %s: that is the half of the %s's genome %s carries. The other " - "half was not passed on and is not seen here." - % (who, n[k], pct(n[k], n["sites"]), who, ch.name, who, ch.name)] - if ch.f_seq and ch.m_seq: - good = sum(1 for r in ch.de_novo if r[-1] == "pass") - L += ["", "De novo candidates (an allele neither parent has): %d, %d of them with GQ >= %d and " - "depth >= %d in all three and, for a heterozygous call, an allele balance of %.2f to %.2f " - "(`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that " - "matters needs confirming by another method." - % (len(ch.de_novo), good, MIN_GQ, MIN_DP, HET_AB[0], HET_AB[1])] - else: - L += ["", "No de novo candidates are called: that needs both parents sequenced."] - if a.phased: - votes = sum(max(b) for b in ch.blocks.values()) - tot = sum(sum(b) for b in ch.blocks.values()) - L += ["", "WhatsHap phased %d heterozygous sites of %s by inheritance. At %d of them the " - "genotypes also give the origin, and the phase agrees with it at %d (%s)." - % (ch.phased_hets, ch.name, tot, votes, pct(votes, tot))] - L += ["", "## Compound heterozygotes", ""] - if not cands: - L.append("No candidate variants were given (the clinical filter, step 23, did not run for a child), " - "so no pair is reported.") - else: - t = sum(1 for r in comp if r[-1] == "in trans") - c_ = sum(1 for r in comp if r[-1] == "in cis") - L.append("Pairs of the clinical filter's candidate variants a child carries heterozygous in one gene: " - "%d; in trans (one from each parent, both copies of the gene hit) %d, in cis (both from one " - "parent) %d, origin unknown %d (`compound_het.tsv`)." % (len(comp), t, c_, len(comp) - t - c_)) - L += list(seg) - with open(os.path.join(a.out_dir, "family_report.md"), "w") as f: - f.write("\n".join(L) + "\n") - print("\n".join(L)) - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - sub = p.add_subparsers(dest="cmd", required=True) - q = sub.add_parser("prepare") - q.add_argument("--pedigree", required=True) - q.add_argument("--gvcf", action="append", metavar="ID=PATH") - q.add_argument("--declared-sex", action="append", metavar="ID=SEX") - q.add_argument("--family", help="the family of a families file to read (needed when it holds more than one)") - q.add_argument("--have-clinvar", nargs="*", metavar="ID", help="the members with step 06's ClinVar hits") - q.add_argument("--ibd-panel", choices=("yes", "no"), default="no") - q.add_argument("--pedigree-out", help="also write the family's rows here, as a pedigree") - q.add_argument("--out-dir", required=True) - q = sub.add_parser("plan") - q.add_argument("--families", required=True) - q.add_argument("--family") - q.add_argument("--sample") - q.add_argument("--sex") - q.add_argument("--have-gvcf", nargs="*", metavar="ID") - q.add_argument("--have-clinvar", nargs="*", metavar="ID") - q.add_argument("--ibd-panel", choices=("yes", "no"), default="no") - q.add_argument("--write-pedigree", metavar="DIR", help="write DIR//pedigree.tsv") - q.add_argument("--which", action="store_true", help="print the family of --sample (nothing when none) and stop") - q.add_argument("--out", help="family_plan.tsv") - q = sub.add_parser("check") - q.add_argument("--members", required=True) - q.add_argument("--pairs", required=True) - q.add_argument("--out", required=True) - q = sub.add_parser("report") - q.add_argument("--members", required=True) - q.add_argument("--relationships", required=True) - q.add_argument("--vcf", required=True) - q.add_argument("--phased") - q.add_argument("--candidates", action="append", metavar="ID=VCF") - q.add_argument("--par-bed") - q.add_argument("--out-dir", required=True) - a = p.parse_args(argv) - {"prepare": cmd_prepare, "plan": cmd_plan, "check": cmd_check, "report": cmd_report}[a.cmd](a) - - -if __name__ == "__main__": - main() diff --git a/bin/family_segments.py b/bin/family_segments.py deleted file mode 100755 index 0d5430bd2..000000000 --- a/bin/family_segments.py +++ /dev/null @@ -1,453 +0,0 @@ -#!/usr/bin/env python3 -"""family_segments.py: the chromosome map of step 38 (scripts/38-family.sh) -and of the Nextflow FAMILY workflow. family_report.py report calls it, so -both write the same files; it also runs on its own over a family folder. -Standard library only. - - family_segments.py --members D/members.tsv --vcf FAMILY.vcf.gz [--par-bed par_grch38.bed] --out-dir D - -For each sequenced child with a sequenced parent, the stretches of each -chromosome the child got from that parent. A site tells something when the -parent is heterozygous and the genotypes give the allele the parent passed -on. When one of that parent's own parents is sequenced too, the site also -tells which of the parent's two copies it was (the one from the -grandfather or the one from the grandmother), and a change from one to the -other along the chromosome is a crossover. With only one parent sequenced, -the child's other copy is mapped over the same stretches as coming from the -other side. - -Stretches are smoothed: a run of sites that points to the other copy is -kept only with MIN_SITES sites over MIN_BP bases; a shorter run (a genotype -error, a gene conversion) is absorbed and counted as discordant. A stretch -with no site for MAX_GAP bases (a centromere, a gap of the reference) is -split there, since nothing is known inside it. Autosomes, and chrX outside -the pseudoautosomal regions from the mother only: a father passes his one X -whole to a daughter and none to a son. Without --par-bed chrX is left out. - -Writes, per child: - _segments.tsv child, side, parent, chrom, start, end, length_bp, - sites, discordant, ancestor, basis - _dnapainter.csv the same segments in DNA Painter's own import - format (Chr,Start,End,cM,SNPs,Match,Confidence, - Group,Side,Notes,Colour,Ancestor); GRCh38 positions - _segments.svg a map of every chromosome, the father's copy above - the mother's -""" -import argparse -import csv -import io -import os -import re -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import family_report as fr # noqa: E402 the pedigree and genotype rules of step 38 - -# A run of sites pointing to the other grandparent's copy needs this much -# support to be a segment of its own (a crossover on each side of it). -MIN_SITES = 20 -MIN_BP = 500_000 -# No site for this many bases: the stretch is split, nothing is claimed inside. -MAX_GAP = 5_000_000 -AUTOSOMES = ["chr%d" % i for i in range(1, 23)] -CHROMS = AUTOSOMES + ["chrX"] -COLOURS = {"paternal": ("#4f8fd6", "#9cc3ef"), "maternal": ("#d65f8f", "#efa3c0")} -SIDE_ONLY = {"paternal": "#3b6fa8", "maternal": "#a84870"} -OTHER = "#6b6b6b" - - -def unique(vals): - vals = set(vals) - return next(iter(vals)) if len(vals) == 1 else None - - -class Link: - """One child and one sequenced parent: the parent's informative sites.""" - - def __init__(self, child, side, parent, members): - self.child, self.side, self.parent = child, side, parent - p = members[parent] - self.gf = p["father"] - self.gm = p["mother"] - self.gf_seq = bool(self.gf and members[self.gf]["sequenced"]) - self.gm_seq = bool(self.gm and members[self.gm]["sequenced"]) - self.sites = {} # chrom -> [pos] where the allele passed on is known - self.hap = {} # chrom -> [(pos, 0 from the grandfather | 1 from the grandmother)] - - @property - def grandparents(self): - return self.gf_seq or self.gm_seq - - def ancestor(self, label): - name = self.gf if label == 0 else self.gm - return name or "%s's %s" % (self.parent, "father" if label == 0 else "mother") - - -def gset(g): - return set(g) if g is not None else None - - -def passed_on(child_gt, parent_gt, other_gt, parent_is_father): - """The allele the parent passed on, when the genotypes give one.""" - f, m = (parent_gt, other_gt) if parent_is_father else (other_gt, parent_gt) - opts = fr.resolve(child_gt, gset(f), gset(m)) - return unique(o[0] if parent_is_father else o[1] for o in opts) if opts else None - - -def collect(members, vcf, par): - """The Link of every sequenced child and sequenced parent, filled from the joint VCF.""" - seq = sorted(s for s, m in members.items() if m["sequenced"]) - links = {} - for s in seq: - m = members[s] - for side, p in (("paternal", m["father"]), ("maternal", m["mother"])): - if p and members[p]["sequenced"]: - links.setdefault(s, []).append(Link(s, side, p, members)) - if not links: - return members, links - idx = None - with fr.open_text(vcf) as f: - for line in f: - if line.startswith("##"): - continue - if line.startswith("#CHROM"): - cols = line.rstrip("\n").split("\t")[9:] - idx = {s: 9 + i for i, s in enumerate(cols)} - lost = [s for s in seq if s not in idx] - if lost: - fr.die(["%s has no column for %s" % (vcf, ", ".join(lost))]) - continue - c = line.rstrip("\n").split("\t") - chrom = c[0] - if chrom not in CHROMS: - continue - pos = int(c[1]) - # without a PAR file the pseudoautosomal sites cannot be told apart, so chrX is left out - if chrom == "chrX" and (not par or fr.in_par(par, chrom, pos)): - continue - fmt = c[8].split(":") - if "GT" not in fmt: - continue - gi = fmt.index("GT") - cache = {} - - def g(s): - if s is None or not members.get(s, {}).get("sequenced"): - return None - if s not in cache: - v = c[idx[s]].split(":") - cache[s] = fr.parse_gt(v[gi] if gi < len(v) else ".")[0] - return cache[s] - - for child, ls in links.items(): - cg = g(child) - if cg is None: - continue - for ln in ls: - pg = g(ln.parent) - if pg is None or len(pg) != 2 or pg[0] == pg[1]: - continue # only a heterozygous parent tells which copy was passed on - if chrom == "chrX": - if ln.side != "maternal": - continue - if members[child]["sex"] == "male": - if len(set(cg)) != 1: - continue - t = cg[0] - else: - fg = g(members[child]["father"]) - if fg is None or len(set(fg)) != 1 or len(cg) != 2: - continue - t = passed_on(cg, pg, (fg[0], fg[0]), False) - else: - if len(cg) != 2: - continue - other = members[child]["mother" if ln.side == "paternal" else "father"] - t = passed_on(cg, pg, g(other), ln.side == "paternal") - if t is None or t not in pg: - continue - ln.sites.setdefault(chrom, []).append(pos) - if not ln.grandparents: - continue - # which of the parent's copies holds t: the parent's own origin here - if chrom == "chrX": - # the mother's X: one from her father (his only X), one from her mother - gfg, gmg = g(ln.gf), g(ln.gm) - if gfg is not None and len(set(gfg)) != 1: - continue - po = passed_on(pg, gmg, (gfg[0], gfg[0]) if gfg else None, False) - if po is None: - continue - label = 1 if t == po else 0 - else: - opts = fr.resolve(pg, gset(g(ln.gf)), gset(g(ln.gm))) - from_gf = unique(o[0] for o in opts) if opts else None - if from_gf is None: - continue - label = 0 if t == from_gf else 1 - ln.hap.setdefault(chrom, []).append((pos, label)) - return members, links - - -def blocks(positions): - """Split sorted positions where no site falls for MAX_GAP bases.""" - out, cur = [], [] - for p in positions: - if cur and p - cur[-1] > MAX_GAP: - out.append(cur) - cur = [] - cur.append(p) - if cur: - out.append(cur) - return out - - -def smooth(sites): - """Runs [label, start, end, agree, discordant] of one block of (pos, label).""" - runs = [] - for pos, label in sites: - if runs and runs[-1][0] == label: - runs[-1][2] = pos - runs[-1][3] += 1 - else: - runs.append([label, pos, pos, 1, 0]) - - def weak(r): - return r[3] < MIN_SITES or r[2] - r[1] + 1 < MIN_BP - - while len(runs) > 1: - w = [i for i, r in enumerate(runs) if weak(r)] - if not w: - break - i = min(w, key=lambda k: (runs[k][3], runs[k][2] - runs[k][1], k)) - lo, hi = max(i - 1, 0), min(i + 1, len(runs) - 1) - keep = runs[lo] if lo != i else runs[hi] - merged = [keep[0], runs[lo][1], runs[hi][2], 0, 0] - for r in runs[lo:hi + 1]: - if r is runs[i]: - # its sites now point to the other copy; the ones it had absorbed agree again - merged[3] += r[4] - merged[4] += r[3] - else: - merged[3] += r[3] - merged[4] += r[4] - runs[lo:hi + 1] = [merged] - return [r for r in runs if r[3] >= MIN_SITES] - - -def segments(members, links): - """{child: [row dict]}, rows in chromosome order, father's side first.""" - order = {c: i for i, c in enumerate(CHROMS)} - out = {} - for child, ls in sorted(links.items()): - rows = [] - for ln in ls: - hap = ln.grandparents and sum(len(v) for v in ln.hap.values()) >= MIN_SITES - for chrom in sorted(ln.sites, key=order.get): - if hap and chrom in ln.hap: - sites = ln.hap[chrom] - for blk in blocks([p for p, _ in sites]): - lo, hi = blk[0], blk[-1] - part = [s for s in sites if lo <= s[0] <= hi] - for label, s, e, n, disc in smooth(part): - rows.append(row(child, ln.side, ln.parent, chrom, s, e, n, disc, - ln.ancestor(label), "parent and grandparents")) - else: - for blk in blocks(ln.sites[chrom]): - if len(blk) >= MIN_SITES: - rows.append(row(child, ln.side, ln.parent, chrom, blk[0], blk[-1], len(blk), 0, - ".", "parent")) - # One parent sequenced: the other copy over the same stretches - if len(ls) == 1: - ln = ls[0] - other_side = "maternal" if ln.side == "paternal" else "paternal" - other = members[child]["mother" if ln.side == "paternal" else "father"] - for chrom in sorted(ln.sites, key=order.get): - if chrom == "chrX": - continue # the X a child got from the father is not seen from the mother's sites - for blk in blocks(ln.sites[chrom]): - if len(blk) >= MIN_SITES: - rows.append(row(child, other_side, other or ".", chrom, blk[0], blk[-1], len(blk), 0, - ".", "other copy")) - rows.sort(key=lambda r: (order[r["chrom"]], r["side"] != "paternal", r["start"])) - out[child] = rows - return out - - -def row(child, side, parent, chrom, start, end, n, disc, ancestor, basis): - return {"child": child, "side": side, "parent": parent, "chrom": chrom, "start": start, "end": end, - "length_bp": end - start + 1, "sites": n, "discordant": disc, "ancestor": ancestor, "basis": basis} - - -TSV = ("child", "side", "parent", "chrom", "start", "end", "length_bp", "sites", "discordant", "ancestor", "basis") -DNAPAINTER = ("Chr", "Start", "End", "cM", "SNPs", "Match", "Confidence", "Group", "Side", "Notes", "Colour", - "Ancestor") - - -def contig_lengths(vcf): - out = {} - with fr.open_text(vcf) as f: - for line in f: - if not line.startswith("##"): - break - m = re.match(r"##contig=]+),.*?length=(\d+)", line) - if m: - out[m.group(1)] = int(m.group(2)) - return out - - -def crossovers(rows, side): - """Changes of ancestor between neighbouring segments of one chromosome.""" - n, prev = 0, None - for r in rows: - if r["side"] != side or r["basis"] != "parent and grandparents": - continue - if prev and prev["chrom"] == r["chrom"] and prev["ancestor"] != r["ancestor"]: - n += 1 - prev = r - return n - - -def write_files(rows_by_child, lengths, out_dir): - os.makedirs(out_dir, exist_ok=True) - for child, rows in rows_by_child.items(): - with open(os.path.join(out_dir, "%s_segments.tsv" % child), "w") as f: - f.write("\t".join(TSV) + "\n") - for r in rows: - f.write("\t".join(str(r[k]) for k in TSV) + "\n") - buf = io.StringIO() - w = csv.writer(buf, lineterminator="\n") - w.writerow(DNAPAINTER) - for r in rows: - who = r["ancestor"] if r["ancestor"] != "." else r["parent"] if r["parent"] != "." else "%s side" % r["side"] - note = "GRCh38 positions; %s; %d discordant sites" % ( - {"parent and grandparents": "the parent's copy, told by a sequenced grandparent", - "parent": "from the sequenced parent", - "other copy": "the copy not from the sequenced parent"}[r["basis"]], r["discordant"]) - # DNA Painter reads Side and Ancestor from the first row of each Group, and Ancestor - # takes only 1 (a known ancestor), so each grandparent (or parent) is a group of its own. - known = r["ancestor"] != "." or r["parent"] != "." - w.writerow([r["chrom"][3:], r["start"], r["end"], "", r["sites"], "%s from %s" % (child, who), "", - who, r["side"], note, "", "1" if known else ""]) - with open(os.path.join(out_dir, "%s_dnapainter.csv" % child), "w") as f: - f.write(buf.getvalue()) - with open(os.path.join(out_dir, "%s_segments.svg" % child), "w") as f: - f.write(svg(child, rows, lengths)) - - -def esc(s): - return str(s).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) - - -def svg(child, rows, lengths): - chroms = [c for c in CHROMS if c in lengths or any(r["chrom"] == c for r in rows)] - longest = max([lengths.get(c, 0) for c in chroms] + [r["end"] for r in rows] + [1]) - width, left, bar, row_h = 900, 60, 9, 28 - scale = (width - left - 20) / longest - colour = {} - for r in rows: - if r["basis"] == "other copy": - colour[(r["side"], r["ancestor"])] = OTHER - elif r["ancestor"] == ".": - colour[(r["side"], ".")] = SIDE_ONLY[r["side"]] - else: - seen = [k for k in colour if k[0] == r["side"] and k[1] != "."] - if (r["side"], r["ancestor"]) not in colour: - colour[(r["side"], r["ancestor"])] = COLOURS[r["side"]][len(seen) % 2] - height = 50 + row_h * len(chroms) + 20 * (len(colour) + 1) - L = ['' - % (width, height), - '', - '%s: which copy of each chromosome came from whom ' - '(GRCh38)' % esc(child)] - for i, c in enumerate(chroms): - y = 40 + i * row_h - L.append('%s' % (y + bar + 3, esc(c[3:]))) - n = lengths.get(c, max([r["end"] for r in rows if r["chrom"] == c] + [1])) - for k, side in enumerate(("paternal", "maternal")): - L.append('' - % (left, y + k * (bar + 1), n * scale, bar)) - for r in rows: - if r["chrom"] != c: - continue - k = 0 if r["side"] == "paternal" else 1 - L.append('%s %s:%d-%d, ' - '%s, %d sites' - % (left + r["start"] * scale, y + k * (bar + 1), max((r["end"] - r["start"]) * scale, 1), bar, - colour[(r["side"], r["ancestor"])], - esc(r["side"]), esc(r["chrom"]), r["start"], r["end"], - esc(r["ancestor"] if r["ancestor"] != "." else r["parent"]), r["sites"])) - y = 50 + row_h * len(chroms) - L.append('Upper bar: the copy from the father. Lower bar: from the mother. ' - 'Grey: not mapped.' % y) - for j, ((side, anc), col) in enumerate(sorted(colour.items())): - label = {OTHER: "%s side, the copy not from the sequenced parent" % side}.get(col) - if label is None: - label = "%s side, %s" % (side, "from the parent" if anc == "." else "from " + anc) - L.append('%s' - % (y + 10 + 20 * j, col, y + 19 + 20 * j, esc(label))) - L.append("") - return "\n".join(L) + "\n" - - -def markdown(members, rows_by_child): - """The "Chromosome map" section of family_report.md.""" - L = ["", "## Chromosome map", ""] - if not rows_by_child: - L.append("No sequenced child has a sequenced parent, so there is no map.") - return L - L += ["For each child, the stretches of each chromosome that came from each parent, from the sites where " - "that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent " - "is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or " - "the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, " - "`_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), " - "`_segments.svg` draws it.", ""] - for child, rows in rows_by_child.items(): - m = members[child] - for side, who in (("paternal", "father"), ("maternal", "mother")): - mine = [r for r in rows if r["side"] == side] - if not mine: - continue - bp = sum(r["length_bp"] for r in mine) - name = m[who] or "not named" - bases = {r["basis"] for r in mine} - if "other copy" in bases: - L.append("- %s: from the %s (%s, not sequenced): %d stretches over %.1f Mb, the copy that did not " - "come from the sequenced parent" % (child, who, name, len(mine), bp / 1e6)) - elif "parent and grandparents" in bases: - x = crossovers(rows, side) - anc = sorted({r["ancestor"] for r in mine}) - L.append("- %s: from the %s (%s): %d stretches over %.1f Mb, told apart as the copy from %s; " - "%d crossover%s, %d discordant sites absorbed" - % (child, who, name, len(mine), bp / 1e6, " or ".join(anc), x, "" if x == 1 else "s", - sum(r["discordant"] for r in mine))) - else: - L.append("- %s: from the %s (%s): %d stretches over %.1f Mb; no grandparent on this side is " - "sequenced, so the two copies of the %s are not told apart" - % (child, who, name, len(mine), bp / 1e6, who)) - L += ["", "A stretch starts and ends at its first and last informative site: a crossover lies somewhere " - "between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced " - "grandparent the map says only which parent a stretch came from."] - return L - - -def run(members, vcf, par, out_dir): - """Map every child, write the files, return the report section's lines.""" - members, links = collect(members, vcf, par) - rows = segments(members, links) - write_files(rows, contig_lengths(vcf), out_dir) - return markdown(members, rows) - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - p.add_argument("--members", required=True) - p.add_argument("--vcf", required=True) - p.add_argument("--par-bed") - p.add_argument("--out-dir", required=True) - a = p.parse_args(argv) - print("\n".join(run(fr.read_members(a.members), a.vcf, fr.read_par(a.par_bed), a.out_dir))) - - -if __name__ == "__main__": - main() diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index b65bd33a0..8dd5d8f08 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -269,7 +269,7 @@ def file_state(d, f): def trio_status(d, order, members, files): """Per child, whether step 38 assessed it for de novo candidates and compound-het pairs, and why not: {child: {"de_novo": {"assessed": bool, "why": text}, "compound_het": {...}}}. The rule is step 38's - (bin/family_report.py): de novo candidates need both parents sequenced, compound-het pairs a sequenced + (og family report): de novo candidates need both parents sequenced, compound-het pairs a sequenced parent; a file that is not here, or is a link out of the folder, means no child was assessed.""" plan = {r.get("analysis"): r for r in read_tsv(os.path.join(d, "family_plan.tsv"))} trio = plan.get("trio_phasing") or {} diff --git a/docs/multi-sample.md b/docs/multi-sample.md index f9e3cef1a..7a446591c 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -47,7 +47,7 @@ GENOME_DIR=/path/to/data FAMILIES=families.tsv ./scripts/run-all.sh partner_x ma GENOME_DIR=/path/to/data FAMILIES=families.tsv ./scripts/run-all.sh partner_y female ``` -Each call processes its own sample only; the other members are never run as part of it. Before the pipeline starts, the sex given must be the one the file gives the sample (exit 2 otherwise). After the pipeline and before the reports, `bin/family_report.py plan` looks at what the family has under `GENOME_DIR` (each member's gVCF with its index, step 06's ClinVar hits, step 26b's panel) and writes `$GENOME_DIR//family_plan.tsv`, one row per analysis: `joint_call`, `trio_phasing`, `inheritance_map`, `shared_segments` (step 38) and `couple_carrier` per couple (step 39), each `runs`, `skipped` or `waiting`, with the reason in words. Step 38 runs when two or more members have a gVCF (a member still without one is left out and named); step 39 runs when both partners have ClinVar hits, with or without a gVCF (it never reads one). Otherwise the step is recorded in `logs/run_status.tsv` as `skipped (waiting: partner_y has no gVCF yet: run run-all.sh partner_y first)` for step 38, `skipped (waiting: partner_y has no ClinVar hits yet (step 06 writes clinvar/partner_y_clinvar_hits.vcf): run run-all.sh partner_y first)` for step 39, or `skipped ()`, so the first member's run waits and the last member's run does the family. A step that ran is recorded `ok` or `failed`, with its log in `logs/38-family.log` and `logs/39-couple.log`. Run step 38 again by hand after a later member is processed if it ran without them. +Each call processes its own sample only; the other members are never run as part of it. Before the pipeline starts, the sex given must be the one the file gives the sample (exit 2 otherwise). After the pipeline and before the reports, `og family plan` looks at what the family has under `GENOME_DIR` (each member's gVCF with its index, step 06's ClinVar hits, step 26b's panel) and writes `$GENOME_DIR//family_plan.tsv`, one row per analysis: `joint_call`, `trio_phasing`, `inheritance_map`, `shared_segments` (step 38) and `couple_carrier` per couple (step 39), each `runs`, `skipped` or `waiting`, with the reason in words. Step 38 runs when two or more members have a gVCF (a member still without one is left out and named); step 39 runs when both partners have ClinVar hits, with or without a gVCF (it never reads one). Otherwise the step is recorded in `logs/run_status.tsv` as `skipped (waiting: partner_y has no gVCF yet: run run-all.sh partner_y first)` for step 38, `skipped (waiting: partner_y has no ClinVar hits yet (step 06 writes clinvar/partner_y_clinvar_hits.vcf): run run-all.sh partner_y first)` for step 39, or `skipped ()`, so the first member's run waits and the last member's run does the family. A step that ran is recorded `ok` or `failed`, with its log in `logs/38-family.log` and `logs/39-couple.log`. Run step 38 again by hand after a later member is processed if it ran without them. By hand, from the scripts, once each member has a gVCF from step 03 (`/vcf/.g.vcf.gz`) and somalier's sites are installed (`setup.sh --sample-qc-data`): @@ -78,7 +78,7 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME When step 23 (the clinical filter) ran for a child, every pair of its candidate variants the child carries heterozygous in one gene is listed in `compound_het.tsv`: `in trans` when one came from each parent (both copies of the gene are hit), `in cis` when both came from one parent, `unknown` when the genotypes do not tell. -5. **The chromosome map.** For each child with a sequenced parent, `bin/family_segments.py` (run by the report) maps the stretches of each chromosome the child got from that parent. It reads the joint VCF's genotypes at the sites where the parent is heterozygous and the genotypes give the allele passed on. When one of that parent's own parents is sequenced too, each such site also says which of the parent's two copies was passed on, the grandfather's or the grandmother's, and a switch from one to the other along a chromosome is a crossover. Without a sequenced grandparent the stretches say only "from this parent". With one parent sequenced, the child's other copy is written over the same stretches as the copy that did not come from the sequenced parent. +5. **The chromosome map.** For each child with a sequenced parent, `og family report` maps the stretches of each chromosome the child got from that parent. The joint VCF must be sorted by position: an unsorted one stops the report with exit 2, naming the two records. It reads the joint VCF's genotypes at the sites where the parent is heterozygous and the genotypes give the allele passed on. When one of that parent's own parents is sequenced too, each such site also says which of the parent's two copies was passed on, the grandfather's or the grandmother's, and a switch from one to the other along a chromosome is a crossover. Without a sequenced grandparent the stretches say only "from this parent". With one parent sequenced, the child's other copy is written over the same stretches as the copy that did not come from the sequenced parent. Isolated sites that point to the other copy are genotype errors or short gene conversions: a run of them becomes a segment of its own only with 20 sites or more over 500 kb or more; a shorter run is absorbed and counted in the segment's `discordant` column. A stretch with no informative site for 5 Mb (a centromere, a gap of the reference) is split there, and a stretch with fewer than 20 informative sites is left out. A segment starts and ends at its first and last informative site, so a crossover lies somewhere in the space between two segments. Autosomes are mapped, and chrX outside the pseudoautosomal regions from the mother only (a father passes his single X whole to a daughter, and none to a son). A section of `family_report.md` counts each child's segments, Mb and crossovers per parent. diff --git a/docs/rust.md b/docs/rust.md index 3319bf7c2..fedb16a7c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -16,14 +16,17 @@ rust/ crates/og-vcf/ VCF records read in place, INFO kept in order, GT and CSQ fields; csq-rank crates/og-bam/ BAM reading as pysam does it, on a statically linked htslib crates/og-filters/ the VCF filters of steps 04, 05 and 07 - crates/og-steps/ the step helpers: constraint-join + crates/og-steps/ the step helpers: constraint-join; og family (step 38) + crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table og/ the binary: og [args] bin/og the shim tasks call; runs bin/.og/og-linux- (og-darwin- on a Mac) tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report` and `segments` (were `family_report.py` and `family_segments.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. + +A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py`, `ibd_report.py`, `pedigree_svg.py`, `annotsv_postfilter.py` and `cnvpytor_to_vcf.py` import. The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2`, `zlib-rs`, `memchr`, `smallvec` and `rust-htslib` so far, declared in `rust/Cargo.toml` and locked in `Cargo.lock`). diff --git a/main.nf b/main.nf index b6c6ca378..2199a8422 100644 --- a/main.nf +++ b/main.nf @@ -343,7 +343,7 @@ workflow { // be a families file (assets/families.example.tsv), a pedigree with a // leading family column: one family runs, the one --family_name names // (needed when the file holds more than one). Here only the family and - // sample columns are read; FAMILY_PREPARE (bin/family_report.py prepare) + // sample columns are read; FAMILY_PREPARE (og family prepare) // checks the rest before the joint call starts. def pedigree = params.pedigree // The family's folder under --outdir: --family_name, else 'family' diff --git a/modules/local/glnexus/main.nf b/modules/local/glnexus/main.nf index 379c28e71..1f432529c 100644 --- a/modules/local/glnexus/main.nf +++ b/modules/local/glnexus/main.nf @@ -2,14 +2,14 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FAMILY_PREPARE, GLNEXUS, GLNEXUS_VCF — The family's joint VCF ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - FAMILY_PREPARE reads the pedigree (--pedigree) with bin/family_report.py - prepare, the rule scripts/38-family.sh runs too: it stops the run on a + FAMILY_PREPARE reads the pedigree (--pedigree) with og family prepare + (the Rust binary, bin/og), the rule scripts/38-family.sh runs too: it stops the run on a pedigree problem, or on a gVCF whose sample column is not the samplesheet's id, before the joint call starts. It writes members.tsv, somalier's pedigree (family.ped), WhatsHap's trios (trios.ped), the family's own rows (pedigree.tsv: one family of a families file, picked by --family_name) and family_plan.tsv (which family analyses run, and - why: the table run-all.sh writes with family_report.py plan). + why: the table run-all.sh writes with og family plan). GLNEXUS joint-calls the members' DeepVariant gVCFs with GLnexus's DeepVariantWGS preset into one BCF, and GLNEXUS_VCF writes it out as @@ -51,7 +51,7 @@ process FAMILY_PREPARE { def cv = clinvar_ids ? "--have-clinvar ${clinvar_ids.join(' ')}" : '' def ibd = params.ancestry_fine_panel ? 'yes' : 'no' """ - family_report.py prepare --pedigree families.tsv ${fam} ${args} ${cv} --ibd-panel ${ibd} \\ + og family prepare --pedigree families.tsv ${fam} ${args} ${cv} --ibd-panel ${ibd} \\ --pedigree-out pedigree.tsv --out-dir . printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml diff --git a/modules/local/somalier/main.nf b/modules/local/somalier/main.nf index 6cad30972..de1b2a308 100644 --- a/modules/local/somalier/main.nf +++ b/modules/local/somalier/main.nf @@ -193,7 +193,7 @@ process SAMPLE_QC { from SOMALIER_RELATE's, so a run with sample_qc keeps both. FAMILY_CHECK compares, pair by pair, the relatedness the pedigree implies - with somalier's (bin/family_report.py check, the rule scripts/38-family.sh + with somalier's (og family check, the rule scripts/38-family.sh runs too) and stops the run on a pair that disagrees: a sample swap or a wrong line in the pedigree. WhatsHap and the report wait for it. @@ -277,7 +277,7 @@ process FAMILY_CHECK { script: """ - family_report.py check --members ${members} --pairs ${pairs} --out relationships.tsv + og family check --members ${members} --pairs ${pairs} --out relationships.tsv printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml """ diff --git a/modules/local/whatshap_ped/main.nf b/modules/local/whatshap_ped/main.nf index 77ea54346..900b11c47 100644 --- a/modules/local/whatshap_ped/main.nf +++ b/modules/local/whatshap_ped/main.nf @@ -8,7 +8,7 @@ child and both parents are heterozygous stays unphased, as no genotype tells it. With no trio it writes nothing. - FAMILY_REPORT runs bin/family_report.py report (the rule + FAMILY_REPORT runs og family report (the rule scripts/38-family.sh runs too) on the phased VCF, or on the joint VCF when there is no trio: per child with a sequenced parent, the allele from each parent at every site where the genotypes tell; de novo candidates @@ -16,7 +16,7 @@ cis among the child's clinical-filter candidates (step 23's VCF, when clinical_filter ran); the chromosome map of each child (which stretches came from which parent, and from which grandparent when one is - sequenced; bin/family_segments.py), as a table, a DNA Painter CSV and + sequenced; og family segments), as a table, a DNA Painter CSV and an SVG; and family_report.md. Equivalent to: scripts/38-family.sh (its last two stages) @@ -101,7 +101,7 @@ process FAMILY_REPORT { def cand = [candidate_ids, cl].transpose().collect { p -> "--candidates ${p[0]}=${p[1]}" }.join(' ') def ph = phased ? "--phased ${phased}" : '' """ - family_report.py report --members ${members} --relationships ${relationships} \\ + og family report --members ${members} --relationships ${relationships} \\ --vcf ${vcf} ${ph} ${cand} --par-bed ${par_bed} --out-dir . printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 11c5e36b1..9430d7fe9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -642,6 +642,15 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-pedigree" +version = "0.1.0" +dependencies = [ + "indexmap", + "og-io", + "pycompat", +] + [[package]] name = "og-runlog" version = "0.1.0" @@ -658,8 +667,14 @@ dependencies = [ name = "og-steps" version = "0.1.0" dependencies = [ + "flate2", + "indexmap", "memchr", "og-io", + "og-pedigree", + "og-vcf", + "pycompat", + "smallvec", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c9c55d60f..3eefe62b8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "crates/og-pedigree", "og"] [workspace.package] version = "0.1.0" @@ -22,6 +22,7 @@ og-nf = { path = "crates/og-nf" } og-runlog = { path = "crates/og-runlog" } og-steps = { path = "crates/og-steps" } og-chip = { path = "crates/og-chip" } +og-pedigree = { path = "crates/og-pedigree" } indexmap = "2" flate2 = "1" memchr = "2" diff --git a/rust/README.md b/rust/README.md index fb08ae3d2..a0c41931d 100644 --- a/rust/README.md +++ b/rust/README.md @@ -8,6 +8,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-io` | plain, gzip and BGZF input; Python's universal newlines; atomic writes | | `crates/og-vcf` | VCF records read in place: INFO kept in order, GT and VEP's CSQ fields, the header passed through | | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically | +| `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 95666c62f..4b88797a6 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -4,17 +4,20 @@ //! port wrote the same, by meaning, as the one the Python it replaces //! wrote? `scripts/ci/oracle-diff.sh` calls it for every output of every //! case in `tests/parity/cases.tsv`. +//! - `og dev remnants` ([`remnants`]): does each Python file kept only for +//! its importers (`tests/parity/remnants.tsv`) hold nothing else? pub mod parity; +pub mod remnants; mod zip; -const USAGE: &str = - "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev remnants [--list FILE] [--root DIR]\n"; /// Runs `og dev ARGS`; returns the exit code. pub fn main(args: &[String]) -> u8 { match args.first().map(String::as_str) { Some("parity-compare") => parity::cli(&args[1..]), + Some("remnants") => remnants::cli(&args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/crates/og-dev/src/parity.rs b/rust/crates/og-dev/src/parity.rs index 5293bba0d..e271525da 100644 --- a/rust/crates/og-dev/src/parity.rs +++ b/rust/crates/og-dev/src/parity.rs @@ -8,6 +8,7 @@ //! |---|---| //! | `bytes` | the bytes are equal | //! | `gz-content` | both are gzip and their decompressed bytes are equal | +//! | `content` | as `gz-content` when the expected file is gzip, else as `bytes`: a case whose outputs mix both | //! | `zip-content` | the same member names with the same decompressed bytes (timestamps and deflate bytes may differ) | //! | `png-pixels` | the same size and the same RGBA pixels | //! | `tsv[:keys=A,B][:unordered]` | `#` lines equal and in order, the header equal, the rows equal cell by cell (csv quotes removed), in order unless `unordered`, or matched by the `keys` columns | @@ -32,6 +33,7 @@ use std::path::{Path, PathBuf}; pub enum Kind { Bytes, GzContent, + Content, ZipContent, PngPixels, Tsv { @@ -70,6 +72,7 @@ impl Kind { match name { "bytes" => no_opts(Kind::Bytes), "gz-content" => no_opts(Kind::GzContent), + "content" => no_opts(Kind::Content), "zip-content" => no_opts(Kind::ZipContent), "png-pixels" => no_opts(Kind::PngPixels), "kv" => no_opts(Kind::Kv), @@ -134,6 +137,18 @@ pub fn compare(kind: &Kind, oracle: &Path, new: &Path) -> Outcome { } bytes(&mut open_text(oracle)?, &mut open_text(new)?) } + Kind::Content => { + let mut magic = [0u8; 2]; + let n = open_raw(oracle)? + .read(&mut magic) + .map_err(|e| err(oracle, e))?; + let kind = if n == 2 && magic == [0x1f, 0x8b] { + Kind::GzContent + } else { + Kind::Bytes + }; + compare(&kind, oracle, new) + } Kind::ZipContent => zip_content(&read(oracle)?, &read(new)?), Kind::PngPixels => Ok(differ(&png_pixels(oracle)?, &png_pixels(new)?, "pixels")), Kind::Tsv { keys, unordered } => { diff --git a/rust/crates/og-dev/src/parity/self_test.rs b/rust/crates/og-dev/src/parity/self_test.rs index 909499b0d..36de62532 100644 --- a/rust/crates/og-dev/src/parity/self_test.rs +++ b/rust/crates/og-dev/src/parity/self_test.rs @@ -70,6 +70,11 @@ fn cases() -> Vec { case("gz-content: same content, other compression", "gz-content", &gz(b"x\n", 1), &gz(b"x\n", 9), true), case("gz-content: one changed byte inside", "gz-content", &gz(b"x\n", 6), &gz(b"y\n", 6), false), case("gz-content: new is not gzip", "gz-content", &gz(b"x\n", 6), b"x\n", false), + case("content: gzip, same content, other compression", "content", &gz(b"x\n", 1), &gz(b"x\n", 9), true), + case("content: gzip, one changed byte inside", "content", &gz(b"x\n", 6), &gz(b"y\n", 6), false), + case("content: gzip expected, plain written", "content", &gz(b"x\n", 6), b"x\n", false), + case("content: plain, one changed byte", "content", b"a\tb\n", b"a\tc\n", false), + case("content: plain, gzip of the same bytes written", "content", b"x\n", &gz(b"x\n", 6), false), case("zip-content: same members, other timestamps and deflate", "zip-content", &crate::zip::write(zfiles, false, 1), &crate::zip::write(zfiles, true, 2), true), case("zip-content: one changed member", "zip-content", &crate::zip::write(zfiles, true, 1), &crate::zip::write(zchanged, true, 1), false), case("zip-content: a missing member", "zip-content", &crate::zip::write(zfiles, true, 1), &crate::zip::write(zmissing, true, 1), false), diff --git a/rust/crates/og-dev/src/remnants.rs b/rust/crates/og-dev/src/remnants.rs new file mode 100644 index 000000000..54efcf59a --- /dev/null +++ b/rust/crates/og-dev/src/remnants.rs @@ -0,0 +1,441 @@ +//! `og dev remnants`: a Python file kept after its tool moved to `og`, +//! only because other Python still imports names from it, may hold those +//! names and nothing else. `tests/parity/remnants.tsv` lists each remnant +//! with the names it may define, the files that import it, and the lane +//! that deletes it. The check fails when: +//! +//! - the remnant has no `# remnant:` header naming that lane; +//! - it defines a top-level name the row does not allow, or the row allows +//! a name it no longer defines; +//! - an allowed name is used by no importer and nowhere else in the remnant; +//! - a listed importer is gone or no longer imports the module; +//! - a Python file under `bin/`, `scripts/` or `tests/` imports the module +//! and is not listed. +//! +//! So a remnant only shrinks, and its last importer leaving shows it can go. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use regex::Regex; + +const USAGE: &str = "usage: og dev remnants [--list FILE] [--root DIR]\n"; + +/// One row of the list. +struct Remnant { + file: String, + names: BTreeSet, + importers: Vec, + lane: String, +} + +fn read_list(path: &Path) -> Result, String> { + let text = + fs::read_to_string(path).map_err(|e| format!("cannot read {}: {e}", path.display()))?; + let mut out = Vec::new(); + for (i, line) in text.lines().enumerate() { + if line.trim().is_empty() || line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + let [file, names, importers, lane] = c.as_slice() else { + return Err(format!( + "{} line {}: {} fields, want 4: file, names, importers, lane", + path.display(), + i + 1, + c.len() + )); + }; + let list = |s: &str| -> Vec { + s.split(',') + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_owned) + .collect() + }; + out.push(Remnant { + file: (*file).to_owned(), + names: list(names).into_iter().collect(), + importers: list(importers), + lane: (*lane).to_owned(), + }); + } + Ok(out) +} + +/// The names a Python file defines at top level: `def`, `class`, and +/// assignments, at column 0. Dunder names are skipped. +fn top_level(text: &str) -> BTreeMap { + let def = Regex::new(r"^(?:async\s+def|def|class)\s+([A-Za-z_]\w*)").expect("regex"); + let assign = Regex::new(r"^([A-Za-z_]\w*)\s*(?::[^=]*)?=[^=]").expect("regex"); + let mut out = BTreeMap::new(); + for (i, line) in text.lines().enumerate() { + let name = def + .captures(line) + .or_else(|| assign.captures(line)) + .map(|c| c[1].to_owned()); + if let Some(n) = name.filter(|n| !n.starts_with("__")) { + out.entry(n).or_insert(i); + } + } + out +} + +/// How a file imports `module`: the names it uses from it, or `None` when +/// it does not import it. `import m` and `import m as x` count `x.name`; +/// `from m import a, b as c` counts `a` and `b`. +fn uses(text: &str, module: &str) -> Option> { + let m = regex::escape(module); + let plain = Regex::new(&format!( + r"(?m)^[ \t]*import[ \t]+{m}(?:[ \t]+as[ \t]+([A-Za-z_]\w*))?[ \t]*(?:#.*)?$" + )) + .expect("regex"); + let from = Regex::new(&format!( + r"(?ms)^[ \t]*from[ \t]+{m}[ \t]+import[ \t]+(\([^)]*\)|[^\n]*)" + )) + .expect("regex"); + let mut found = false; + let mut used = BTreeSet::new(); + for c in plain.captures_iter(text) { + found = true; + let alias = c.get(1).map_or(module, |a| a.as_str()); + let attr = + Regex::new(&format!(r"\b{}\.([A-Za-z_]\w*)", regex::escape(alias))).expect("regex"); + used.extend(attr.captures_iter(text).map(|a| a[1].to_owned())); + } + for c in from.captures_iter(text) { + found = true; + let list = c[1] + .split('#') + .next() + .unwrap_or_default() + .trim_matches(|ch| ch == '(' || ch == ')'); + for item in list.split(',') { + if let Some(name) = item.split_whitespace().next() { + used.insert(name.to_owned()); + } + } + } + found.then_some(used) +} + +/// The Python files under `dir`, skipping hidden folders. +fn python_files(dir: &Path, out: &mut Vec) { + let Ok(rd) = fs::read_dir(dir) else { return }; + for e in rd.flatten() { + let p = e.path(); + let hidden = e.file_name().to_string_lossy().starts_with('.'); + if p.is_dir() && !hidden { + python_files(&p, out); + } else if p.extension().is_some_and(|x| x == "py") { + out.push(p); + } + } +} + +/// The problems of one remnant, relative to `root`. +fn check(root: &Path, r: &Remnant) -> Result, String> { + let path = root.join(&r.file); + let text = fs::read_to_string(&path).map_err(|e| format!("cannot read {}: {e}", r.file))?; + let module = Path::new(&r.file) + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut bad = Vec::new(); + let head = text.lines().take(5).find(|l| l.starts_with("# remnant:")); + match head { + None => bad.push(format!( + "{}: no '# remnant:' header in its first 5 lines", + r.file + )), + Some(h) if !h.contains(&r.lane) => { + bad.push(format!( + "{}: the remnant header does not name lane {}", + r.file, r.lane + )); + } + Some(_) => {} + } + let defined = top_level(&text); + for n in defined.keys().filter(|n| !r.names.contains(*n)) { + bad.push(format!( + "{}: defines '{n}', which no importer is allowed to need", + r.file + )); + } + for n in r.names.iter().filter(|n| !defined.contains_key(*n)) { + bad.push(format!( + "{}: '{n}' is allowed but no longer defined; drop it from the list", + r.file + )); + } + + let mut used: BTreeSet = BTreeSet::new(); + for imp in &r.importers { + let Ok(t) = fs::read_to_string(root.join(imp)) else { + bad.push(format!( + "{}: importer {imp} is gone; drop it from the list", + r.file + )); + continue; + }; + match uses(&t, &module) { + Some(u) => used.extend(u), + None => bad.push(format!( + "{}: {imp} no longer imports {module}; drop it from the list", + r.file + )), + } + } + // A name the remnant itself uses outside its own definition line. + let lines: Vec<&str> = text.lines().collect(); + for (n, &at) in &defined { + let word = Regex::new(&format!(r"\b{}\b", regex::escape(n))).expect("regex"); + if lines + .iter() + .enumerate() + .any(|(i, l)| i != at && word.is_match(l)) + { + used.insert(n.clone()); + } + } + for n in r + .names + .iter() + .filter(|n| defined.contains_key(*n) && !used.contains(*n)) + { + bad.push(format!( + "{}: '{n}' is used by no importer; delete it", + r.file + )); + } + + let mut files = Vec::new(); + for d in ["bin", "scripts", "tests"] { + python_files(&root.join(d), &mut files); + } + files.sort(); + for f in files { + let rel = f + .strip_prefix(root) + .unwrap_or(&f) + .to_string_lossy() + .into_owned(); + if rel == r.file || r.importers.contains(&rel) { + continue; + } + if fs::read_to_string(&f).is_ok_and(|t| uses(&t, &module).is_some()) { + bad.push(format!( + "{}: {rel} imports {module} but is not listed; port it, or list it", + r.file + )); + } + } + Ok(bad) +} + +/// Runs `og dev remnants ARGS`: 0 when every remnant holds only what its +/// importers use, 1 when one does not, 2 on a usage or read error. +pub fn cli(args: &[String]) -> u8 { + let (mut list, mut root) = ( + PathBuf::from("tests/parity/remnants.tsv"), + PathBuf::from("."), + ); + let mut it = args.iter(); + while let Some(a) = it.next() { + match (a.as_str(), it.clone().next()) { + ("--list", Some(v)) => { + list = PathBuf::from(v); + it.next(); + } + ("--root", Some(v)) => { + root = PathBuf::from(v); + it.next(); + } + ("-h" | "--help", _) => { + print!("{USAGE}"); + return 0; + } + _ => { + eprint!("{USAGE}"); + eprintln!("og dev remnants: error: unexpected argument '{a}'"); + return 2; + } + } + } + let list = if list.is_absolute() { + list + } else { + root.join(list) + }; + let rows = match read_list(&list) { + Ok(r) => r, + Err(e) => { + eprintln!("og dev remnants: {e}"); + return 2; + } + }; + let mut failed = false; + for r in &rows { + match check(&root, r) { + Ok(bad) if bad.is_empty() => println!( + "ok {}: {} names, {} importers, deleted in lane {}", + r.file, + r.names.len(), + r.importers.len(), + r.lane + ), + Ok(bad) => { + failed = true; + for b in bad { + println!("FAIL {b}"); + } + } + Err(e) => { + eprintln!("og dev remnants: {e}"); + return 2; + } + } + } + u8::from(failed) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Tree(PathBuf); + + impl Tree { + fn new(tag: &str) -> Tree { + let d = std::env::temp_dir().join(format!("og-remnants-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("bin")).unwrap(); + fs::create_dir_all(d.join("tests")).unwrap(); + Tree(d) + } + fn put(&self, rel: &str, text: &str) { + fs::write(self.0.join(rel), text).unwrap(); + } + fn run(&self) -> (u8, Vec) { + let r = &read_list(&self.0.join("list.tsv")).unwrap()[0]; + let bad = check(&self.0, r).unwrap(); + (u8::from(!bad.is_empty()), bad) + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + const LIB: &str = "#!/usr/bin/env python3\n# remnant: kept for a, b; deleted in lane R9\nimport os\n\ + MISSING = {'.', ''}\nSEX: dict = {}\n\n\ndef missing(v):\n return v in MISSING\n\n\ + def die(m):\n raise SystemExit(m)\n\n\nclass Thing:\n pass\n"; + + fn good(tag: &str) -> Tree { + let t = Tree::new(tag); + t.put("bin/lib.py", LIB); + t.put( + "bin/a.py", + "import lib # the rules\nlib.die(lib.missing(1))\n", + ); + t.put( + "bin/b.py", + "def f():\n from lib import (SEX,\n Thing as T)\n", + ); + t.put("list.tsv", "# file\tnames\timporters\tlane\nbin/lib.py\tMISSING,SEX,missing,die,Thing\tbin/a.py,bin/b.py\tR9\n"); + t + } + + #[test] + fn a_clean_remnant_passes() { + let t = good("clean"); + assert_eq!(t.run(), (0, vec![])); + } + + #[test] + fn an_extra_function_fails() { + let t = good("extra"); + t.put("bin/lib.py", &format!("{LIB}\n\ndef helper():\n pass\n")); + let (code, bad) = t.run(); + assert_eq!(code, 1); + assert_eq!( + bad, + ["bin/lib.py: defines 'helper', which no importer is allowed to need"] + ); + } + + #[test] + fn a_dead_name_fails() { + let t = good("dead"); + t.put("bin/a.py", "import lib as L\nL.die(1)\n"); + let (code, bad) = t.run(); + assert_eq!(code, 1); + // MISSING stays used inside the remnant; missing has no user left. + assert_eq!( + bad, + ["bin/lib.py: 'missing' is used by no importer; delete it"] + ); + } + + #[test] + fn a_removed_importer_fails() { + let t = good("gone"); + t.put("bin/b.py", "print('ported')\n"); + let (code, bad) = t.run(); + assert_eq!(code, 1); + assert!( + bad.contains( + &"bin/lib.py: bin/b.py no longer imports lib; drop it from the list".to_owned() + ), + "{bad:?}" + ); + fs::remove_file(t.0.join("bin/b.py")).unwrap(); + let (_, bad) = t.run(); + assert!( + bad.contains( + &"bin/lib.py: importer bin/b.py is gone; drop it from the list".to_owned() + ), + "{bad:?}" + ); + } + + #[test] + fn a_new_importer_or_a_lost_header_fails() { + let t = good("new"); + t.put("tests/test_x.py", "import lib\n"); + let (_, bad) = t.run(); + assert_eq!( + bad, + ["bin/lib.py: tests/test_x.py imports lib but is not listed; port it, or list it"] + ); + let t = good("head"); + t.put( + "bin/lib.py", + &LIB.replace("# remnant: kept for a, b; deleted in lane R9\n", ""), + ); + let (_, bad) = t.run(); + assert_eq!( + bad, + ["bin/lib.py: no '# remnant:' header in its first 5 lines"] + ); + } + + #[test] + fn names_and_imports_are_read_like_python() { + let d = top_level( + "A = 1\nB: int = 2\nC == 3\n D = 4\ndef e():\n F = 5\nclass G(object):\n__all__ = []\nasync def h():\n", + ); + assert_eq!( + d.keys().cloned().collect::>(), + ["A", "B", "G", "e", "h"] + ); + assert_eq!(uses("import libx\n", "lib"), None); + assert_eq!(uses("import lib\n", "lib"), Some(BTreeSet::new())); + let u = uses("from lib import a, b as c # x\n", "lib").unwrap(); + assert_eq!(u.into_iter().collect::>(), ["a", "b"]); + } +} diff --git a/rust/crates/og-pedigree/Cargo.toml b/rust/crates/og-pedigree/Cargo.toml new file mode 100644 index 000000000..54fa8bbe6 --- /dev/null +++ b/rust/crates/og-pedigree/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "og-pedigree" +description = "The pedigree rules of step 38: the pedigree and families file reader, members, kinship, KING's degree bins, the PAR intervals" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +indexmap.workspace = true +og-io.workspace = true +pycompat.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-pedigree/src/lib.rs b/rust/crates/og-pedigree/src/lib.rs new file mode 100644 index 000000000..e05f68c02 --- /dev/null +++ b/rust/crates/og-pedigree/src/lib.rs @@ -0,0 +1,849 @@ +//! The pedigree rules of step 38 and the Nextflow FAMILY workflow. +//! +//! A families file is a pedigree (sample, sex, father, mother, partner, tab +//! separated) with an optional leading family column, so one file can +//! declare several families. This crate reads it as `bin/family_report.py` +//! did, with the same checks and the same messages, word for word: +//! +//! - [`read_pedigree`], [`split_families`], [`family_rows`]: the rows of the +//! file and of one family; +//! - [`build_members`]: every person the rows name, and what is wrong; +//! - [`read_members`]: the `members.tsv` that `og family prepare` writes; +//! - [`Kinship`], [`declared_relationship`], [`degree`]: the relatedness a +//! pedigree implies, binned as KING bins it; +//! - [`Par`]: the pseudoautosomal regions of a BED file. +//! +//! A problem is a [`Die`]: `ERROR: ` lines on stderr and an exit code. + +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{self, BufReader, Write}; + +use indexmap::IndexMap; +use og_io::{Lines, Utf8}; +use pycompat::text::{lower, strip}; + +/// A families file's columns; a pedigree is the same without `family`. +pub const COLUMNS: [&str; 6] = ["family", "sample", "sex", "father", "mother", "partner"]; +/// The pedigree's own columns. +pub const PED_COLUMNS: [&str; 5] = ["sample", "sex", "father", "mother", "partner"]; +/// The family of a file without the family column. +pub const DEFAULT_FAMILY: &str = "family"; +/// Relatedness (twice the kinship coefficient) bins, as KING draws them: +/// the geometric middle between two degrees. +pub const BINS: [(f64, &str); 4] = [ + (0.707, "same person"), + (0.354, "first degree"), + (0.177, "second degree"), + (0.0884, "third degree"), +]; + +/// A stop: each line is printed as `ERROR: ` on stderr, then the +/// tool exits with `code`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Die { + pub lines: Vec, + pub code: u8, +} + +impl Die { + /// A stop with exit 2, the code for a problem in the input. + pub fn new(lines: Vec) -> Die { + Die { lines, code: 2 } + } + + /// One line, exit 2. + pub fn one(line: impl Into) -> Die { + Die::new(vec![line.into()]) + } + + /// A file that cannot be read or written. + pub fn io(path: &str, e: &io::Error) -> Die { + Die::one(format!("cannot read {path}: {e}")) + } + + /// Prints the lines on stderr and returns the exit code. + pub fn report(&self) -> u8 { + let mut err = io::stderr().lock(); + for line in &self.lines { + let _ = writeln!(err, "ERROR: {line}"); + } + self.code + } +} + +/// Values that mean "not given". +fn is_missing_word(v: &str) -> bool { + matches!(v, "" | "." | "0" | "-9") +} + +/// `missing(v)`: `None`, or a value that strips to a missing word. +pub fn missing(v: Option<&str>) -> bool { + v.is_none_or(|v| is_missing_word(strip(v, None))) +} + +/// The sex words a pedigree may use: `Some(Some(sex))`, `Some(None)` for +/// unknown, `None` when the word is not one. +pub fn sex_word(v: &str) -> Option> { + match v { + "male" | "m" | "1" => Some(Some("male")), + "female" | "f" | "2" => Some(Some("female")), + "unknown" | "" | "." | "0" | "-9" => Some(None), + _ => None, + } +} + +/// A sample or family name: letters, digits, `.`, `_` and `-`, not +/// starting with `-`. +pub fn bad_name(v: &str) -> bool { + v.is_empty() + || v.starts_with('-') + || !v + .bytes() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'-')) +} + +/// Text lines of a file as Python's `open(path)` reads them: universal +/// newlines, strict UTF-8, each line without its ending. +pub fn text_lines(path: &str) -> Result, Die> { + let f = File::open(path).map_err(|e| Die::io(path, &e))?; + Lines::new(BufReader::new(f), Utf8::Strict) + .collect::>>() + .map_err(|e| Die::io(path, &e)) +} + +/// One row of a pedigree: the header's columns and the line number. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + pub line: usize, + /// Column name to stripped value, in header order (a column named + /// twice keeps its last value). + pub vals: IndexMap, +} + +impl Row { + /// `r.get(k)`: the value, `None` when the header has no such column. + pub fn get(&self, k: &str) -> Option<&str> { + self.vals.get(k).map(String::as_str) + } + + /// `r.get(k, "").strip()`. + pub fn field(&self, k: &str) -> &str { + strip(self.get(k).unwrap_or(""), None) + } +} + +/// The rows of a pedigree file, and the problems found. The first +/// non-blank line is the header (leading `#` dropped); later lines +/// starting with `#` are comments. +pub fn read_pedigree(path: &str) -> Result<(Vec, Vec), Die> { + Ok(parse_pedigree(path, &text_lines(path)?)) +} + +/// [`read_pedigree`] on lines already read; `path` names the file in messages. +pub fn parse_pedigree(path: &str, lines: &[String]) -> (Vec, Vec) { + let (mut rows, mut errors) = (Vec::new(), Vec::new()); + let mut header: Option> = None; + for (i, line) in lines.iter().enumerate() { + let n = i + 1; + let line = line.trim_end_matches(['\r', '\n']); + if strip(line, None).is_empty() { + continue; + } + let Some(head) = &header else { + let head: Vec = line + .trim_start_matches('#') + .split('\t') + .map(|c| lower(strip(c, None))) + .collect(); + if !head.iter().any(|c| c == "sample") { + return ( + Vec::new(), + vec![format!( + "{path}: the first line must be a header with a 'sample' column \ + (columns: {}, tab separated); it is '{line}'", + COLUMNS.join(", ") + )], + ); + } + let bad: Vec<&str> = head + .iter() + .map(String::as_str) + .filter(|c| !COLUMNS.contains(c)) + .collect(); + if !bad.is_empty() { + errors.push(format!( + "{path}: unknown column{} {}; the columns are {}", + if bad.len() > 1 { "s" } else { "" }, + bad.join(", "), + COLUMNS.join(", ") + )); + } + header = Some(head); + continue; + }; + if line.starts_with('#') { + continue; + } + let vals: Vec<&str> = line.split('\t').map(|v| strip(v, None)).collect(); + if vals.len() > head.len() { + errors.push(format!( + "{path} line {n}: {} fields, the header has {} (fields are tab separated)", + vals.len(), + head.len() + )); + continue; + } + let mut rec = IndexMap::new(); + for (k, col) in head.iter().enumerate() { + rec.insert(col.clone(), vals.get(k).copied().unwrap_or("").to_owned()); + } + rows.push(Row { line: n, vals: rec }); + } + if header.is_none() { + errors.push(format!("{path} is empty")); + } + (rows, errors) +} + +/// One person of a family. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Member { + pub sex: Option, + pub father: Option, + pub mother: Option, + pub partner: Option, + /// Has a row of their own in the pedigree. + pub row: bool, + /// Has a gVCF in the joint call. + pub sequenced: bool, +} + +impl Member { + /// The father, then the mother, those that are named. + pub fn parents(&self) -> impl Iterator { + [&self.father, &self.mother] + .into_iter() + .filter_map(|p| p.as_deref()) + } + + fn rel(&self, k: &str) -> Option<&str> { + match k { + "father" => self.father.as_deref(), + "mother" => self.mother.as_deref(), + _ => self.partner.as_deref(), + } + } +} + +/// Every member by name, in the order the pedigree names them. +pub type Members = IndexMap; + +/// Every person the rows name: the rows, and the parents and partners they +/// name without a row of their own (not sequenced, no parents known), with +/// the problems found. `declared` is the sex given elsewhere (`source` +/// names where, in messages). +pub fn build_members( + rows: &[Row], + declared: &HashMap, + path: &str, + source: &str, +) -> (Members, Vec) { + let mut errors = Vec::new(); + let mut members = Members::new(); + for r in rows { + let where_ = format!("{path} line {}", r.line); + let s = r.field("sample"); + if missing(Some(s)) { + errors.push(format!("{where_}: no sample name")); + continue; + } + if bad_name(s) { + errors.push(format!( + "{where_}: sample name '{s}' may hold only letters, digits, '.', '_' and '-' \ + and may not start with '-'" + )); + continue; + } + if members.contains_key(s) { + errors.push(format!( + "{where_}: sample '{s}' has a second row; each person has one" + )); + continue; + } + let sex_raw = lower(r.field("sex")); + let sex = match sex_word(&sex_raw) { + Some(x) => x, + None => { + errors.push(format!( + "{where_}: sex '{}' of '{s}' is not male, female or unknown", + r.get("sex").unwrap_or("None") + )); + None + } + }; + let mut m = Member { + sex: sex.map(str::to_owned), + row: true, + ..Member::default() + }; + for k in ["father", "mother", "partner"] { + let v = r.field(k); + let val = if missing(Some(v)) { + None + } else if bad_name(v) { + errors.push(format!( + "{where_}: {k} '{v}' of '{s}' is not a valid sample name" + )); + None + } else if v == s { + errors.push(format!("{where_}: '{s}' is named as its own {k}")); + None + } else { + Some(v.to_owned()) + }; + match k { + "father" => m.father = val, + "mother" => m.mother = val, + _ => m.partner = val, + } + } + if m.father.is_some() && m.father == m.mother { + errors.push(format!( + "{where_}: '{s}' has the same person, '{}', as father and mother", + m.father.as_deref().unwrap_or_default() + )); + } + if let Some(d) = declared.get(s).filter(|d| !d.is_empty()) { + if let Some(own) = &m.sex + && own != d + { + errors.push(format!( + "{where_}: the pedigree says '{s}' is {own}, {source} says {d}" + )); + } + if m.sex.is_none() { + m.sex = Some(d.clone()); + } + } + members.insert(s.to_owned(), m); + } + + // People named but without a row: parents and partners not in the file. + let named: Vec = members.values().cloned().collect(); + for m in &named { + for (k, sex) in [ + ("father", Some("male")), + ("mother", Some("female")), + ("partner", None), + ] { + if let Some(p) = m.rel(k) + && !members.contains_key(p) + { + members.insert( + p.to_owned(), + Member { + sex: sex.map(str::to_owned), + ..Member::default() + }, + ); + } + } + } + + let sex_of = |p: &str| members.get(p).and_then(|m| m.sex.as_deref()); + for (s, m) in &members { + if let Some(f) = &m.father + && sex_of(f) == Some("female") + { + errors.push(format!( + "'{f}' is the father of '{s}' but its sex is female" + )); + } + if let Some(mo) = &m.mother + && sex_of(mo) == Some("male") + { + errors.push(format!("'{mo}' is the mother of '{s}' but its sex is male")); + } + } + let fathers: HashSet<&str> = members + .values() + .filter_map(|m| m.father.as_deref()) + .collect(); + let mothers: HashSet<&str> = members + .values() + .filter_map(|m| m.mother.as_deref()) + .collect(); + let mut both: Vec<&str> = fathers.intersection(&mothers).copied().collect(); + both.sort_unstable(); + for p in both { + errors.push(format!("'{p}' is named as a father and as a mother")); + } + + for (s, m) in &members { + if let Some(p) = &m.partner + && let Some(pm) = members.get(p) + && pm.row + && pm.partner.as_deref().is_some_and(|q| q != s) + { + errors.push(format!( + "'{s}' names '{p}' as partner, but '{p}' names '{}'", + pm.partner.as_deref().unwrap_or_default() + )); + } + } + + // No one is their own ancestor. + for s in members.keys() { + let mut seen: HashSet<&str> = HashSet::new(); + let mut todo: Vec<&str> = vec![s]; + 'walk: while let Some(x) = todo.pop() { + let Some(mx) = members.get(x) else { continue }; + for p in mx.parents() { + if p == s { + errors.push(format!( + "'{s}' is among its own ancestors: the parent columns form a loop" + )); + break 'walk; + } + if seen.insert(p) { + todo.push(p); + } + } + } + } + (members, errors) +} + +/// The families of a file: one family keyed `None` for a file without +/// the family column, else each family's rows in file order. +pub type Families = IndexMap, Vec>; + +/// The families of a file's rows, and the problems found. The family column +/// is filled on every row or on none; a person is in one family; the +/// parents and partner a row names are in its own family; a family name +/// follows the sample-name rule and is no person's name. +pub fn split_families(rows: &[Row], path: &str) -> (Families, Vec) { + let single = |rows: &[Row]| -> Families { + let mut f = Families::new(); + if !rows.is_empty() { + f.insert(None, rows.to_vec()); + } + f + }; + if !rows.iter().any(|r| r.vals.contains_key("family")) { + return (single(rows), Vec::new()); + } + let mut errors = Vec::new(); + let mut fams: IndexMap> = IndexMap::new(); + let mut where_of: HashMap = HashMap::new(); + let filled: Vec<&Row> = rows.iter().filter(|r| !missing(r.get("family"))).collect(); + for r in rows { + let fam = r.field("family"); + let where_ = format!("{path} line {}", r.line); + if missing(Some(fam)) { + if let Some(first) = filled.first() { + errors.push(format!( + "{where_}: no family name, but other rows have one (line {}: '{}'); fill the family \ + column on every row, or leave it out of the file", + first.line, + first.field("family") + )); + } + continue; + } + if bad_name(fam) { + errors.push(format!( + "{where_}: family name '{fam}' may hold only letters, digits, '.', '_' and '-' and may not start \ + with '-' (it names the family's folder)" + )); + continue; + } + let s = r.field("sample"); + if !s.is_empty() && !missing(Some(s)) { + if let Some((f0, l0)) = where_of.get(s) + && f0 != fam + { + errors.push(format!( + "{where_}: sample '{s}' is in family '{fam}' here and in family '{f0}' on line {l0}; a person \ + belongs to one family" + )); + continue; + } + where_of + .entry(s.to_owned()) + .or_insert_with(|| (fam.to_owned(), r.line)); + } + fams.entry(fam.to_owned()).or_default().push(r.clone()); + } + if filled.is_empty() { + return (single(rows), errors); + } + let mut people: HashMap<&str, usize> = HashMap::new(); + for r in rows { + for k in ["sample", "father", "mother", "partner"] { + let v = r.field(k); + if !v.is_empty() && !missing(Some(v)) { + people.entry(v).or_insert(r.line); + } + } + } + let mut names: Vec<&String> = fams.keys().collect(); + names.sort(); + for fam in names { + if let Some(l) = people.get(fam.as_str()) { + errors.push(format!( + "{path}: the family name '{fam}' is also a person's name (line {l}); its folder would be the \ + sample's. Give the family another name" + )); + } + } + for (fam, rs) in &fams { + for r in rs { + let s = r.field("sample"); + for k in ["father", "mother", "partner"] { + let v = r.field(k); + if !v.is_empty() + && !missing(Some(v)) + && let Some((f0, l0)) = where_of.get(v) + && f0 != fam + { + errors.push(format!( + "{path} line {}: the {k} of '{s}', '{v}', is in family '{f0}' (line {l0}), '{s}' in family \ + '{fam}'; a family lists its own parents and partners", + r.line + )); + } + } + } + } + let out = fams.into_iter().map(|(k, v)| (Some(k), v)).collect(); + (out, errors) +} + +/// One family of a families file (or a pedigree): the family named, else +/// the one `sample` is in, else the file's only family. The name is +/// [`DEFAULT_FAMILY`] (or the `family` given) for a file without the +/// family column; `None` with no rows when `sample` is in no family. +pub fn family_rows( + path: &str, + family: Option<&str>, + sample: Option<&str>, +) -> Result<(Option, Vec), Die> { + let (rows, errors) = read_pedigree(path)?; + if !errors.is_empty() { + return Err(Die::new(errors)); + } + let (mut fams, errors) = split_families(&rows, path); + if !errors.is_empty() { + return Err(Die::new(errors)); + } + let has_sample = |rs: &[Row], s: &str| rs.iter().any(|r| r.field("sample") == s); + if let Some(rs) = fams.shift_remove(&None) { + let name = family.unwrap_or(DEFAULT_FAMILY); + if bad_name(name) { + return Err(Die::one(format!( + "family name '{name}' may hold only letters, digits, '.', '_' and '-' and may not start with '-' \ + (it names the family's folder)" + ))); + } + for r in &rs { + if ["sample", "father", "mother", "partner"] + .iter() + .any(|k| r.field(k) == name) + { + return Err(Die::one(format!( + "{path} line {}: the family name '{name}' is also a person's name; its folder would be the \ + sample's. Give the family another name", + r.line + ))); + } + } + if let Some(s) = sample + && family.is_none() + && !has_sample(&rs, s) + { + return Ok((Some(name.to_owned()), Vec::new())); + } + return Ok((Some(name.to_owned()), rs)); + } + let mut names: Vec = fams.keys().flatten().cloned().collect(); + names.sort(); + if let Some(f) = family { + let key = Some(f.to_owned()); + return match fams.shift_remove(&key) { + Some(rs) => Ok((key, rs)), + None => Err(Die::one(format!( + "{path} has no family '{f}'; its families are: {}", + if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + } + ))), + }; + } + if let Some(s) = sample { + for name in &names { + let key = Some(name.clone()); + if has_sample(&fams[&key], s) { + let rs = fams.shift_remove(&key).unwrap_or_default(); + return Ok((key, rs)); + } + } + return Ok((None, Vec::new())); + } + if names.len() > 1 { + return Err(Die::one(format!( + "{path} holds {} families ({}); name the one to run (--family, or --family_name in Nextflow)", + names.len(), + names.join(", ") + ))); + } + match names.into_iter().next() { + Some(name) => { + let key = Some(name); + let rs = fams.shift_remove(&key).unwrap_or_default(); + Ok((key, rs)) + } + None => Ok((Some(DEFAULT_FAMILY.to_owned()), Vec::new())), + } +} + +/// A family's rows as a pedigree of its own: the pedigree columns, no +/// family column, `.` for an empty value. +pub fn pedigree_text(rows: &[Row]) -> String { + let mut out = PED_COLUMNS.join("\t"); + out.push('\n'); + for r in rows { + let cells: Vec<&str> = PED_COLUMNS + .iter() + .map(|k| match r.field(k) { + "" => ".", + v => v, + }) + .collect(); + out.push_str(&cells.join("\t")); + out.push('\n'); + } + out +} + +/// A `\t`-separated table read as Python's `dict(zip(head, line.split("\t")))` +/// per line: a short line lacks the last columns. +pub fn read_table(path: &str, strip_hash: bool) -> Result>, Die> { + let lines = text_lines(path)?; + let mut it = lines.into_iter(); + let head_line = it.next().unwrap_or_default(); + let head_line = if strip_hash { + head_line.trim_start_matches('#').to_owned() + } else { + head_line + }; + let head: Vec = head_line.split('\t').map(str::to_owned).collect(); + Ok(it + .map(|l| { + let mut r = IndexMap::new(); + for (k, v) in head.iter().zip(l.split('\t')) { + r.insert(k.clone(), v.to_owned()); + } + r + }) + .collect()) +} + +/// The value of `k` in a table row, or a stop naming the file. +pub fn cell<'a>(r: &'a IndexMap, k: &str, path: &str) -> Result<&'a str, Die> { + r.get(k) + .map(String::as_str) + .ok_or_else(|| Die::one(format!("{path}: a row has no '{k}' column"))) +} + +/// The `members.tsv` that `og family prepare` writes. +pub fn read_members(path: &str) -> Result { + let mut members = Members::new(); + for r in read_table(path, false)? { + let opt = |k: &str| -> Result, Die> { + let v = cell(&r, k, path)?; + Ok((!v.is_empty()).then(|| v.to_owned())) + }; + let m = Member { + sex: opt("sex")?, + father: opt("father")?, + mother: opt("mother")?, + partner: opt("partner")?, + row: cell(&r, "in_pedigree", path)? == "yes", + sequenced: cell(&r, "sequenced", path)? == "yes", + }; + members.insert(cell(&r, "sample", path)?.to_owned(), m); + } + Ok(members) +} + +/// phi(a, b), the kinship coefficient of two members, from the pedigree. +/// Founders (no parent known) are unrelated; an unknown parent is a +/// founder. The same floating-point operations as the Python, memoised. +pub struct Kinship<'a> { + members: &'a Members, + depth: HashMap, + phi: HashMap<(String, String), f64>, +} + +impl<'a> Kinship<'a> { + pub fn new(members: &'a Members) -> Self { + Kinship { + members, + depth: HashMap::new(), + phi: HashMap::new(), + } + } + + fn parents(&self, x: &str) -> (Option<&'a str>, Option<&'a str>) { + self.members + .get(x) + .map_or((None, None), |m| (m.father.as_deref(), m.mother.as_deref())) + } + + fn depth(&mut self, x: &str) -> u32 { + if let Some(&d) = self.depth.get(x) { + return d; + } + let (f, m) = self.parents(x); + let d = [f, m] + .into_iter() + .flatten() + .map(|p| self.depth(p)) + .max() + .map_or(0, |d| d + 1); + self.depth.insert(x.to_owned(), d); + d + } + + /// The kinship coefficient of `a` and `b`. + pub fn phi(&mut self, a: &str, b: &str) -> f64 { + if a == b { + return match self.parents(a) { + (Some(f), Some(m)) => 0.5 * (1.0 + self.phi(f, m)), + _ => 0.5, + }; + } + let key = if a < b { + (a.to_owned(), b.to_owned()) + } else { + (b.to_owned(), a.to_owned()) + }; + if let Some(&v) = self.phi.get(&key) { + return v; + } + let (a, b) = if self.depth(a) < self.depth(b) { + (b, a) + } else { + (a, b) + }; + let v = match self.parents(a) { + (None, None) => 0.0, + (f, m) => { + let terms: Vec = [f, m] + .into_iter() + .flatten() + .map(|p| self.phi(p, b)) + .collect(); + 0.5 * pycompat::num::sum(terms) + } + }; + self.phi.insert(key, v); + v + } +} + +/// The KING degree of a relatedness. +pub fn degree(r: f64) -> &'static str { + BINS.iter() + .find(|(lo, _)| r >= *lo) + .map_or("unrelated", |(_, name)| name) +} + +/// The relationship the pedigree declares between `a` and `b`, with +/// `expected` (twice their kinship) for the pairs it does not name. +pub fn declared_relationship(a: &str, b: &str, members: &Members, expected: f64) -> String { + let none = Member::default(); + let ma = members.get(a).unwrap_or(&none); + let mb = members.get(b).unwrap_or(&none); + if mb.parents().any(|p| p == a) || ma.parents().any(|p| p == b) { + return "parent-child".to_owned(); + } + let pa: HashSet<&str> = ma.parents().collect(); + let shared = mb + .parents() + .collect::>() + .intersection(&pa) + .count(); + match shared { + 2 => return "full siblings".to_owned(), + 1 => return "half siblings".to_owned(), + _ => {} + } + if ma.partner.as_deref() == Some(b) || mb.partner.as_deref() == Some(a) { + return if expected == 0.0 { + "partners".to_owned() + } else { + format!("partners, {}", degree(expected)) + }; + } + if expected > 0.0 { + degree(expected).to_owned() + } else { + "unrelated".to_owned() + } +} + +/// The pseudoautosomal regions of a BED file, per contig. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Par(HashMap>); + +impl Par { + /// Reads `path` (no path: no regions). A line with fewer than three + /// tab-separated fields is skipped. + pub fn read(path: Option<&str>) -> Result { + let mut par: HashMap> = HashMap::new(); + let Some(path) = path else { + return Ok(Par(par)); + }; + for (i, line) in text_lines(path)?.iter().enumerate() { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 3 { + continue; + } + let num = |s: &str| { + pycompat::num::parse_int(s) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| { + Die::one(format!( + "{path} line {}: '{s}' is not a whole number", + i + 1 + )) + }) + }; + let (s, e) = (num(c[1])?, num(c[2])?); + par.entry(c[0].to_owned()).or_default().push((s, e)); + } + Ok(Par(par)) + } + + /// No region at all (no file, or an empty one). + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Whether the 1-based `pos` on `chrom` is in a region (BED: 0-based + /// start, end exclusive). + pub fn contains(&self, chrom: &str, pos: i64) -> bool { + self.0 + .get(chrom) + .is_some_and(|v| v.iter().any(|&(s, e)| s < pos && pos <= e)) + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-pedigree/src/tests.rs b/rust/crates/og-pedigree/src/tests.rs new file mode 100644 index 000000000..aa7ca0b9a --- /dev/null +++ b/rust/crates/og-pedigree/src/tests.rs @@ -0,0 +1,122 @@ +use super::*; + +fn lines(text: &str) -> Vec { + text.split_inclusive('\n').map(str::to_owned).collect() +} + +fn members(text: &str) -> (Members, Vec) { + let (rows, errors) = parse_pedigree("p.tsv", &lines(text)); + assert!(errors.is_empty(), "{errors:?}"); + build_members(&rows, &HashMap::new(), "p.tsv", "the samplesheet") +} + +/// The twin of `test_kinship` in tests/test_family_report.py. +#[test] +fn expected_relatedness() { + let (m, errors) = members( + "sample\tfather\tmother\tpartner\n\ + gp\t.\t.\t.\ngm\t.\t.\t.\n\ + f\tgp\tgm\t.\nu\tgp\tgm\t.\nm\t.\t.\tf\nw\t.\t.\t.\nw2\t.\t.\t.\n\ + a\tf\tm\t.\nb\tf\tm\t.\nh\tf\tw\t.\nc\tu\tw2\t.\n", + ); + assert!(errors.is_empty(), "{errors:?}"); + let mut kin = Kinship::new(&m); + for (x, y, want, label) in [ + ("a", "f", 0.5, "parent-child"), + ("a", "b", 0.5, "full siblings"), + ("a", "h", 0.25, "half siblings"), + ("a", "gp", 0.25, "second degree"), + ("a", "c", 0.125, "third degree"), + ("f", "m", 0.0, "partners"), + ("m", "w", 0.0, "unrelated"), + ] { + let got = 2.0 * kin.phi(x, y); + assert!((got - want).abs() < 1e-9, "{x} {y}: {got}"); + assert_eq!(declared_relationship(x, y, &m, got), label, "{x} {y}"); + } +} + +#[test] +fn pedigree_problems_are_named() { + let (_, e) = parse_pedigree("p.tsv", &lines("kid\tdad\n")); + assert!( + e[0].contains("must be a header with a 'sample' column"), + "{e:?}" + ); + let (_, e) = parse_pedigree("p.tsv", &lines("sample\tcolour\n")); + assert_eq!( + e[0], + "p.tsv: unknown column colour; the columns are family, sample, sex, father, mother, partner" + ); + let (_, e) = parse_pedigree("p.tsv", &[]); + assert_eq!(e, ["p.tsv is empty"]); + + let all = |text: &str| { + let (rows, _) = parse_pedigree("p.tsv", &lines(text)); + build_members(&rows, &HashMap::new(), "p.tsv", "the samplesheet").1 + }; + let e = all("sample\tsex\tfather\nkid\tmale\tmum\nmum\tfemale\t.\n"); + assert_eq!(e, ["'mum' is the father of 'kid' but its sex is female"]); + let e = all("sample\tfather\nkid\tkid\n"); + assert_eq!(e, ["p.tsv line 2: 'kid' is named as its own father"]); + let e = all("sample\tfather\na\tb\nb\ta\n"); + assert!( + e.iter().all(|l| l.contains("among its own ancestors")) && e.len() == 2, + "{e:?}" + ); + let e = all("sample\nkid\nkid\n"); + assert_eq!( + e, + ["p.tsv line 3: sample 'kid' has a second row; each person has one"] + ); + + let (rows, _) = parse_pedigree("p.tsv", &lines("sample\tsex\nkid\tmale\n")); + let declared: HashMap = [("kid".to_owned(), "female".to_owned())].into(); + let (_, e) = build_members(&rows, &declared, "p.tsv", "the samplesheet"); + assert_eq!( + e, + ["p.tsv line 2: the pedigree says 'kid' is male, the samplesheet says female"] + ); +} + +#[test] +fn unnamed_relatives_get_a_member() { + let (m, errors) = members("sample\tsex\tfather\tmother\nkid\tmale\tdad\tmum\n"); + assert!(errors.is_empty()); + assert_eq!(m.keys().collect::>(), ["kid", "dad", "mum"]); + assert_eq!(m["dad"].sex.as_deref(), Some("male")); + assert!(!m["dad"].row && m["kid"].row); +} + +#[test] +fn families_split_by_column() { + let (rows, _) = parse_pedigree( + "p.tsv", + &lines("family\tsample\tfather\nfa\tkid\tdad\nfa\tdad\t.\nfb\tx\t.\n"), + ); + let (f, e) = split_families(&rows, "p.tsv"); + assert!(e.is_empty(), "{e:?}"); + assert_eq!( + f.keys().cloned().collect::>(), + [Some("fa".to_owned()), Some("fb".to_owned())] + ); + assert_eq!(f[&Some("fa".to_owned())].len(), 2); + let (rows, _) = parse_pedigree("p.tsv", &lines("family\tsample\nfa\tkid\n\tdad\n")); + let (_, e) = split_families(&rows, "p.tsv"); + assert!(!e.is_empty()); +} + +#[test] +fn par_is_bed() { + let dir = std::env::temp_dir().join(format!("og-pedigree-par-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("par.bed"); + std::fs::write(&p, "chrX\t10\t20\nshort\n").unwrap(); + let par = Par::read(p.to_str()).unwrap(); + assert!(!par.contains("chrX", 10) && par.contains("chrX", 11) && par.contains("chrX", 20)); + assert!(!par.contains("chrX", 21) && !par.contains("chrY", 15)); + assert!(Par::read(None).unwrap().is_empty()); + std::fs::write(&p, "chrX\tten\t20\n").unwrap(); + assert!(Par::read(p.to_str()).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); +} diff --git a/rust/crates/og-steps/Cargo.toml b/rust/crates/og-steps/Cargo.toml index 58347234f..98e3a9ca2 100644 --- a/rust/crates/og-steps/Cargo.toml +++ b/rust/crates/og-steps/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-steps" -description = "The helpers of the step scripts and modules, one module per tool: constraint, the gnomAD gene constraint join" +description = "The helpers of the step scripts and modules, one module per tool: constraint, the gnomAD gene constraint join; family, step 38's family analysis" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,12 @@ publish.workspace = true [dependencies] og-io.workspace = true memchr.workspace = true +flate2.workspace = true +indexmap.workspace = true +og-pedigree.workspace = true +og-vcf.workspace = true +pycompat.workspace = true +smallvec.workspace = true [lints] workspace = true diff --git a/rust/crates/og-steps/src/family/args.rs b/rust/crates/og-steps/src/family/args.rs new file mode 100644 index 000000000..90ddfd857 --- /dev/null +++ b/rust/crates/og-steps/src/family/args.rs @@ -0,0 +1,229 @@ +//! The command lines of `og family`, read as argparse read the Python's: +//! `--name VALUE` or `--name=VALUE`, repeated options (`append`), options +//! that take every value up to the next option (`nargs="*"`), flags, and +//! fixed choices. No abbreviated options: no caller abbreviates one. + +use std::collections::HashMap; + +use super::Stop; + +/// How an option takes its values. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// One value; the last one given wins. + One, + /// One value each time, every one kept (`action="append"`). + Append, + /// Every value up to the next option (`nargs="*"`); the last use wins. + Star, + /// No value (`action="store_true"`). + Flag, + /// One of these values; the default when absent. + Choice(&'static [&'static str], &'static str), +} + +/// What a verb takes. +pub struct Spec { + pub prog: &'static str, + pub usage: &'static str, + pub options: &'static [(&'static str, Kind)], + pub required: &'static [&'static str], +} + +/// A parsed command line. +#[derive(Debug, Default)] +pub struct Args { + vals: HashMap<&'static str, Vec>, +} + +impl Args { + /// The value of a `One` or `Choice` option. + pub fn one(&self, name: &str) -> Option<&str> { + self.vals + .get(name) + .and_then(|v| v.last()) + .map(String::as_str) + } + + /// Every value of an `Append` or `Star` option (none when absent). + pub fn many(&self, name: &str) -> &[String] { + self.vals.get(name).map_or(&[], Vec::as_slice) + } + + /// Whether a `Flag` was given. + pub fn flag(&self, name: &str) -> bool { + self.vals.contains_key(name) + } +} + +fn takes_value(a: &str) -> bool { + a == "-" + || !a.starts_with('-') + || a.strip_prefix('-') + .is_some_and(|d| d.parse::().is_ok()) +} + +/// Parses `args` by `spec`; a refusal prints argparse's message (exit 2), +/// `--help` the usage (exit 0). +pub fn parse(spec: &Spec, args: &[String]) -> Result { + let usage_error = |msg: String| -> Stop { + eprint!("{}", spec.usage); + eprintln!("{}: error: {msg}", spec.prog); + Stop::Exit(2) + }; + let mut out = Args::default(); + let mut extra: Vec = Vec::new(); + let mut i = 0; + while i < args.len() { + let a = &args[i]; + i += 1; + if a == "-h" || a == "--help" { + print!("{}", spec.usage); + return Err(Stop::Exit(0)); + } + let (name, inline) = match a.split_once('=') { + Some((n, v)) if n.starts_with("--") => (n, Some(v.to_owned())), + _ => (a.as_str(), None), + }; + let Some(&(known, kind)) = spec.options.iter().find(|(o, _)| *o == name) else { + extra.push(a.clone()); + continue; + }; + let one = |inline: Option, i: &mut usize| -> Result { + match inline { + Some(v) => Ok(v), + None => match args.get(*i) { + Some(v) if takes_value(v) => { + *i += 1; + Ok(v.clone()) + } + _ => Err(usage_error(format!( + "argument {name}: expected one argument" + ))), + }, + } + }; + match kind { + Kind::Flag => { + if inline.is_some() { + return Err(usage_error(format!( + "argument {name}: ignored explicit argument '{}'", + inline.unwrap_or_default() + ))); + } + out.vals.insert(known, Vec::new()); + } + Kind::One => { + let v = one(inline, &mut i)?; + out.vals.insert(known, vec![v]); + } + Kind::Append => { + let v = one(inline, &mut i)?; + out.vals.entry(known).or_default().push(v); + } + Kind::Choice(choices, _) => { + let v = one(inline, &mut i)?; + if !choices.contains(&v.as_str()) { + let list: Vec = choices.iter().map(|c| format!("'{c}'")).collect(); + return Err(usage_error(format!( + "argument {name}: invalid choice: '{v}' (choose from {})", + list.join(", ") + ))); + } + out.vals.insert(known, vec![v]); + } + Kind::Star => { + let mut vs = Vec::new(); + if let Some(v) = inline { + vs.push(v); + } else { + while let Some(v) = args.get(i).filter(|v| takes_value(v)) { + vs.push(v.clone()); + i += 1; + } + } + out.vals.insert(known, vs); + } + } + } + let missing: Vec<&str> = spec + .required + .iter() + .copied() + .filter(|r| !out.vals.contains_key(r)) + .collect(); + if !missing.is_empty() { + return Err(usage_error(format!( + "the following arguments are required: {}", + missing.join(", ") + ))); + } + if !extra.is_empty() { + return Err(usage_error(format!( + "unrecognized arguments: {}", + extra.join(" ") + ))); + } + for (name, kind) in spec.options { + if let Kind::Choice(_, default) = kind { + out.vals + .entry(name) + .or_insert_with(|| vec![(*default).to_owned()]); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPEC: Spec = Spec { + prog: "og family t", + usage: "usage: og family t\n", + options: &[ + ("--out", Kind::One), + ("--gvcf", Kind::Append), + ("--have", Kind::Star), + ("--which", Kind::Flag), + ("--panel", Kind::Choice(&["yes", "no"], "no")), + ], + required: &["--out"], + }; + + fn p(args: &[&str]) -> Result { + let v: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + parse(&SPEC, &v) + } + + #[test] + fn kinds() { + let a = p(&[ + "--have", + "a", + "b", + "--gvcf", + "x=1", + "--out", + "o", + "--gvcf=y=2", + "--which", + ]) + .unwrap(); + assert_eq!(a.many("--have"), ["a", "b"]); + assert_eq!(a.many("--gvcf"), ["x=1", "y=2"]); + assert_eq!(a.one("--out"), Some("o")); + assert!(a.flag("--which")); + assert_eq!(a.one("--panel"), Some("no")); + let a = p(&["--out", "o", "--have", "--panel", "yes"]).unwrap(); + assert!(a.many("--have").is_empty() && a.vals.contains_key("--have")); + assert_eq!(a.one("--panel"), Some("yes")); + assert!(matches!(p(&["--have", "a"]), Err(Stop::Exit(2)))); + assert!(matches!( + p(&["--out", "o", "--panel", "maybe"]), + Err(Stop::Exit(2)) + )); + assert!(matches!(p(&["--out", "o", "stray"]), Err(Stop::Exit(2)))); + assert!(matches!(p(&["--out"]), Err(Stop::Exit(2)))); + } +} diff --git a/rust/crates/og-steps/src/family/check.rs b/rust/crates/og-steps/src/family/check.rs new file mode 100644 index 000000000..509547e8c --- /dev/null +++ b/rust/crates/og-steps/src/family/check.rs @@ -0,0 +1,190 @@ +//! `og family check`: for every pair of sequenced members, the relatedness +//! the pedigree implies against the one somalier relate measured. Exits 1 +//! when a pair disagrees: a sample swap, or a wrong line in the pedigree. + +use std::collections::{BTreeSet, HashMap}; +use std::path::Path; + +use og_pedigree::{Die, Kinship, cell, declared_relationship, degree, read_members, read_table}; +use pycompat::fmt::fixed; + +use super::args::{Args, Kind, Spec}; +use super::{Result, print_out, write_file}; + +pub const SPEC: Spec = Spec { + prog: "og family check", + usage: "usage: og family check [-h] --members MEMBERS --pairs PAIRS --out OUT\n", + options: &[ + ("--members", Kind::One), + ("--pairs", Kind::One), + ("--out", Kind::One), + ], + required: &["--members", "--pairs", "--out"], +}; + +/// somalier relate needs this many sites both samples have a genotype at +/// before its relatedness is read as a verdict. +const MIN_SITES: i128 = 1000; +/// A parent and child share an allele at every site: the share of sites +/// where they are opposite homozygotes (IBS0) stays near 0. +const PARENT_CHILD_MAX_IBS0: f64 = 0.01; + +/// (verdict, reason) for one pair. +fn verdict( + declared: &str, + expected: f64, + rel: f64, + ibs0_rate: f64, + n: i128, +) -> (&'static str, String) { + if n < MIN_SITES { + return ( + "not_checked", + format!( + "somalier compared the two at {n} sites, fewer than {MIN_SITES}: too few to tell \ + (a sliced or targeted sample, or the wrong sites file)" + ), + ); + } + let (want, got) = (degree(expected), degree(rel)); + if expected >= 0.177 { + if got != want { + return ( + "mismatch", + format!("declared {declared} ({want}), but somalier measures {got}"), + ); + } + if declared == "parent-child" && ibs0_rate > PARENT_CHILD_MAX_IBS0 { + return ( + "mismatch", + format!( + "first degree, but opposite homozygotes at {}% of sites: a parent and \ + child share an allele at every site (siblings do not)", + fixed(100.0 * ibs0_rate, 1) + ), + ); + } + return ("ok", format!("{got}, as declared")); + } + if rel >= 0.354 { + return ( + "mismatch", + format!("declared {declared}, but somalier measures {got}"), + ); + } + if got != want && rel > expected { + return ( + "warn", + format!("declared {declared}, somalier measures {got}: closer than the pedigree says"), + ); + } + ("ok", format!("{got}, as declared")) +} + +pub fn run(a: &Args) -> Result<()> { + let members_path = a.one("--members").unwrap_or_default(); + let pairs_path = a.one("--pairs").unwrap_or_default(); + let members = read_members(members_path)?; + let mut pairs: HashMap, indexmap::IndexMap> = HashMap::new(); + for r in read_table(pairs_path, true)? { + let key: BTreeSet = [ + cell(&r, "sample_a", pairs_path)?.to_owned(), + cell(&r, "sample_b", pairs_path)?.to_owned(), + ] + .into_iter() + .collect(); + pairs.insert(key, r); + } + let mut kin = Kinship::new(&members); + let mut seq: Vec<&String> = members + .iter() + .filter(|(_, m)| m.sequenced) + .map(|(s, _)| s) + .collect(); + seq.sort(); + let (mut out, mut bad, mut errors) = (Vec::new(), Vec::new(), Vec::new()); + let mut warn = String::new(); + let num_err = |v: &str, k: &str| Die::one(format!("{pairs_path}: {k} '{v}' is not a number")); + for (i, x) in seq.iter().enumerate() { + for y in &seq[i + 1..] { + let key: BTreeSet = [(*x).clone(), (*y).clone()].into_iter().collect(); + let Some(r) = pairs.get(&key) else { + let all: BTreeSet<&String> = pairs.keys().flatten().collect(); + let all: Vec<&str> = all.into_iter().map(String::as_str).collect(); + errors.push(format!( + "somalier relate has no row for {x} and {y}; its samples are: {}", + all.join(", ") + )); + continue; + }; + let e = 2.0 * kin.phi(x, y); + let d = declared_relationship(x, y, &members, e); + let float = |k: &str| -> std::result::Result { + let v = cell(r, k, pairs_path)?; + pycompat::num::parse_float(v).map_err(|_| num_err(v, k)) + }; + let int = |k: &str| -> std::result::Result { + let v = cell(r, k, pairs_path)?; + pycompat::num::parse_int(v).map_err(|_| num_err(v, k)) + }; + let (rel, ibs0, ibs2, n) = + (float("relatedness")?, int("ibs0")?, int("ibs2")?, int("n")?); + let rate = if n != 0 { ibs0 as f64 / n as f64 } else { 0.0 }; + let (v, why) = verdict(&d, e, rel, rate, n); + if v == "mismatch" { + bad.push(format!("{x} and {y}: {why}")); + } else if v != "ok" { + warn.push_str(&format!("WARNING: {x} and {y}: {why}\n")); + } + out.push([ + (*x).clone(), + (*y).clone(), + d, + fixed(e, 3), + fixed(rel, 3), + ibs0.to_string(), + fixed(rate, 4), + ibs2.to_string(), + n.to_string(), + v.to_owned(), + why, + ]); + } + } + if !warn.is_empty() { + eprint!("{warn}"); + } + if !errors.is_empty() { + return Err(Die::new(errors).into()); + } + let mut text = String::from( + "sample_a\tsample_b\tdeclared\texpected_relatedness\trelatedness\tibs0\tibs0_rate\tibs2\tsites\tverdict\treason\n", + ); + for row in &out { + text.push_str(&row.join("\t")); + text.push('\n'); + } + write_file( + Path::new(a.one("--out").unwrap_or_default()), + text.as_bytes(), + )?; + let mut say = String::new(); + for row in &out { + say.push_str(&format!( + "{} and {}: {}; relatedness {} (expected {}), {} sites: {}\n", + row[0], row[1], row[2], row[4], row[3], row[8], row[9] + )); + } + print_out(&say); + if !bad.is_empty() { + let mut lines = vec!["the data disagree with the pedigree:".to_owned()]; + lines.extend(bad.iter().map(|b| format!(" {b}"))); + lines.push( + "A sample swap (at the lab or in the samplesheet), or a wrong line in the pedigree. \ + Check the pedigree and the samplesheet; somalier's own tables are beside relationships.tsv." + .to_owned(), + ); + return Err(Die { lines, code: 1 }.into()); + } + Ok(()) +} diff --git a/rust/crates/og-steps/src/family/mod.rs b/rust/crates/og-steps/src/family/mod.rs new file mode 100644 index 000000000..30051fdd6 --- /dev/null +++ b/rust/crates/og-steps/src/family/mod.rs @@ -0,0 +1,94 @@ +//! `og family`: the family analysis of step 38 (scripts/38-family.sh) and +//! of the Nextflow FAMILY workflow, port of `bin/family_report.py` and +//! `bin/family_segments.py`. +//! +//! - `og family plan`: which family analyses run, are skipped or wait for a +//! member's own run (`plan.rs`); +//! - `og family prepare`: members, somalier's and WhatsHap's pedigrees and +//! the plan for the members with a gVCF (`prepare.rs`); +//! - `og family check`: the relatedness somalier measured against the one +//! the pedigree implies (`check.rs`); +//! - `og family report`: the allele each parent passed on, de novo +//! candidates, compound heterozygotes and the chromosome map, from one +//! read of the joint VCF (`report.rs`); +//! - `og family segments`: the chromosome map alone ([`segments`]). +//! +//! The pedigree rules live in `og-pedigree`. Exit codes are the Python's: +//! 2 for a problem in the input, 1 when `check` finds a pair that +//! disagrees with the pedigree. + +mod args; +mod check; +mod plan; +mod prepare; +mod report; +pub mod segments; +mod vcf; + +use og_pedigree::Die; + +const USAGE: &str = "usage: og family {plan,prepare,check,report,segments} ...\n"; + +/// Runs `og family ARGS`; returns the exit code. +pub fn cli(args: &[String]) -> u8 { + let Some(verb) = args.first() else { + eprint!("{USAGE}"); + eprintln!("og family: error: the following arguments are required: verb"); + return 2; + }; + let rest = &args[1..]; + let out = match verb.as_str() { + "plan" => args::parse(&plan::SPEC, rest).and_then(|a| plan::run(&a)), + "prepare" => args::parse(&prepare::SPEC, rest).and_then(|a| prepare::run(&a)), + "check" => args::parse(&check::SPEC, rest).and_then(|a| check::run(&a)), + "report" => args::parse(&report::SPEC, rest).and_then(|a| report::run(&a)), + "segments" => args::parse(&segments::SPEC, rest).and_then(|a| segments::run(&a)), + "-h" | "--help" => { + print!("{USAGE}"); + return 0; + } + other => { + eprint!("{USAGE}"); + eprintln!("og family: error: unknown verb '{other}'"); + return 2; + } + }; + match out { + Ok(()) => 0, + Err(Stop::Die(d)) => d.report(), + Err(Stop::Exit(code)) => code, + } +} + +/// Why a verb stopped early. +#[derive(Debug)] +pub enum Stop { + /// `ERROR:` lines and an exit code. + Die(Die), + /// Already reported (usage, `--help`): just this exit code. + Exit(u8), +} + +impl From for Stop { + fn from(d: Die) -> Stop { + Stop::Die(d) + } +} + +type Result = std::result::Result; + +/// Writes `text` to `path` through a temporary file renamed into place. +fn write_file(path: &std::path::Path, text: &[u8]) -> std::result::Result<(), Die> { + use std::io::Write; + let shown = path.display().to_string(); + let fail = |e: std::io::Error| Die::one(format!("cannot write {shown}: {e}")); + let mut f = og_io::AtomicFile::create(path).map_err(fail)?; + f.write_all(text).map_err(fail)?; + f.commit().map_err(fail) +} + +/// Prints `text` on stdout; a closed pipe is not an error. +fn print_out(text: &str) { + use std::io::Write; + let _ = std::io::stdout().lock().write_all(text.as_bytes()); +} diff --git a/rust/crates/og-steps/src/family/plan.rs b/rust/crates/og-steps/src/family/plan.rs new file mode 100644 index 000000000..8f517f595 --- /dev/null +++ b/rust/crates/og-steps/src/family/plan.rs @@ -0,0 +1,350 @@ +//! `og family plan`: checks a families file and picks one family, then +//! says, for each analysis of steps 38 and 39, whether it runs, is skipped +//! or waits for a member's own run, and why. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; + +use og_pedigree::{Die, Members, build_members, family_rows, pedigree_text}; +use pycompat::fmt::ljust; + +use super::args::{Args, Kind, Spec}; +use super::{Result, print_out, write_file}; + +pub const SPEC: Spec = Spec { + prog: "og family plan", + usage: "usage: og family plan [-h] --families FAMILIES [--family FAMILY] [--sample SAMPLE] [--sex SEX]\n [--have-gvcf [ID ...]] [--have-clinvar [ID ...]] [--ibd-panel {yes,no}]\n [--write-pedigree DIR] [--which] [--out OUT]\n", + options: &[ + ("--families", Kind::One), + ("--family", Kind::One), + ("--sample", Kind::One), + ("--sex", Kind::One), + ("--have-gvcf", Kind::Star), + ("--have-clinvar", Kind::Star), + ("--ibd-panel", Kind::Choice(&["yes", "no"], "no")), + ("--write-pedigree", Kind::One), + ("--which", Kind::Flag), + ("--out", Kind::One), + ], + required: &["--families"], +}; + +/// The columns of `family_plan.tsv`. +pub const PLAN_COLUMNS: [&str; 5] = ["family", "analysis", "members", "decision", "reason"]; + +/// One row of `family_plan.tsv`. +pub struct PlanRow { + pub family: String, + pub analysis: &'static str, + pub members: String, + pub decision: &'static str, + pub reason: String, +} + +/// The rows of `family_plan.tsv`: for each analysis of steps 38 and 39, +/// whether it runs, is skipped, or waits for a member's own run, and why. +/// `have_gvcf` and `have_clinvar` are the samples with a gVCF and with +/// step 06's ClinVar hits. +pub fn plan_family( + name: &str, + members: &Members, + have_gvcf: &HashSet, + have_clinvar: &HashSet, + ibd_panel: bool, +) -> Vec { + let rows_of: Vec<&str> = members + .iter() + .filter(|(_, m)| m.row) + .map(|(s, _)| s.as_str()) + .collect::>() + .into_iter() + .collect(); + let seq: Vec<&str> = rows_of + .iter() + .copied() + .filter(|s| have_gvcf.contains(*s)) + .collect(); + let lacking: Vec<&str> = rows_of + .iter() + .copied() + .filter(|s| !have_gvcf.contains(*s)) + .collect(); + let no_gvcf = |ss: &[&str]| -> String { + ss.iter() + .map(|s| format!("{s} has no gVCF yet: run run-all.sh {s} first")) + .collect::>() + .join("; ") + }; + let mut out = Vec::new(); + let mut row = |analysis: &'static str, who: &[&str], decision: &'static str, reason: String| { + out.push(PlanRow { + family: name.to_owned(), + analysis, + members: if who.is_empty() { + ".".to_owned() + } else { + who.join(",") + }, + decision, + reason, + }); + }; + + let joint: (&'static str, String) = if rows_of.len() < 2 { + ( + "skipped", + format!( + "the family has {} sample with a row ({}); the family analysis needs two or more", + rows_of.len(), + if rows_of.is_empty() { + "none".to_owned() + } else { + rows_of.join(", ") + } + ), + ) + } else if seq.len() < 2 { + ("waiting", no_gvcf(&lacking)) + } else { + let mut why = format!( + "GLnexus joint call, somalier relationship check and the family report of {}", + seq.join(", ") + ); + if !lacking.is_empty() { + why.push_str(&format!( + "; left out: {} (run step 38 again after its run)", + no_gvcf(&lacking) + )); + } + ("runs", why) + }; + let runs = joint.0 == "runs"; + row( + "joint_call", + if runs { &seq } else { &rows_of }, + joint.0, + joint.1.clone(), + ); + + let in_seq = |p: &Option| p.as_deref().is_some_and(|p| seq.contains(&p)); + let trios: Vec<&str> = seq + .iter() + .copied() + .filter(|s| in_seq(&members[*s].father) && in_seq(&members[*s].mother)) + .collect(); + let duos: Vec<&str> = seq + .iter() + .copied() + .filter(|s| { + !trios.contains(s) && (in_seq(&members[*s].father) || in_seq(&members[*s].mother)) + }) + .collect(); + if !runs { + for a in ["trio_phasing", "inheritance_map", "shared_segments"] { + row(a, &rows_of, joint.0, joint.1.clone()); + } + } else { + if !trios.is_empty() { + row( + "trio_phasing", + &trios, + "runs", + format!( + "WhatsHap phases {} by inheritance from both parents", + trios.join(", ") + ), + ); + } else { + let has_one = if duos.is_empty() { + String::new() + } else { + format!(" ({} has one)", duos.join(", ")) + }; + row( + "trio_phasing", + &duos, + "skipped", + format!( + "no child has both parents sequenced{has_one}; phasing by inheritance needs a trio" + ), + ); + } + let mut kids: Vec<&str> = trios.iter().chain(duos.iter()).copied().collect(); + kids.sort_unstable(); + if !kids.is_empty() { + row( + "inheritance_map", + &kids, + "runs", + format!( + "the allele each sequenced parent passed on, and the chromosome map, of {}", + kids.join(", ") + ), + ); + } else { + row( + "inheritance_map", + &[], + "skipped", + "no child has a sequenced parent".to_owned(), + ); + } + if ibd_panel { + row( + "shared_segments", + &seq, + "runs", + "Beagle phases the joint call against step 26b's panel and hap-ibd lists the DNA each pair shares" + .to_owned(), + ); + } else { + row( + "shared_segments", + &seq, + "skipped", + "no phasing panel: install it with setup.sh --ancestry-fine-panel".to_owned(), + ); + } + } + + let couples: BTreeSet<(String, String)> = rows_of + .iter() + .filter_map(|s| { + let p = members[*s].partner.as_deref()?; + members.get(p).filter(|pm| pm.row)?; + let (a, b) = if *s < p { (*s, p) } else { (p, *s) }; + Some((a.to_owned(), b.to_owned())) + }) + .collect(); + if couples.is_empty() { + row( + "couple_carrier", + &[], + "skipped", + "no two samples of the family are marked as partners".to_owned(), + ); + } + // Step 39 reads each partner's ClinVar hits, never a gVCF: a partner run + // from a VCF alone still gets the screen. + for (a, b) in &couples { + let pair = [a.as_str(), b.as_str()]; + let lack: Vec<&str> = pair + .iter() + .copied() + .filter(|s| !have_clinvar.contains(*s)) + .collect(); + if !lack.is_empty() { + let why: Vec = lack + .iter() + .map(|s| { + format!( + "{s} has no ClinVar hits yet (step 06 writes clinvar/{s}_clinvar_hits.vcf): run run-all.sh {s} first" + ) + }) + .collect(); + row("couple_carrier", &pair, "waiting", why.join("; ")); + } else { + row( + "couple_carrier", + &pair, + "runs", + format!("carrier screen of {a} and {b}"), + ); + } + } + out +} + +/// `family_plan.tsv` as text. +pub fn plan_text(rows: &[PlanRow]) -> String { + let mut s = PLAN_COLUMNS.join("\t"); + s.push('\n'); + for r in rows { + s.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\n", + r.family, r.analysis, r.members, r.decision, r.reason + )); + } + s +} + +pub fn run(a: &Args) -> Result<()> { + let families = a.one("--families").unwrap_or_default(); + let sample = a.one("--sample"); + let sex = a + .one("--sex") + .map(|s| pycompat::text::lower(pycompat::text::strip(s, None))) + .unwrap_or_default(); + if !sex.is_empty() && sex != "male" && sex != "female" { + return Err(Die::one(format!( + "--sex {}: the sex is male or female", + a.one("--sex").unwrap_or_default() + )) + .into()); + } + if !sex.is_empty() && sample.is_none_or(str::is_empty) { + return Err(Die::one("--sex needs --sample").into()); + } + let sample = sample.filter(|s| !s.is_empty()); + let family = a.one("--family").filter(|s| !s.is_empty()); + let (name, rows) = family_rows(families, family, sample)?; + if let Some(s) = sample + && rows.is_empty() + { + if a.flag("--which") { + return Ok(()); + } + return Err(Die::one(format!("{s} is in no family of {families}")).into()); + } + let name = name.unwrap_or_default(); + let mut declared = HashMap::new(); + if !sex.is_empty() + && let Some(s) = sample + { + declared.insert(s.to_owned(), sex); + } + let (members, errors) = + build_members(&rows, &declared, families, "the sex given to run-all.sh"); + if !errors.is_empty() { + return Err(Die::new(errors).into()); + } + if let Some(s) = sample + && !rows.is_empty() + && !members.contains_key(s) + { + return Err(Die::one(format!("{families}: '{s}' is not in family '{name}'")).into()); + } + if a.flag("--which") { + print_out(&format!("{name}\n")); + return Ok(()); + } + if let Some(dir) = a.one("--write-pedigree") { + let d = Path::new(dir).join(&name); + std::fs::create_dir_all(&d) + .map_err(|e| Die::one(format!("cannot create {}: {e}", d.display())))?; + write_file(&d.join("pedigree.tsv"), pedigree_text(&rows).as_bytes())?; + } + let set = |k: &str| -> HashSet { a.many(k).iter().cloned().collect() }; + let plan = plan_family( + &name, + &members, + &set("--have-gvcf"), + &set("--have-clinvar"), + a.one("--ibd-panel") == Some("yes"), + ); + if let Some(out) = a.one("--out") { + write_file(Path::new(out), plan_text(&plan).as_bytes())?; + } + let mut text = String::new(); + for r in &plan { + text.push_str(&format!( + "{} {} {} {}\n", + r.family, + ljust(r.analysis, 16), + ljust(r.decision, 8), + r.reason + )); + } + print_out(&text); + Ok(()) +} diff --git a/rust/crates/og-steps/src/family/prepare.rs b/rust/crates/og-steps/src/family/prepare.rs new file mode 100644 index 000000000..4e13ce3ac --- /dev/null +++ b/rust/crates/og-steps/src/family/prepare.rs @@ -0,0 +1,238 @@ +//! `og family prepare`: reads and checks the pedigree, or one family of a +//! families file, and checks that each gVCF names its own sample. Writes +//! `members.tsv` (every person of the pedigree), `family.ped` (somalier +//! relate's pedigree: the sequenced members), `trios.ped` (WhatsHap's: the +//! children whose two parents are sequenced) and `family_plan.tsv`. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use indexmap::IndexMap; +use og_pedigree::{Die, Kinship, build_members, declared_relationship, family_rows, pedigree_text}; +use pycompat::fmt::fixed; + +use super::args::{Args, Kind, Spec}; +use super::plan::{plan_family, plan_text}; +use super::{Result, print_out, write_file}; + +pub const SPEC: Spec = Spec { + prog: "og family prepare", + usage: "usage: og family prepare [-h] --pedigree PEDIGREE [--gvcf ID=PATH] [--declared-sex ID=SEX] [--family FAMILY]\n [--have-clinvar [ID ...]] [--ibd-panel {yes,no}] [--pedigree-out PEDIGREE_OUT]\n --out-dir OUT_DIR\n", + options: &[ + ("--pedigree", Kind::One), + ("--gvcf", Kind::Append), + ("--declared-sex", Kind::Append), + ("--family", Kind::One), + ("--have-clinvar", Kind::Star), + ("--ibd-panel", Kind::Choice(&["yes", "no"], "no")), + ("--pedigree-out", Kind::One), + ("--out-dir", Kind::One), + ], + required: &["--pedigree", "--out-dir"], +}; + +/// The sample columns of a VCF's `#CHROM` line (reads the header only); +/// `None` when it has none. +fn gvcf_samples(path: &str) -> std::result::Result>, Die> { + let r = og_io::open(path).map_err(|e| Die::io(path, &e))?; + let mut lines = og_io::Lines::new(r, og_io::Utf8::Replace); + let mut line = String::new(); + while lines.read_line(&mut line).map_err(|e| Die::io(path, &e))? { + if line.starts_with("#CHROM") { + let l = line.trim_end_matches('\n'); + return Ok(Some(l.split('\t').skip(9).map(str::to_owned).collect())); + } + if !line.starts_with('#') { + break; + } + } + Ok(None) +} + +pub fn run(a: &Args) -> Result<()> { + let pedigree = a.one("--pedigree").unwrap_or_default(); + let mut declared = HashMap::new(); + for kv in a.many("--declared-sex") { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + let v = pycompat::text::lower(pycompat::text::strip(v, None)); + if !v.is_empty() { + if v != "male" && v != "female" { + return Err( + Die::one(format!("--declared-sex {kv}: the sex is male or female")).into(), + ); + } + declared.insert(k.to_owned(), v); + } + } + let family = a.one("--family").filter(|s| !s.is_empty()); + let (name, rows) = family_rows(pedigree, family, None)?; + let name = name.unwrap_or_default(); + let (mut members, mut errors) = build_members(&rows, &declared, pedigree, "the samplesheet"); + let mut gvcfs: IndexMap = IndexMap::new(); + for kv in a.many("--gvcf") { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + gvcfs.insert(k.to_owned(), v.to_owned()); + } + let mut seq: Vec = gvcfs + .keys() + .filter(|s| members.get(*s).is_some_and(|m| m.row)) + .cloned() + .collect(); + seq.sort(); + if seq.len() < 2 { + errors.push(format!( + "the pedigree names {} of the processed samples ({}); the family analysis needs two \ + or more, each with a row in {pedigree} and a gVCF", + seq.len(), + if seq.is_empty() { + "none".to_owned() + } else { + seq.join(", ") + } + )); + } + for s in &seq { + let path = &gvcfs[s]; + let names = gvcf_samples(path)?; + if names.as_deref() != Some(std::slice::from_ref(s)) { + let base = Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let (plural, listed) = match names.as_deref() { + Some(n) if !n.is_empty() => (if n.len() > 1 { "s" } else { "" }, n.join(", ")), + _ => ("", "none (no #CHROM line)".to_owned()), + }; + errors.push(format!( + "the gVCF of '{s}' ({base}) holds the sample{plural} {listed}, not '{s}'. The joint call names each \ + column after the gVCF's own sample. Rename it: echo {s} > name.txt; bcftools reheader \ + -s name.txt -o {s}.g.vcf.gz {base}; bcftools index -t {s}.g.vcf.gz" + )); + } + } + if !errors.is_empty() { + return Err(Die::new(errors).into()); + } + for s in &seq { + if let Some(m) = members.get_mut(s) { + m.sequenced = true; + } + } + let out_dir = Path::new(a.one("--out-dir").unwrap_or_default()); + std::fs::create_dir_all(out_dir) + .map_err(|e| Die::one(format!("cannot create {}: {e}", out_dir.display())))?; + + let mut names: Vec<&String> = members.keys().collect(); + names.sort(); + let mut text = String::from("sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n"); + for s in names { + let m = &members[s]; + let o = |v: &Option| v.clone().unwrap_or_default(); + let yn = |b: bool| if b { "yes" } else { "no" }; + text.push_str(&format!( + "{s}\t{}\t{}\t{}\t{}\t{}\t{}\n", + o(&m.sex), + o(&m.father), + o(&m.mother), + o(&m.partner), + yn(m.row), + yn(m.sequenced) + )); + } + write_file(&out_dir.join("members.tsv"), text.as_bytes())?; + + let in_seq = |p: &Option| p.as_deref().is_some_and(|p| seq.iter().any(|s| s == p)); + let code = |sex: &Option, none: &'static str| match sex.as_deref() { + Some("male") => "1", + Some("female") => "2", + _ => none, + }; + let mut ped = String::new(); + for s in &seq { + let m = &members[s]; + let parent = |p: &Option| { + if in_seq(p) { + p.clone().unwrap_or_default() + } else { + "-9".to_owned() + } + }; + ped.push_str(&format!( + "family\t{s}\t{}\t{}\t{}\t-9\n", + parent(&m.father), + parent(&m.mother), + code(&m.sex, "-9") + )); + } + write_file(&out_dir.join("family.ped"), ped.as_bytes())?; + let trios: Vec<&String> = seq + .iter() + .filter(|s| in_seq(&members[*s].father) && in_seq(&members[*s].mother)) + .collect(); + let mut tp = String::new(); + for s in &trios { + let m = &members[*s]; + tp.push_str(&format!( + "family\t{s}\t{}\t{}\t{}\t0\n", + m.father.as_deref().unwrap_or_default(), + m.mother.as_deref().unwrap_or_default(), + code(&m.sex, "0") + )); + } + write_file(&out_dir.join("trios.ped"), tp.as_bytes())?; + if let Some(p) = a.one("--pedigree-out") { + write_file(Path::new(p), pedigree_text(&rows).as_bytes())?; + } + // The same decision plan writes before the step runs (run-all.sh). + let have: HashSet = seq.iter().cloned().collect(); + let clinvar: HashSet = a.many("--have-clinvar").iter().cloned().collect(); + let plan = plan_family( + &name, + &members, + &have, + &clinvar, + a.one("--ibd-panel") == Some("yes"), + ); + write_file( + &out_dir.join("family_plan.tsv"), + plan_text(&plan).as_bytes(), + )?; + + let mut say = format!( + "Family: {} people in the pedigree, {} sequenced ({})\n", + members.len(), + seq.len(), + seq.join(", ") + ); + let mut kin = Kinship::new(&members); + for (i, x) in seq.iter().enumerate() { + for y in &seq[i + 1..] { + let e = 2.0 * kin.phi(x, y); + say.push_str(&format!( + " {x} and {y}: declared {} (expected relatedness {})\n", + declared_relationship(x, y, &members, e), + fixed(e, 3) + )); + } + } + for s in &seq { + let ps: Vec<&str> = members[s] + .parents() + .filter(|p| seq.iter().any(|q| q == p)) + .collect(); + if !ps.is_empty() { + say.push_str(&format!(" child {s}: {} sequenced\n", ps.join(" and "))); + } + } + let tr: Vec<&str> = trios.iter().map(|s| s.as_str()).collect(); + say.push_str(&format!( + "Trios WhatsHap phases: {}\n", + if tr.is_empty() { + "none (no child has both parents sequenced)".to_owned() + } else { + tr.join(", ") + } + )); + print_out(&say); + Ok(()) +} diff --git a/rust/crates/og-steps/src/family/report.rs b/rust/crates/og-steps/src/family/report.rs new file mode 100644 index 000000000..0a9cd8905 --- /dev/null +++ b/rust/crates/og-steps/src/family/report.rs @@ -0,0 +1,1022 @@ +//! `og family report`: per child with a sequenced parent, the allele each +//! parent passed on at every site of the joint VCF where the genotypes +//! tell (`_inheritance.tsv.gz`), de novo candidates when both +//! parents are sequenced (`de_novo.tsv`), pairs of candidate variants in +//! one gene in trans or in cis (`compound_het.tsv`), the chromosome map +//! (`_segments.tsv`, `_dnapainter.csv`, `_segments.svg`) +//! and `family_report.md`, which says it in words. +//! +//! The joint VCF is read once: the inheritance and the map take their +//! genotypes from the same pass. With `--phased` the inheritance reads +//! WhatsHap's file (it needs the phase) while the map reads `--vcf` on a +//! second thread. Each child's inheritance file is compressed on a thread +//! of its own. + +use std::collections::HashMap; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{SyncSender, sync_channel}; +use std::thread::{self, JoinHandle}; + +use flate2::{Compression, GzBuilder}; +use indexmap::IndexMap; +use og_io::AtomicFile; +use og_pedigree::{Die, Members, Par, cell, read_members, read_table}; +use pycompat::fmt::fixed; + +use super::args::{Args, Kind, Spec}; +use super::segments::{self, Collector, Map}; +use super::vcf::{self, Calls, resolve, set}; +use super::{Result, print_out, write_file}; + +pub const SPEC: Spec = Spec { + prog: "og family report", + usage: "usage: og family report [-h] --members MEMBERS --relationships RELATIONSHIPS --vcf VCF [--phased PHASED]\n [--candidates ID=VCF] [--par-bed PAR_BED] --out-dir OUT_DIR\n", + options: &[ + ("--members", Kind::One), + ("--relationships", Kind::One), + ("--vcf", Kind::One), + ("--phased", Kind::One), + ("--candidates", Kind::Append), + ("--par-bed", Kind::One), + ("--out-dir", Kind::One), + ], + required: &["--members", "--relationships", "--vcf", "--out-dir"], +}; + +/// De novo candidates: every one of the three must have these. +const MIN_GQ: i128 = 20; +const MIN_DP: i128 = 10; +const HET_AB: (f64, f64) = (0.25, 0.75); +/// Bytes of inheritance rows handed to a child's compressing thread at a time. +const CHUNK: usize = 1 << 18; +/// The level of the inheritance files (the content is what is compared). +const GZ_LEVEL: u32 = 6; + +fn is_primary(c: &[u8]) -> bool { + c == b"chrY" || segments::chrom_index(c).is_some() +} + +type Genes = std::rc::Rc>; +type CandKey = (Vec, i64, Vec, Vec); + +/// `{(chrom, pos, ref, alt): genes}` of a candidate VCF: VEP's CSQ SYMBOL, +/// else snpEff's ANN gene, else INFO/GENE. +fn candidate_genes(path: &str) -> std::result::Result, Die> { + let r = og_io::open(path).map_err(|e| Die::io(path, &e))?; + let mut lines = og_vcf::Lines::new(r); + let (mut symbol, mut ann) = (None::, false); + let mut out = HashMap::new(); + let mut number = 0; + while let Some(line) = lines.next_line().map_err(|e| Die::io(path, &e))? { + number += 1; + if line.starts_with(b"##INFO= 0 { + symbol = rest[..end] + .split(|&c| c == b'|') + .position(|f| f == b"SYMBOL"); + } + } + continue; + } + if line.starts_with(b"##INFO= = HashMap::new(); + for kv in c[7].split(|&b| b == b';') { + match kv.iter().position(|&b| b == b'=') { + Some(k) => info.insert(&kv[..k], &kv[k + 1..]), + None => info.insert(kv, b""), + }; + } + let mut genes: Vec = Vec::new(); + let mut add = |g: &[u8]| { + let g = String::from_utf8_lossy(g).into_owned(); + if !genes.contains(&g) { + genes.push(g); + } + }; + if let (Some(si), Some(csq)) = (symbol, info.get(&b"CSQ"[..])) { + for entry in csq.split(|&b| b == b',') { + if let Some(g) = entry + .split(|&b| b == b'|') + .nth(si) + .filter(|g| !g.is_empty()) + { + add(g); + } + } + } else if let (true, Some(a)) = (ann, info.get(&b"ANN"[..])) { + for entry in a.split(|&b| b == b',') { + if let Some(g) = entry.split(|&b| b == b'|').nth(3).filter(|g| !g.is_empty()) { + add(g); + } + } + } else if let Some(g) = info.get(&b"GENE"[..]).filter(|g| !g.is_empty()) { + for x in g.split(|&b| b == b',').filter(|x| !x.is_empty()) { + add(x); + } + } + if !genes.is_empty() { + let pos = vcf::pos(c[1], path, number)?; + let genes = Genes::new(genes); + for alt in c[4].split(|&b| b == b',') { + out.insert( + (c[0].to_vec(), pos, c[3].to_vec(), alt.to_vec()), + genes.clone(), + ); + } + } + } + Ok(out) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum How { + Genotype, + Ambiguous, + Inconsistent, +} + +impl How { + fn as_bytes(self) -> &'static [u8] { + match self { + How::Genotype => b"genotype", + How::Ambiguous => b"ambiguous", + How::Inconsistent => b"inconsistent", + } + } +} + +#[derive(Default)] +struct Counts { + sites: u64, + father: u64, + mother: u64, + both: u64, + ambiguous: u64, + inconsistent: u64, + skipped_sex: u64, +} + +/// A child's inheritance file, compressed on its own thread. +struct GzThread { + tx: Option>>, + handle: Option>>, + buf: Vec, + path: PathBuf, +} + +impl GzThread { + fn start(path: PathBuf) -> std::result::Result { + let file = AtomicFile::create(&path) + .map_err(|e| Die::one(format!("cannot write {}: {e}", path.display())))?; + let (tx, rx) = sync_channel::>(4); + let handle = thread::spawn(move || { + let mut gz = GzBuilder::new() + .mtime(0) + .operating_system(255) + .write(file, Compression::new(GZ_LEVEL)); + for chunk in rx { + gz.write_all(&chunk)?; + } + gz.finish() + }); + Ok(GzThread { + tx: Some(tx), + handle: Some(handle), + buf: Vec::with_capacity(CHUNK + 4096), + path, + }) + } + + fn flush(&mut self) -> std::result::Result<(), Die> { + let chunk = std::mem::replace(&mut self.buf, Vec::with_capacity(CHUNK + 4096)); + let sent = self.tx.as_ref().is_some_and(|tx| tx.send(chunk).is_ok()); + if sent { + Ok(()) + } else { + Err(self + .close() + .err() + .unwrap_or_else(|| self.fail("the writer stopped"))) + } + } + + fn fail(&self, why: &str) -> Die { + Die::one(format!("cannot write {}: {why}", self.path.display())) + } + + /// Sends what is left, waits for the thread and returns the finished + /// file, not yet renamed into place. + fn close(&mut self) -> std::result::Result { + if !self.buf.is_empty() + && let Some(tx) = &self.tx + { + let _ = tx.send(std::mem::take(&mut self.buf)); + } + self.tx = None; + match self.handle.take().map(JoinHandle::join) { + Some(Ok(Ok(f))) => Ok(f), + Some(Ok(Err(e))) => Err(self.fail(&e.to_string())), + _ => Err(self.fail("the writer stopped")), + } + } +} + +impl Drop for GzThread { + /// On an early stop: end the thread, which drops its file unrenamed. + fn drop(&mut self) { + self.tx = None; + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +struct Child { + name: String, + sex: Option, + father: Option, + mother: Option, + f_seq: bool, + m_seq: bool, + col: usize, + fcol: usize, + mcol: usize, + n: Counts, + de_novo: Vec>, + cands: Vec<(Genes, String, Option<&'static str>)>, + /// WhatsHap: per (chrom, PS) block, sites where the first haplotype + /// holds the father's allele, and sites where it holds the mother's. + blocks: HashMap<(Vec, Vec), [u64; 2]>, + phased_hets: u64, + out: GzThread, +} + +/// The allele with index `i`: REF for 0, else the `i`-th ALT. +fn allele<'a>( + cols: &[&'a [u8]], + i: u32, + path: &str, + line: u64, +) -> std::result::Result<&'a [u8], Die> { + let found = if i == 0 { + Some(cols[3]) + } else { + cols[4].split(|&c| c == b',').nth(i as usize - 1) + }; + found.ok_or_else(|| { + Die::one(format!( + "{path} line {line}: a genotype names allele {i}, the record has fewer" + )) + }) +} + +/// `int(v)` of a FORMAT value, `None` when it is missing or not a number. +fn to_int(v: Option<&[u8]>) -> Option { + let v = v.filter(|v| !v.is_empty() && *v != b".")?; + pycompat::num::parse_int(std::str::from_utf8(v).ok()?).ok() +} + +fn s_int(x: Option) -> String { + x.map_or_else(|| ".".to_owned(), |v| v.to_string()) +} + +fn text(b: &[u8]) -> String { + String::from_utf8_lossy(b).into_owned() +} + +/// What the joint VCF pass needs. +struct Pass<'p> { + path: &'p str, + phased: bool, + par: &'p Par, + cands: &'p HashMap>, +} + +impl Pass<'_> { + fn child( + &self, + ch: &mut Child, + cols: &[&[u8]], + pos: i64, + gi: usize, + format: &[u8], + calls: &mut Calls<'_>, + ) -> std::result::Result<(), Die> { + let (path, line) = (self.path, calls.line); + let chrom = cols[0]; + let (cg, cphased) = calls.gt_phased(ch.col)?; + let Some(cg) = cg else { return Ok(()) }; + let fset = if ch.f_seq { + calls.gt(ch.fcol)?.map(|g| set(&g)) + } else { + None + }; + let mset = if ch.m_seq { + calls.gt(ch.mcol)?.map(|g| set(&g)) + } else { + None + }; + // a sequenced parent with no call here tells nothing + let (f_known, m_known) = (fset.is_some(), mset.is_some()); + if !f_known && !m_known { + return Ok(()); + } + let is_x = chrom == b"chrX"; + let sexchr = (is_x || chrom == b"chrY") && !self.par.contains(&text(chrom), pos); + let mut how = None; + let (mut frm_f, mut frm_m) = (None::, None::); + if sexchr { + let sex = ch.sex.as_deref(); + if !matches!(sex, Some("male" | "female")) || (!is_x && sex == Some("female")) { + ch.n.skipped_sex += 1; + return Ok(()); + } + if sex == Some("male") { + if set(&cg).len() != 1 { + ch.n.skipped_sex += 1; // a heterozygous call on a single X or Y + return Ok(()); + } + let a = cg[0]; + if (is_x && !m_known) || (!is_x && !f_known) { + return Ok(()); + } + let src = if is_x { &mset } else { &fset }; + if src.as_ref().is_some_and(|s| s.contains(&a)) { + how = Some(How::Genotype); + if is_x { + frm_m = Some(a); + } else { + frm_f = Some(a); + } + } else { + how = Some(How::Inconsistent); + } + } + } + let how = match how { + Some(h) => h, + None => { + if cg.len() != 2 { + return Ok(()); + } + let opts = resolve(&cg, fset.as_deref(), mset.as_deref()); + match opts.as_slice() { + [] => How::Inconsistent, + [(f, m)] => { + frm_f = Some(*f); + frm_m = Some(*m); + How::Genotype + } + _ => How::Ambiguous, + } + } + }; + ch.n.sites += 1; + match how { + How::Genotype => { + ch.n.both += u64::from(frm_f.is_some() && frm_m.is_some()); + ch.n.father += u64::from(frm_f.is_some()); + ch.n.mother += u64::from(frm_m.is_some()); + } + How::Ambiguous => ch.n.ambiguous += 1, + How::Inconsistent => ch.n.inconsistent += 1, + } + let cgt = calls.val(ch.col, gi)?.unwrap_or(b"."); + let fgt = if ch.f_seq { + calls.val(ch.fcol, gi)?.unwrap_or(b".") + } else { + b"." + }; + let mgt = if ch.m_seq { + calls.val(ch.mcol, gi)?.unwrap_or(b".") + } else { + b"." + }; + let from = |x: Option| -> std::result::Result<&[u8], Die> { + match x { + Some(i) => allele(cols, i, path, line), + None => Ok(b"."), + } + }; + let (af, am) = (from(frm_f)?, from(frm_m)?); + let buf = &mut ch.out.buf; + for (k, f) in [ + chrom, + cols[1], + cols[3], + cols[4], + cgt, + fgt, + mgt, + af, + am, + how.as_bytes(), + ] + .into_iter() + .enumerate() + { + if k > 0 { + buf.push(b'\t'); + } + buf.extend_from_slice(f); + } + buf.push(b'\n'); + if buf.len() >= CHUNK { + ch.out.flush()?; + } + let cset = set(&cg); + let het = cset.len() == 2 && cg.contains(&0); + // WhatsHap's phase of the child against the genotype origin + if self.phased && cphased && het { + ch.phased_hets += 1; + let ps = vcf::format_index(format, b"PS") + .and_then(|i| calls.val(ch.col, i).ok().flatten()) + .filter(|v| !v.is_empty() && *v != b".") + .unwrap_or(b"."); + if how == How::Genotype + && let Some(f) = frm_f + { + let b = ch.blocks.entry((chrom.to_vec(), ps.to_vec())).or_default(); + b[usize::from(cg[0] != f)] += 1; + } + } + // de novo: an allele the child carries that neither parent has + if how == How::Inconsistent && f_known && m_known { + let (f, m) = ( + fset.as_deref().unwrap_or(&[]), + mset.as_deref().unwrap_or(&[]), + ); + // on a male's single X only the mother's alleles count, on his Y the father's; + // a female's two X come one from each parent, so both parents' alleles count + let seen: Vec = if sexchr && is_x { + if ch.sex.as_deref() == Some("male") { + m.to_vec() + } else { + [f, m].concat() + } + } else if sexchr { + f.to_vec() + } else { + [f, m].concat() + }; + let novel: Vec = cset + .iter() + .copied() + .filter(|a| *a != 0 && !seen.contains(a)) + .collect(); + if !novel.is_empty() { + let row = de_novo_row(ch, cols, format, gi, &novel, &cset, calls, path)?; + ch.de_novo.push(row); + } + } + // compound heterozygote candidates + if het && let Some(table) = self.cands.get(&ch.name) { + let ai = if cg[0] == 0 { cg[1] } else { cg[0] }; + let alt = allele(cols, ai, path, line)?; + if let Some(genes) = table.get(&(chrom.to_vec(), pos, cols[3].to_vec(), alt.to_vec())) { + let first = if cols[3] == alt { + 0 + } else { + cols[4] + .split(|&c| c == b',') + .position(|x| x == alt) + .map_or(ai, |k| k as u32 + 1) + }; + let origin = if frm_f == Some(first) { + Some("father") + } else if frm_m == Some(first) { + Some("mother") + } else { + None + }; + ch.cands.push(( + genes.clone(), + format!("{}:{pos}:{}>{}", text(chrom), text(cols[3]), text(alt)), + origin, + )); + } + } + Ok(()) + } +} + +#[allow(clippy::too_many_arguments)] +fn de_novo_row( + ch: &Child, + cols: &[&[u8]], + format: &[u8], + gi: usize, + novel: &[u32], + cset: &[u32], + calls: &mut Calls<'_>, + path: &str, +) -> std::result::Result, Die> { + let line = calls.line; + fn field<'a>( + calls: &mut Calls<'a>, + format: &[u8], + col: usize, + key: &[u8], + ) -> std::result::Result, Die> { + match vcf::format_index(format, key) { + Some(i) => Ok(calls.val(col, i)?.filter(|v| !v.is_empty() && *v != b".")), + None => Ok(None), + } + } + let three = [ch.col, ch.fcol, ch.mcol]; + let mut dp = [None; 3]; + let mut gq = [None; 3]; + for (k, &c) in three.iter().enumerate() { + dp[k] = to_int(field(calls, format, c, b"DP")?); + gq[k] = to_int(field(calls, format, c, b"GQ")?); + } + let mut ab = None; + if let Some(ad) = field(calls, format, ch.col, b"AD")? { + let n: Vec = ad + .split(|&c| c == b',') + .map(|x| to_int(Some(x)).unwrap_or(0)) + .collect(); + let tot: i128 = n.iter().sum(); + if tot != 0 { + ab = n.get(novel[0] as usize).map(|&v| v as f64 / tot as f64); + } + } + let het = cset.len() == 2; + let mut good = gq.iter().all(|x| x.is_some_and(|v| v >= MIN_GQ)) + && dp.iter().all(|x| x.is_some_and(|v| v >= MIN_DP)); + if het && ab.is_none_or(|ab| !(HET_AB.0 <= ab && ab <= HET_AB.1)) { + good = false; + } + let mut alts = Vec::new(); + for &i in novel { + alts.push(text(allele(cols, i, path, line)?)); + } + let gt_of = |calls: &mut Calls<'_>, col: usize| -> std::result::Result { + Ok(calls.val(col, gi)?.map_or_else(|| ".".to_owned(), text)) + }; + Ok(vec![ + ch.name.clone(), + text(cols[0]), + text(cols[1]), + text(cols[3]), + alts.join(","), + gt_of(calls, ch.col)?, + gt_of(calls, ch.fcol)?, + gt_of(calls, ch.mcol)?, + s_int(dp[0]), + s_int(gq[0]), + ab.map_or_else(|| ".".to_owned(), |v| fixed(v, 2)), + s_int(dp[1]), + s_int(gq[1]), + s_int(dp[2]), + s_int(gq[2]), + if good { "pass" } else { "low" }.to_owned(), + ]) +} + +fn pct(k: f64, n: f64) -> String { + if n != 0.0 { + format!("{}%", fixed(100.0 * k / n, 1)) + } else { + "-".to_owned() + } +} + +pub fn run(a: &Args) -> Result<()> { + let members_path = a.one("--members").unwrap_or_default(); + let members = read_members(members_path)?; + let rel_path = a.one("--relationships").unwrap_or_default(); + let rels = read_table(rel_path, false)?; + let par = Par::read(a.one("--par-bed"))?; + let out_dir = PathBuf::from(a.one("--out-dir").unwrap_or_default()); + std::fs::create_dir_all(&out_dir) + .map_err(|e| Die::one(format!("cannot create {}: {e}", out_dir.display())))?; + let seq = segments::sequenced(&members); + let is_seq = |p: &Option| { + p.as_deref() + .and_then(|p| members.get(p)) + .is_some_and(|m| m.sequenced) + }; + let mut cands: HashMap> = HashMap::new(); + for kv in a.many("--candidates") { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + if members.contains_key(k) { + cands.insert(k.to_owned(), candidate_genes(v)?); + } + } + let vcf_path = a.one("--vcf").unwrap_or_default(); + let phased_path = a.one("--phased"); + let path = phased_path.unwrap_or(vcf_path); + + // With --phased the map reads --vcf on its own thread. + let seg_thread = phased_path.map(|_| { + let (m, v, p) = (members.clone(), vcf_path.to_owned(), par.clone()); + thread::spawn(move || segments::collect(&m, &v, p)) + }); + + let mut children = Vec::new(); + for s in &seq { + let m = &members[s]; + if !(is_seq(&m.father) || is_seq(&m.mother)) { + continue; + } + children.push(Child { + name: s.clone(), + sex: m.sex.clone(), + father: m.father.clone(), + mother: m.mother.clone(), + f_seq: is_seq(&m.father), + m_seq: is_seq(&m.mother), + col: 0, + fcol: 0, + mcol: 0, + n: Counts::default(), + de_novo: Vec::new(), + cands: Vec::new(), + blocks: HashMap::new(), + phased_hets: 0, + out: GzThread::start(out_dir.join(format!("{s}_inheritance.tsv.gz")))?, + }); + } + for ch in &mut children { + ch.out.buf.extend_from_slice( + b"chrom\tpos\tref\talt\tchild_gt\tfather_gt\tmother_gt\tfrom_father\tfrom_mother\thow\n", + ); + } + + let r = og_io::bgzf::open(path, og_io::bgzf::workers()).map_err(|e| Die::io(path, &e))?; + let mut lines = og_vcf::Lines::new(r); + let mut meta: Vec> = Vec::new(); + let names = segments::read_header(&mut lines, path, |l| { + if l.starts_with(b"##contig") { + meta.push(l.to_vec()); + } + })?; + let Some(names) = names else { + return Err(Die::one(format!("{path} has no #CHROM line")).into()); + }; + let idx = vcf::sample_index(&names); + let lost: Vec<&str> = seq + .iter() + .filter(|s| !idx.contains_key(*s)) + .map(String::as_str) + .collect(); + if !lost.is_empty() { + return Err(Die::one(format!( + "{path} has no column for {} (its columns: {})", + lost.join(", "), + names.join(", ") + )) + .into()); + } + for ch in &mut children { + ch.col = idx[&ch.name]; + ch.fcol = ch + .father + .as_ref() + .and_then(|f| idx.get(f)) + .copied() + .unwrap_or(0); + ch.mcol = ch + .mother + .as_ref() + .and_then(|m| idx.get(m)) + .copied() + .unwrap_or(0); + } + // Without --phased the map is built in this same pass. + let mut inline: Option = phased_path.is_none().then(|| { + let mut c = Collector::new(&members, &idx, par.clone()); + for l in &meta { + c.header(l); + } + c + }); + let pass = Pass { + path, + phased: phased_path.is_some(), + par: &par, + cands: &cands, + }; + let mut n = lines.number(); + while let Some(line) = lines.next_line().map_err(|e| Die::io(path, &e))? { + n += 1; + let cols = vcf::columns(line); + if cols.len() < 2 { + return Err(Die::one(format!("{path} line {n}: not a VCF record")).into()); + } + let pos = vcf::pos(cols[1], path, n)?; + let primary = is_primary(cols[0]); + if primary && cols.len() < 9 { + return Err(Die::one(format!("{path} line {n}: no FORMAT column")).into()); + } + let format = cols.get(8).copied().unwrap_or(b""); + let gi = if primary { + vcf::format_index(format, b"GT") + } else { + None + }; + let mut calls = Calls::new(&cols, gi.unwrap_or(0), path, n); + if let Some(c) = &mut inline { + c.record(path, pos, &mut calls, &cols, gi)?; + } + let Some(gi) = gi else { continue }; + for ch in &mut children { + pass.child(ch, &cols, pos, gi, format, &mut calls)?; + } + } + + let map: Map = match (inline, seg_thread) { + (Some(c), _) => c.finish(&members), + (None, Some(h)) => { + let (c, _) = h + .join() + .map_err(|_| Die::one("the chromosome map stopped"))??; + c.finish(&members) + } + (None, None) => unreachable!("one of the two builds the map"), + }; + + // Every output is complete: write them. + let mut dn = String::from( + "child\tchrom\tpos\tref\talt\tchild_gt\tfather_gt\tmother_gt\tchild_dp\tchild_gq\tchild_ab\tfather_dp\tfather_gq\tmother_dp\tmother_gq\tquality\n", + ); + for ch in &children { + for row in &ch.de_novo { + dn.push_str(&row.join("\t")); + dn.push('\n'); + } + } + write_file(&out_dir.join("de_novo.tsv"), dn.as_bytes())?; + let mut comp: Vec<[String; 7]> = Vec::new(); + for ch in &children { + let mut by_gene: IndexMap<&str, Vec<(&str, Option<&str>)>> = IndexMap::new(); + for (genes, var, origin) in &ch.cands { + for g in genes.iter() { + by_gene.entry(g).or_default().push((var, *origin)); + } + } + by_gene.sort_keys(); + for (g, list) in &by_gene { + for (i, (v1, o1)) in list.iter().enumerate() { + for (v2, o2) in &list[i + 1..] { + let phase = match (o1, o2) { + (Some(a), Some(b)) if a == b => "in cis", + (Some(_), Some(_)) => "in trans", + _ => "unknown", + }; + comp.push([ + ch.name.clone(), + (*g).to_owned(), + (*v1).to_owned(), + (*v2).to_owned(), + o1.unwrap_or(".").to_owned(), + o2.unwrap_or(".").to_owned(), + phase.to_owned(), + ]); + } + } + } + } + let mut ct = String::from("child\tgene\tvariant_a\tvariant_b\ta_from\tb_from\tphase\n"); + for row in &comp { + ct.push_str(&row.join("\t")); + ct.push('\n'); + } + write_file(&out_dir.join("compound_het.tsv"), ct.as_bytes())?; + for ch in &mut children { + let f = ch.out.close()?; + f.commit().map_err(|e| ch.out.fail(&e.to_string()))?; + } + map.write(&out_dir)?; + let seg = map.markdown(&members); + + let md = markdown( + &members, + &rels, + rel_path, + &children, + &comp, + !cands.is_empty(), + phased_path.is_some(), + seg, + )?; + write_file(&out_dir.join("family_report.md"), md.as_bytes())?; + print_out(&md); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn markdown( + members: &Members, + rels: &[IndexMap], + rel_path: &str, + children: &[Child], + comp: &[[String; 7]], + have_cands: bool, + phased: bool, + seg: Vec, +) -> std::result::Result { + let mut l: Vec = vec!["# Family analysis".into(), String::new()]; + let mut seq: Vec<&str> = members + .iter() + .filter(|(_, m)| m.sequenced) + .map(|(s, _)| s.as_str()) + .collect(); + seq.sort_unstable(); + let mut others: Vec<&str> = members + .iter() + .filter(|(_, m)| !m.sequenced) + .map(|(s, _)| s.as_str()) + .collect(); + others.sort_unstable(); + l.push(format!( + "Sequenced: {}. In the pedigree, not sequenced: {}.", + seq.join(", "), + if others.is_empty() { + "none".to_owned() + } else { + others.join(", ") + } + )); + l.extend([ + String::new(), + "## Relationships".into(), + String::new(), + "Relatedness is twice the kinship coefficient: about the share of the genome the two have \ + inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, \ + 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the \ + expected value." + .into(), + String::new(), + "| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict |".into(), + "|---|---|---|---|---|---|---|".into(), + ]); + for r in rels { + let c = |k: &str| cell(r, k, rel_path); + let rate = c("ibs0_rate")?; + let rate = pycompat::num::parse_float(rate) + .map_err(|_| Die::one(format!("{rel_path}: ibs0_rate '{rate}' is not a number")))?; + l.push(format!( + "| {} and {} | {} | {} | {} | {} | {} | {}: {} |", + c("sample_a")?, + c("sample_b")?, + c("declared")?, + c("expected_relatedness")?, + c("relatedness")?, + pct(rate, 1.0), + c("sites")?, + c("verdict")?, + c("reason")? + )); + } + for ch in children { + let n = &ch.n; + let sites = n.sites as f64; + l.extend([String::new(), format!("## {}", ch.name), String::new()]); + let who = |p: &Option, s: bool| match p { + Some(p) => format!("{p} ({})", if s { "sequenced" } else { "not sequenced" }), + None => "not named".to_owned(), + }; + l.push(format!( + "Father: {}. Mother: {}.", + who(&ch.father, ch.f_seq), + who(&ch.mother, ch.m_seq) + )); + let file = Path::new(&ch.out.path) + .file_name() + .map(|f| f.to_string_lossy().into_owned()) + .unwrap_or_default(); + l.extend([ + String::new(), + format!( + "Sites of the joint VCF where {} has a genotype and a sequenced parent does too: {}. `{file}` lists them.", + ch.name, n.sites + ), + String::new(), + ]); + l.push(format!( + "- the allele from the father is known at {} ({})", + n.father, + pct(n.father as f64, sites) + )); + l.push(format!( + "- the allele from the mother is known at {} ({})", + n.mother, + pct(n.mother as f64, sites) + )); + l.push(format!( + "- both parents heterozygous like the child, so no origin from the genotypes: {} ({})", + n.ambiguous, + pct(n.ambiguous as f64, sites) + )); + l.push(format!( + "- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): {} ({})", + n.inconsistent, + pct(n.inconsistent as f64, sites) + )); + if n.skipped_sex > 0 { + l.push(format!( + "- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a \ + heterozygous call on a single X or Y): {}", + n.skipped_sex + )); + } + for (who, s, k) in [ + ("father", ch.f_seq, n.father), + ("mother", ch.m_seq, n.mother), + ] { + if !s { + l.extend([ + String::new(), + format!( + "The {who} is not sequenced. At {k} of these sites ({}) the genotypes give the allele \ + the {who} passed on to {}: that is the half of the {who}'s genome {} carries. The other \ + half was not passed on and is not seen here.", + pct(k as f64, sites), + ch.name, + ch.name + ), + ]); + } + } + if ch.f_seq && ch.m_seq { + let good = ch + .de_novo + .iter() + .filter(|r| r.last().is_some_and(|q| q == "pass")) + .count(); + l.extend([ + String::new(), + format!( + "De novo candidates (an allele neither parent has): {}, {good} of them with GQ >= {MIN_GQ} and \ + depth >= {MIN_DP} in all three and, for a heterozygous call, an allele balance of {} to {} \ + (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that \ + matters needs confirming by another method.", + ch.de_novo.len(), + fixed(HET_AB.0, 2), + fixed(HET_AB.1, 2) + ), + ]); + } else { + l.extend([ + String::new(), + "No de novo candidates are called: that needs both parents sequenced.".into(), + ]); + } + if phased { + let votes: u64 = ch.blocks.values().map(|b| b[0].max(b[1])).sum(); + let tot: u64 = ch.blocks.values().map(|b| b[0] + b[1]).sum(); + l.extend([ + String::new(), + format!( + "WhatsHap phased {} heterozygous sites of {} by inheritance. At {tot} of them the \ + genotypes also give the origin, and the phase agrees with it at {votes} ({}).", + ch.phased_hets, + ch.name, + pct(votes as f64, tot as f64) + ), + ]); + } + } + l.extend([ + String::new(), + "## Compound heterozygotes".into(), + String::new(), + ]); + if !have_cands { + l.push( + "No candidate variants were given (the clinical filter, step 23, did not run for a child), \ + so no pair is reported." + .into(), + ); + } else { + let t = comp.iter().filter(|r| r[6] == "in trans").count(); + let c = comp.iter().filter(|r| r[6] == "in cis").count(); + l.push(format!( + "Pairs of the clinical filter's candidate variants a child carries heterozygous in one gene: \ + {}; in trans (one from each parent, both copies of the gene hit) {t}, in cis (both from one \ + parent) {c}, origin unknown {} (`compound_het.tsv`).", + comp.len(), + comp.len() - t - c + )); + } + l.extend(seg); + let mut s = l.join("\n"); + s.push('\n'); + Ok(s) +} diff --git a/rust/crates/og-steps/src/family/segments.rs b/rust/crates/og-steps/src/family/segments.rs new file mode 100644 index 000000000..593a00281 --- /dev/null +++ b/rust/crates/og-steps/src/family/segments.rs @@ -0,0 +1,1110 @@ +//! The chromosome map of step 38: for each sequenced child with a +//! sequenced parent, the stretches of each chromosome the child got from +//! that parent, and, with a sequenced grandparent, which of the parent's +//! two copies each stretch is. +//! +//! A site tells something when the parent is heterozygous and the +//! genotypes give the allele the parent passed on. Stretches are smoothed: +//! a run of sites pointing to the other copy is kept only with +//! [`MIN_SITES`] sites over [`MIN_BP`] bases; a shorter run (a genotype +//! error, a gene conversion) is absorbed and counted as discordant. A +//! stretch with no site for [`MAX_GAP`] bases is split there. Autosomes, +//! and chrX outside the pseudoautosomal regions from the mother only. +//! +//! Writes, per child, `_segments.tsv`, `_dnapainter.csv` +//! (DNA Painter's import format, GRCh38 positions) and +//! `_segments.svg`. + +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashMap}; +use std::path::Path; + +use indexmap::IndexMap; +use og_pedigree::{Die, Members, Par, read_members}; +use pycompat::fmt::fixed; + +use super::args::{Args, Kind, Spec}; +use super::vcf::{self, Alleles, Calls, resolve, set}; +use super::{Result, print_out, write_file}; + +pub const SPEC: Spec = Spec { + prog: "og family segments", + usage: "usage: og family segments [-h] --members MEMBERS --vcf VCF [--par-bed PAR_BED] --out-dir OUT_DIR\n", + options: &[ + ("--members", Kind::One), + ("--vcf", Kind::One), + ("--par-bed", Kind::One), + ("--out-dir", Kind::One), + ], + required: &["--members", "--vcf", "--out-dir"], +}; + +/// A run of sites pointing to the other grandparent's copy needs this +/// much support to be a segment of its own. +pub const MIN_SITES: usize = 20; +pub const MIN_BP: i64 = 500_000; +/// No site for this many bases: the stretch is split. +pub const MAX_GAP: i64 = 5_000_000; +/// chr1 to chr22, then chrX: the map's chromosomes, in its order. +pub const N_CHROMS: usize = 23; +const X: usize = 22; + +/// The index of a map chromosome. +pub fn chrom_index(c: &[u8]) -> Option { + let n = c.strip_prefix(b"chr")?; + if n == b"X" { + return Some(X); + } + if n.is_empty() || n.len() > 2 || n[0] == b'0' || !n.iter().all(u8::is_ascii_digit) { + return None; + } + let v: usize = std::str::from_utf8(n).ok()?.parse().ok()?; + (1..=22).contains(&v).then(|| v - 1) +} + +fn chrom_name(i: usize) -> String { + if i == X { + "chrX".to_owned() + } else { + format!("chr{}", i + 1) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum Side { + Paternal, + Maternal, +} + +impl Side { + fn as_str(self) -> &'static str { + match self { + Side::Paternal => "paternal", + Side::Maternal => "maternal", + } + } + fn other(self) -> Side { + match self { + Side::Paternal => Side::Maternal, + Side::Maternal => Side::Paternal, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Basis { + Grandparents, + Parent, + OtherCopy, +} + +impl Basis { + fn as_str(self) -> &'static str { + match self { + Basis::Grandparents => "parent and grandparents", + Basis::Parent => "parent", + Basis::OtherCopy => "other copy", + } + } +} + +/// One child and one sequenced parent: the parent's informative sites. +struct Link { + side: Side, + parent: String, + pcol: usize, + gf: Option, + gm: Option, + gfcol: Option, + gmcol: Option, + /// Per chromosome, the positions where the allele passed on is known. + sites: Vec>, + /// Per chromosome, (position, 0 from the grandfather | 1 from the grandmother). + hap: Vec>, +} + +impl Link { + fn grandparents(&self) -> bool { + self.gfcol.is_some() || self.gmcol.is_some() + } + + fn ancestor(&self, label: u8) -> String { + let name = if label == 0 { &self.gf } else { &self.gm }; + name.clone().unwrap_or_else(|| { + format!( + "{}'s {}", + self.parent, + if label == 0 { "father" } else { "mother" } + ) + }) + } +} + +struct ChildLinks { + name: String, + male: bool, + col: usize, + /// The child's father's column, when sequenced (chrX of a daughter). + fcol: Option, + /// The other parent's column per link side, when sequenced. + other: [Option; 2], + links: Vec, +} + +/// The chromosome map being built from the joint VCF, record by record. +pub struct Collector { + children: Vec, + par: Par, + sorted: vcf::Sorted, + /// `##contig` lengths of the VCF's header. + pub lengths: HashMap, +} + +/// Whether a sequenced child has a sequenced parent: without one there +/// is no map and the VCF need not be read. +pub fn has_links(members: &Members) -> bool { + let seq = |p: &Option| { + p.as_deref() + .and_then(|p| members.get(p)) + .is_some_and(|m| m.sequenced) + }; + members + .values() + .any(|m| m.sequenced && (seq(&m.father) || seq(&m.mother))) +} + +/// The sequenced members, sorted. +pub fn sequenced(members: &Members) -> Vec { + let mut seq: Vec = members + .iter() + .filter(|(_, m)| m.sequenced) + .map(|(s, _)| s.clone()) + .collect(); + seq.sort(); + seq +} + +/// `contig_lengths`: the `##contig=` lines of a header. +pub fn contig_length(line: &[u8]) -> Option<(String, i64)> { + let rest = line.strip_prefix(b"##contig=') + .unwrap_or(rest.len()); + if end == 0 || rest.get(end) != Some(&b',') { + return None; + } + let id = String::from_utf8_lossy(&rest[..end]).into_owned(); + let mut tail = &rest[end + 1..]; + while let Some(k) = memchr::memmem::find(tail, b"length=") { + let after = &tail[k + 7..]; + let n = after.iter().take_while(|c| c.is_ascii_digit()).count(); + if n > 0 { + let v = std::str::from_utf8(&after[..n]).ok()?.parse().ok()?; + return Some((id, v)); + } + tail = &tail[k + 1..]; + } + None +} + +impl Collector { + /// The links of every sequenced child and sequenced parent. `idx` maps + /// a sample to its column. + pub fn new(members: &Members, idx: &HashMap, par: Par) -> Collector { + let col = |s: &Option| -> Option { + let s = s.as_deref()?; + members.get(s).filter(|m| m.sequenced)?; + idx.get(s).copied() + }; + let mut children = Vec::new(); + for s in sequenced(members) { + let m = &members[&s]; + let mut links = Vec::new(); + for (side, p) in [(Side::Paternal, &m.father), (Side::Maternal, &m.mother)] { + let Some(pcol) = col(p) else { continue }; + let pm = &members[p.as_deref().unwrap_or_default()]; + links.push(Link { + side, + parent: p.clone().unwrap_or_default(), + pcol, + gf: pm.father.clone(), + gm: pm.mother.clone(), + gfcol: col(&pm.father), + gmcol: col(&pm.mother), + sites: vec![Vec::new(); N_CHROMS], + hap: vec![Vec::new(); N_CHROMS], + }); + } + if links.is_empty() { + continue; + } + children.push(ChildLinks { + male: m.sex.as_deref() == Some("male"), + col: idx[&s], + fcol: col(&m.father), + other: [col(&m.mother), col(&m.father)], + links, + name: s, + }); + } + Collector { + children, + par, + sorted: vcf::Sorted::default(), + lengths: HashMap::new(), + } + } + + /// Whether any child has a link: without one the VCF need not be read. + pub fn any(&self) -> bool { + !self.children.is_empty() + } + + /// A `##` header line. + pub fn header(&mut self, line: &[u8]) { + if let Some((id, n)) = contig_length(line) { + self.lengths.insert(id, n); + } + } + + /// One record. `cols` are its columns, `pos` its parsed POS. + pub fn record( + &mut self, + path: &str, + pos: i64, + calls: &mut Calls<'_>, + cols: &[&[u8]], + gi: Option, + ) -> std::result::Result<(), Die> { + if !self.any() { + return Ok(()); + } + self.sorted.next(path, cols[0], pos)?; + let Some(ci) = chrom_index(cols[0]) else { + return Ok(()); + }; + // Without a PAR file the pseudoautosomal sites cannot be told apart, + // so chrX is left out. + if ci == X && (self.par.is_empty() || self.par.contains("chrX", pos)) { + return Ok(()); + } + if gi.is_none() { + return Ok(()); + } + for ch in &mut self.children { + let Some(cg) = calls.gt(ch.col)? else { + continue; + }; + for ln in &mut ch.links { + let Some(pg) = calls.gt(ln.pcol)? else { + continue; + }; + if pg.len() != 2 || pg[0] == pg[1] { + continue; // only a heterozygous parent tells which copy was passed on + } + let t = if ci == X { + if ln.side != Side::Maternal { + continue; + } + if ch.male { + if set(&cg).len() != 1 { + continue; + } + Some(cg[0]) + } else { + let fg = match ch.fcol { + Some(c) => calls.gt(c)?, + None => None, + }; + let Some(fg) = fg else { continue }; + if set(&fg).len() != 1 || cg.len() != 2 { + continue; + } + passed_on(&cg, Some(&pg), Some(&[fg[0], fg[0]]), false) + } + } else { + if cg.len() != 2 { + continue; + } + let k = usize::from(ln.side == Side::Maternal); + let other = match ch.other[k] { + Some(c) => calls.gt(c)?, + None => None, + }; + passed_on(&cg, Some(&pg), other.as_deref(), ln.side == Side::Paternal) + }; + let Some(t) = t.filter(|t| pg.contains(t)) else { + continue; + }; + ln.sites[ci].push(pos); + if !ln.grandparents() { + continue; + } + let gt_of = |c: Option, + calls: &mut Calls<'_>| + -> std::result::Result, Die> { + match c { + Some(c) => calls.gt(c), + None => Ok(None), + } + }; + // which of the parent's copies holds t: the parent's own origin here + let label = if ci == X { + // the mother's X: one from her father (his only X), one from her mother + let gfg = gt_of(ln.gfcol, calls)?; + let gmg = gt_of(ln.gmcol, calls)?; + if gfg.as_ref().is_some_and(|g| set(g).len() != 1) { + continue; + } + let pair = gfg.as_ref().map(|g| [g[0], g[0]]); + let Some(po) = passed_on( + &pg, + gmg.as_deref(), + pair.as_ref().map(|p| p.as_slice()), + false, + ) else { + continue; + }; + u8::from(t == po) + } else { + let gf = gt_of(ln.gfcol, calls)?.map(|g| set(&g)); + let gm = gt_of(ln.gmcol, calls)?.map(|g| set(&g)); + let opts = resolve(&pg, gf.as_deref(), gm.as_deref()); + let Some(from_gf) = unique(opts.iter().map(|o| o.0)) else { + continue; + }; + u8::from(t != from_gf) + }; + ln.hap[ci].push((pos, label)); + } + } + Ok(()) + } +} + +/// The one value of an iterator, or `None` when it has none or several. +fn unique(mut it: impl Iterator) -> Option { + let first = it.next()?; + it.all(|v| v == first).then_some(first) +} + +/// The allele the parent passed on, when the genotypes give one. +fn passed_on( + child: &[u32], + parent: Option<&[u32]>, + other: Option<&[u32]>, + parent_is_father: bool, +) -> Option { + let ps = parent.map(set); + let os = other.map(set); + let (f, m) = if parent_is_father { (ps, os) } else { (os, ps) }; + let opts = resolve(child, f.as_deref(), m.as_deref()); + unique( + opts.iter() + .map(|o| if parent_is_father { o.0 } else { o.1 }), + ) +} + +/// Splits sorted positions where no site falls for [`MAX_GAP`] bases: +/// the index ranges of the blocks. +fn blocks(positions: impl Iterator) -> Vec> { + let mut out = Vec::new(); + let (mut start, mut len) = (0, 0); + let mut prev: Option = None; + for (i, p) in positions.enumerate() { + if let Some(q) = prev + && p - q > MAX_GAP + { + out.push(start..i); + start = i; + } + prev = Some(p); + len = i + 1; + } + if len > start { + out.push(start..len); + } + out +} + +/// A run of one block: `[label, start, end, agree, discordant]`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Run { + pub label: u8, + pub start: i64, + pub end: i64, + pub agree: usize, + pub discordant: usize, +} + +fn weak(r: &Run) -> bool { + r.agree < MIN_SITES || r.end - r.start + 1 < MIN_BP +} + +fn runs_of(sites: &[(i64, u8)]) -> Vec { + let mut runs: Vec = Vec::new(); + for &(pos, label) in sites { + match runs.last_mut() { + Some(r) if r.label == label => { + r.end = pos; + r.agree += 1; + } + _ => runs.push(Run { + label, + start: pos, + end: pos, + agree: 1, + discordant: 0, + }), + } + } + runs +} + +/// Merges the weak run `i` with its neighbours `lo..=hi`: the merged run +/// takes the label of `runs[lo]` (or `runs[hi]` when `lo` is `i`), its +/// sites now point to the other copy, and the ones it had absorbed agree. +fn merge(lo: &Run, mid: &[&Run], hi: &Run, i_is: impl Fn(usize) -> bool, keep: u8) -> Run { + let mut m = Run { + label: keep, + start: lo.start, + end: hi.end, + agree: 0, + discordant: 0, + }; + for (k, r) in mid.iter().enumerate() { + if i_is(k) { + m.agree += r.discordant; + m.discordant += r.agree; + } else { + m.agree += r.agree; + m.discordant += r.discordant; + } + } + m +} + +/// The tie-break of the weakest run: fewest sites, then shortest span, +/// then the first in the block (`order` grows along it). +type Key = (usize, i64, usize); + +fn key(r: &Run, order: usize) -> Key { + (r.agree, r.end - r.start, order) +} + +/// The smoothed runs of one block of (position, label): repeatedly absorb +/// the weakest weak run into its neighbours. A min-heap with lazy +/// deletion over a linked list of runs, so O(R log R); the same merges, in +/// the same order, as taking the minimum of `(agree, end - start, index)` +/// over the list every time. +pub fn smooth(sites: &[(i64, u8)]) -> Vec { + let runs = runs_of(sites); + let n = runs.len(); + // Node i: its run, its list neighbours, its order (index of its first + // original run), and whether it is still in the list. + let mut node: Vec = runs; + let mut prev: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); + let mut next: Vec> = (0..n).map(|i| (i + 1 < n).then_some(i + 1)).collect(); + let mut order: Vec = (0..n).collect(); + let mut alive: Vec = vec![true; n]; + let mut count = n; + let mut heap: BinaryHeap> = BinaryHeap::new(); + for (i, r) in node.iter().enumerate() { + if weak(r) { + heap.push(Reverse((key(r, i), i))); + } + } + while count > 1 { + let Some(Reverse((_, i))) = heap.pop() else { + break; + }; + if !alive[i] { + continue; + } + let lo = prev[i].unwrap_or(i); + let hi = next[i].unwrap_or(i); + let keep = if lo != i { + node[lo].label + } else { + node[hi].label + }; + let ids: Vec = [lo, i, hi].into_iter().fold(Vec::new(), |mut v, x| { + if !v.contains(&x) { + v.push(x); + } + v + }); + let mid: Vec<&Run> = ids.iter().map(|&x| &node[x]).collect(); + let merged = merge(&node[lo], &mid, &node[hi], |k| ids[k] == i, keep); + let id = node.len(); + node.push(merged); + prev.push(prev[lo]); + next.push(next[hi]); + order.push(order[lo]); + alive.push(true); + if let Some(p) = prev[lo] { + next[p] = Some(id); + } + if let Some(q) = next[hi] { + prev[q] = Some(id); + } + for &x in &ids { + alive[x] = false; + } + count -= ids.len() - 1; + if weak(&node[id]) { + heap.push(Reverse((key(&node[id], order[id]), id))); + } + } + // Walk the list from its head. + let mut out = Vec::new(); + let mut cur = (0..node.len()).find(|&x| alive[x] && prev[x].is_none()); + while let Some(x) = cur { + if node[x].agree >= MIN_SITES { + out.push(node[x]); + } + cur = next[x]; + } + out +} + +/// One row of `_segments.tsv`. +#[derive(Clone, Debug)] +struct Row { + side: Side, + parent: String, + chrom: usize, + start: i64, + end: i64, + sites: usize, + discordant: usize, + ancestor: String, + basis: Basis, +} + +impl Row { + fn length(&self) -> i64 { + self.end - self.start + 1 + } +} + +/// The map: per child (sorted), its rows in chromosome order, father's side first. +pub struct Map { + rows: Vec<(String, Vec)>, + /// `##contig` lengths of the VCF the map was read from. + lengths: HashMap, +} + +impl Collector { + /// The rows of every child, from the sites collected. + pub fn finish(self, members: &Members) -> Map { + let lengths = self.lengths; + let mut out = Vec::new(); + for ch in self.children { + let mut rows = Vec::new(); + for ln in &ch.links { + let total: usize = ln.hap.iter().map(Vec::len).sum(); + let hap = ln.grandparents() && total >= MIN_SITES; + for c in 0..N_CHROMS { + if ln.sites[c].is_empty() { + continue; + } + if hap && !ln.hap[c].is_empty() { + let sites = &ln.hap[c]; + for blk in blocks(sites.iter().map(|s| s.0)) { + for r in smooth(&sites[blk]) { + rows.push(Row { + side: ln.side, + parent: ln.parent.clone(), + chrom: c, + start: r.start, + end: r.end, + sites: r.agree, + discordant: r.discordant, + ancestor: ln.ancestor(r.label), + basis: Basis::Grandparents, + }); + } + } + } else { + let ps = &ln.sites[c]; + for blk in blocks(ps.iter().copied()) { + if blk.len() >= MIN_SITES { + rows.push(Row { + side: ln.side, + parent: ln.parent.clone(), + chrom: c, + start: ps[blk.start], + end: ps[blk.end - 1], + sites: blk.len(), + discordant: 0, + ancestor: ".".to_owned(), + basis: Basis::Parent, + }); + } + } + } + } + } + // One parent sequenced: the other copy over the same stretches. + if let [ln] = ch.links.as_slice() { + let other_side = ln.side.other(); + let m = &members[&ch.name]; + let other = match ln.side { + Side::Paternal => &m.mother, + Side::Maternal => &m.father, + }; + for c in 0..N_CHROMS { + // the X a child got from the father is not seen from the mother's sites + if c == X || ln.sites[c].is_empty() { + continue; + } + let ps = &ln.sites[c]; + for blk in blocks(ps.iter().copied()) { + if blk.len() >= MIN_SITES { + rows.push(Row { + side: other_side, + parent: other.clone().unwrap_or_else(|| ".".to_owned()), + chrom: c, + start: ps[blk.start], + end: ps[blk.end - 1], + sites: blk.len(), + discordant: 0, + ancestor: ".".to_owned(), + basis: Basis::OtherCopy, + }); + } + } + } + } + rows.sort_by_key(|r| (r.chrom, r.side != Side::Paternal, r.start)); + out.push((ch.name, rows)); + } + Map { rows: out, lengths } + } +} + +const TSV: &str = + "child\tside\tparent\tchrom\tstart\tend\tlength_bp\tsites\tdiscordant\tancestor\tbasis\n"; +const DNAPAINTER: [&str; 12] = [ + "Chr", + "Start", + "End", + "cM", + "SNPs", + "Match", + "Confidence", + "Group", + "Side", + "Notes", + "Colour", + "Ancestor", +]; +const OTHER: &str = "#6b6b6b"; + +fn colours(side: Side) -> [&'static str; 2] { + match side { + Side::Paternal => ["#4f8fd6", "#9cc3ef"], + Side::Maternal => ["#d65f8f", "#efa3c0"], + } +} + +fn side_only(side: Side) -> &'static str { + match side { + Side::Paternal => "#3b6fa8", + Side::Maternal => "#a84870", + } +} + +fn esc(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +/// `x * scale` as `%.1f`, with `x` an int (Python: int times float). +fn f1(x: f64) -> String { + fixed(x, 1) +} + +fn svg(child: &str, rows: &[Row], lengths: &HashMap) -> String { + let chroms: Vec = (0..N_CHROMS) + .filter(|&c| lengths.contains_key(&chrom_name(c)) || rows.iter().any(|r| r.chrom == c)) + .collect(); + let longest = chroms + .iter() + .map(|&c| lengths.get(&chrom_name(c)).copied().unwrap_or(0)) + .chain(rows.iter().map(|r| r.end)) + .chain([1]) + .max() + .unwrap_or(1); + let (width, left, bar, row_h) = (900i64, 60i64, 9i64, 28i64); + let scale = (width - left - 20) as f64 / longest as f64; + let mut colour: IndexMap<(Side, String), &'static str> = IndexMap::new(); + for r in rows { + if r.basis == Basis::OtherCopy { + colour.insert((r.side, r.ancestor.clone()), OTHER); + } else if r.ancestor == "." { + colour.insert((r.side, ".".to_owned()), side_only(r.side)); + } else { + let seen = colour + .keys() + .filter(|k| k.0 == r.side && k.1 != ".") + .count(); + colour + .entry((r.side, r.ancestor.clone())) + .or_insert(colours(r.side)[seen % 2]); + } + } + let height = 50 + row_h * chroms.len() as i64 + 20 * (colour.len() as i64 + 1); + let mut l = vec![ + format!( + "" + ), + "".to_owned(), + format!( + "{}: which copy of each chromosome came from whom (GRCh38)", + esc(child) + ), + ]; + for (i, &c) in chroms.iter().enumerate() { + let y = 40 + i as i64 * row_h; + let name = chrom_name(c); + l.push(format!( + "{}", + y + bar + 3, + esc(&name[3..]) + )); + let n = lengths.get(&name).copied().unwrap_or_else(|| { + rows.iter() + .filter(|r| r.chrom == c) + .map(|r| r.end) + .chain([1]) + .max() + .unwrap_or(1) + }); + for k in 0..2 { + l.push(format!( + "", + y + k * (bar + 1), + f1(n as f64 * scale) + )); + } + for r in rows.iter().filter(|r| r.chrom == c) { + let k = i64::from(r.side != Side::Paternal); + let w = ((r.end - r.start) as f64 * scale).max(1.0); + let who = if r.ancestor != "." { + &r.ancestor + } else { + &r.parent + }; + l.push(format!( + "{} {}:{}-{}, {}, {} sites", + f1(left as f64 + r.start as f64 * scale), + y + k * (bar + 1), + f1(w), + colour[&(r.side, r.ancestor.clone())], + esc(r.side.as_str()), + esc(&name), + r.start, + r.end, + esc(who), + r.sites + )); + } + } + let y = 50 + row_h * chroms.len() as i64; + l.push(format!( + "Upper bar: the copy from the father. Lower bar: from the mother. Grey: not mapped." + )); + let mut legend: Vec<(&(Side, String), &&str)> = colour.iter().collect(); + legend.sort_by(|a, b| (a.0.0.as_str(), &a.0.1).cmp(&(b.0.0.as_str(), &b.0.1))); + for (j, ((side, anc), col)) in legend.into_iter().enumerate() { + let label = if *col == OTHER { + format!( + "{} side, the copy not from the sequenced parent", + side.as_str() + ) + } else if anc == "." { + format!("{} side, from the parent", side.as_str()) + } else { + format!("{} side, from {anc}", side.as_str()) + }; + let j = j as i64; + l.push(format!( + "{}", + y + 10 + 20 * j, + y + 19 + 20 * j, + esc(&label) + )); + } + l.push("".to_owned()); + let mut s = l.join("\n"); + s.push('\n'); + s +} + +impl Map { + /// Writes `_segments.tsv`, `_dnapainter.csv` and + /// `_segments.svg` for every child. + pub fn write(&self, out_dir: &Path) -> std::result::Result<(), Die> { + let lengths = &self.lengths; + std::fs::create_dir_all(out_dir) + .map_err(|e| Die::one(format!("cannot create {}: {e}", out_dir.display())))?; + for (child, rows) in &self.rows { + let mut t = String::from(TSV); + for r in rows { + t.push_str(&format!( + "{child}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + r.side.as_str(), + r.parent, + chrom_name(r.chrom), + r.start, + r.end, + r.length(), + r.sites, + r.discordant, + r.ancestor, + r.basis.as_str() + )); + } + write_file(&out_dir.join(format!("{child}_segments.tsv")), t.as_bytes())?; + let mut w = pycompat::csv::Writer::new(Vec::new(), ','); + let csv_err = + |e: std::io::Error| Die::one(format!("cannot write the DNA Painter CSV: {e}")); + w.write_row(DNAPAINTER).map_err(csv_err)?; + for r in rows { + let who = if r.ancestor != "." { + r.ancestor.clone() + } else if r.parent != "." { + r.parent.clone() + } else { + format!("{} side", r.side.as_str()) + }; + let how = match r.basis { + Basis::Grandparents => "the parent's copy, told by a sequenced grandparent", + Basis::Parent => "from the sequenced parent", + Basis::OtherCopy => "the copy not from the sequenced parent", + }; + let note = format!("GRCh38 positions; {how}; {} discordant sites", r.discordant); + // DNA Painter reads Side and Ancestor from the first row of each Group, and Ancestor + // takes only 1 (a known ancestor), so each grandparent (or parent) is a group of its own. + let known = r.ancestor != "." || r.parent != "."; + let name = chrom_name(r.chrom); + w.write_row([ + &name[3..], + &r.start.to_string(), + &r.end.to_string(), + "", + &r.sites.to_string(), + &format!("{child} from {who}"), + "", + &who, + r.side.as_str(), + ¬e, + "", + if known { "1" } else { "" }, + ]) + .map_err(csv_err)?; + } + write_file( + &out_dir.join(format!("{child}_dnapainter.csv")), + &w.into_inner(), + )?; + write_file( + &out_dir.join(format!("{child}_segments.svg")), + svg(child, rows, lengths).as_bytes(), + )?; + } + Ok(()) + } + + /// The "Chromosome map" section of `family_report.md`. + pub fn markdown(&self, members: &Members) -> Vec { + let mut l = vec![String::new(), "## Chromosome map".to_owned(), String::new()]; + if self.rows.is_empty() { + l.push("No sequenced child has a sequenced parent, so there is no map.".to_owned()); + return l; + } + l.push( + "For each child, the stretches of each chromosome that came from each parent, from the sites where \ + that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent \ + is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or \ + the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, \ + `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), \ + `_segments.svg` draws it." + .to_owned(), + ); + l.push(String::new()); + for (child, rows) in &self.rows { + let m = &members[child]; + for (side, who) in [(Side::Paternal, "father"), (Side::Maternal, "mother")] { + let mine: Vec<&Row> = rows.iter().filter(|r| r.side == side).collect(); + if mine.is_empty() { + continue; + } + let bp: i64 = mine.iter().map(|r| r.length()).sum(); + let mb = fixed(bp as f64 / 1e6, 1); + let name = match side { + Side::Paternal => m.father.as_deref(), + Side::Maternal => m.mother.as_deref(), + } + .unwrap_or("not named"); + let n = mine.len(); + if mine.iter().any(|r| r.basis == Basis::OtherCopy) { + l.push(format!( + "- {child}: from the {who} ({name}, not sequenced): {n} stretches over {mb} Mb, the copy that did not \ + come from the sequenced parent" + )); + } else if mine.iter().any(|r| r.basis == Basis::Grandparents) { + let x = crossovers(rows, side); + let mut anc: Vec<&str> = mine.iter().map(|r| r.ancestor.as_str()).collect(); + anc.sort_unstable(); + anc.dedup(); + let disc: usize = mine.iter().map(|r| r.discordant).sum(); + l.push(format!( + "- {child}: from the {who} ({name}): {n} stretches over {mb} Mb, told apart as the copy from {}; \ + {x} crossover{}, {disc} discordant sites absorbed", + anc.join(" or "), + if x == 1 { "" } else { "s" } + )); + } else { + l.push(format!( + "- {child}: from the {who} ({name}): {n} stretches over {mb} Mb; no grandparent on this side is \ + sequenced, so the two copies of the {who} are not told apart" + )); + } + } + } + l.push(String::new()); + l.push( + "A stretch starts and ends at its first and last informative site: a crossover lies somewhere \ + between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced \ + grandparent the map says only which parent a stretch came from." + .to_owned(), + ); + l + } +} + +/// Changes of ancestor between neighbouring segments of one chromosome. +fn crossovers(rows: &[Row], side: Side) -> usize { + let mut n = 0; + let mut prev: Option<&Row> = None; + for r in rows { + if r.side != side || r.basis != Basis::Grandparents { + continue; + } + if let Some(p) = prev + && p.chrom == r.chrom + && p.ancestor != r.ancestor + { + n += 1; + } + prev = Some(r); + } + n +} + +/// Reads the VCF's header: the `##` lines go to the collector, the +/// `#CHROM` line gives the sample columns. Returns the lines reader at +/// the first record. +pub fn read_header( + lines: &mut og_vcf::Lines, + path: &str, + mut on_meta: impl FnMut(&[u8]), +) -> std::result::Result>, Die> { + let io = |e: std::io::Error| Die::io(path, &e); + // Peek by reading; the caller handles the record that ends the header. + loop { + let Some(line) = lines.next_line().map_err(io)? else { + return Ok(None); + }; + if line.starts_with(b"##") { + on_meta(line); + continue; + } + if line.starts_with(b"#CHROM") { + return Ok(Some(vcf::sample_names(line))); + } + return Err(Die::one(format!( + "{path}: a record comes before the #CHROM line" + ))); + } +} + +/// Reads the whole `--vcf` for the map alone (`og family segments`, or +/// the second reader of `og family report --phased`). +pub fn collect( + members: &Members, + path: &str, + par: Par, +) -> std::result::Result<(Collector, Option>), Die> { + let r = og_io::bgzf::open(path, og_io::bgzf::workers()).map_err(|e| Die::io(path, &e))?; + let mut lines = og_vcf::Lines::new(r); + let mut lengths = HashMap::new(); + let names = read_header(&mut lines, path, |l| { + if let Some((id, n)) = contig_length(l) { + lengths.insert(id, n); + } + })?; + let idx = names.as_deref().map(vcf::sample_index).unwrap_or_default(); + if !has_links(members) { + let mut col = Collector::new(members, &idx, par); + col.lengths = lengths; + return Ok((col, names)); + } + let Some(names) = names else { + return Err(Die::one(format!("{path} has no #CHROM line"))); + }; + let lost: Vec = sequenced(members) + .into_iter() + .filter(|s| !idx.contains_key(s)) + .collect(); + if !lost.is_empty() { + return Err(Die::one(format!( + "{path} has no column for {}", + lost.join(", ") + ))); + } + let mut col = Collector::new(members, &idx, par); + col.lengths = lengths; + let mut n = lines.number(); + while let Some(line) = lines.next_line().map_err(|e| Die::io(path, &e))? { + n += 1; + let cols = vcf::columns(line); + if cols.len() < 2 { + return Err(Die::one(format!("{path} line {n}: not a VCF record"))); + } + let pos = vcf::pos(cols[1], path, n)?; + if chrom_index(cols[0]).is_none() { + col.sorted.next(path, cols[0], pos)?; + continue; + } + let Some(format) = cols.get(8) else { + return Err(Die::one(format!("{path} line {n}: no FORMAT column"))); + }; + let gi = vcf::format_index(format, b"GT"); + let mut calls = Calls::new(&cols, gi.unwrap_or(0), path, n); + col.record(path, pos, &mut calls, &cols, gi)?; + } + Ok((col, Some(names))) +} + +pub fn run(a: &Args) -> Result<()> { + let members = read_members(a.one("--members").unwrap_or_default())?; + let vcf_path = a.one("--vcf").unwrap_or_default(); + let par = Par::read(a.one("--par-bed"))?; + let (col, _) = collect(&members, vcf_path, par)?; + let map = col.finish(&members); + map.write(Path::new(a.one("--out-dir").unwrap_or_default()))?; + let mut text = map.markdown(&members).join("\n"); + text.push('\n'); + print_out(&text); + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-steps/src/family/segments/tests.rs b/rust/crates/og-steps/src/family/segments/tests.rs new file mode 100644 index 000000000..6c9e5ddb7 --- /dev/null +++ b/rust/crates/og-steps/src/family/segments/tests.rs @@ -0,0 +1,110 @@ +use super::*; + +/// The Python's smoothing, line for line: take the minimum of +/// `(agree, end - start, index)` over the whole list at every merge. O(R^2). +fn naive(sites: &[(i64, u8)]) -> Vec { + let mut runs = runs_of(sites); + while runs.len() > 1 { + let Some(i) = (0..runs.len()) + .filter(|&k| weak(&runs[k])) + .min_by_key(|&k| (runs[k].agree, runs[k].end - runs[k].start, k)) + else { + break; + }; + let lo = i.saturating_sub(1); + let hi = (i + 1).min(runs.len() - 1); + let keep = if lo != i { + runs[lo].label + } else { + runs[hi].label + }; + let mid: Vec<&Run> = runs[lo..=hi].iter().collect(); + let merged = merge(&runs[lo], &mid, &runs[hi], |k| lo + k == i, keep); + runs.splice(lo..=hi, [merged]); + } + runs.retain(|r| r.agree >= MIN_SITES); + runs +} + +/// A small fixed generator, so a failure names its seed. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self, n: u64) -> u64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (self.0 >> 33) % n + } +} + +/// Sites with runs of every strength: switches are rare or frequent, the +/// spacing small or large, and positions repeat now and then (so starts tie). +fn sites(seed: u64) -> Vec<(i64, u8)> { + let mut g = Lcg(seed); + let n = 1 + g.next(600) as usize; + let switch = 1 + g.next(40); + let step = [1, 1_000, 25_000, 60_000][g.next(4) as usize]; + let (mut pos, mut label) = (1 + g.next(1_000) as i64, g.next(2) as u8); + let mut out = Vec::with_capacity(n); + for _ in 0..n { + if g.next(switch) == 0 { + label ^= 1; + } + out.push((pos, label)); + if g.next(10) != 0 { + pos += 1 + g.next(step) as i64; + } + } + out +} + +#[test] +fn heap_smooth_matches_the_python_order() { + for seed in 0..3_000 { + let s = sites(seed); + assert_eq!(smooth(&s), naive(&s), "seed {seed}"); + } +} + +/// Two weak runs of equal strength: the first one in the block goes first. +/// Taking the later one leaves a different map. +#[test] +fn ties_go_to_the_first_run() { + let mut s = Vec::new(); + let mut pos = 0; + let mut push = |label: u8, n: usize, s: &mut Vec<(i64, u8)>| { + for _ in 0..n { + pos += 30_000; + s.push((pos, label)); + } + }; + push(0, 25, &mut s); + push(1, 3, &mut s); + push(0, 3, &mut s); + push(1, 25, &mut s); + let got = smooth(&s); + assert_eq!(got, naive(&s)); + assert_eq!( + got.iter() + .map(|r| (r.label, r.agree, r.discordant)) + .collect::>(), + [(0, 28, 3), (1, 25, 0)] + ); +} + +#[test] +fn nothing_weak_stays_as_is() { + let s: Vec<(i64, u8)> = (0..40).map(|i| (i * 100_000, u8::from(i >= 20))).collect(); + let got = smooth(&s); + assert_eq!(got.len(), 2); + assert!(smooth(&[]).is_empty()); + assert!(smooth(&[(5, 1)]).is_empty()); +} + +#[test] +fn blocks_split_at_gaps() { + let b = blocks([1, 2, MAX_GAP + 3, MAX_GAP + 4].into_iter()); + assert_eq!(b, [0..2, 2..4]); +} diff --git a/rust/crates/og-steps/src/family/vcf.rs b/rust/crates/og-steps/src/family/vcf.rs new file mode 100644 index 000000000..5c71763c1 --- /dev/null +++ b/rust/crates/og-steps/src/family/vcf.rs @@ -0,0 +1,272 @@ +//! The joint VCF as `og family report` and `og family segments` read it: +//! the columns of a record found once, each sample's genotype parsed at +//! most once per record. + +use memchr::memchr_iter; +use og_pedigree::Die; +use smallvec::SmallVec; + +/// Allele indices of a called genotype. +pub type Alleles = SmallVec<[u32; 2]>; + +/// The tab-separated columns of a record. +pub type Cols<'a> = SmallVec<[&'a [u8]; 16]>; + +/// Splits a record into its columns. +pub fn columns(line: &[u8]) -> Cols<'_> { + let mut out = Cols::new(); + let mut start = 0; + for t in memchr_iter(b'\t', line) { + out.push(&line[start..t]); + start = t + 1; + } + out.push(&line[start..]); + out +} + +/// A GT value as the Python read it: `(alleles, phased)`, alleles `None` +/// when the value is empty, `.`, or any allele is `.` or empty. `phased` +/// is whether a `|` appears. An allele that is not a whole number from 0 +/// up is an error (the Python stopped with a traceback). +pub fn parse_gt(s: &[u8]) -> Result<(Option, bool), String> { + if s.is_empty() || s == b"." { + return Ok((None, false)); + } + let phased = s.contains(&b'|'); + let parts = s.split(|&c| c == b'/' || c == b'|'); + let mut out = Alleles::new(); + for p in parts.clone() { + if p.is_empty() || p == b"." { + return Ok((None, phased)); + } + } + for p in parts { + let v = std::str::from_utf8(p) + .ok() + .and_then(|t| pycompat::num::parse_int(t).ok()) + .and_then(|v| u32::try_from(v).ok()) + .ok_or_else(|| { + format!( + "genotype '{}' is not a list of allele numbers", + String::from_utf8_lossy(s) + ) + })?; + out.push(v); + } + Ok((Some(out), phased)) +} + +/// `set(alleles)` as a short sorted list without repeats. +pub fn set(a: &[u32]) -> Alleles { + let mut s: Alleles = a.iter().copied().collect(); + s.sort_unstable(); + s.dedup(); + s +} + +/// The `(from_father, from_mother)` assignments a diploid child genotype +/// allows; a parent with no call (`None`) allows any allele. +pub fn resolve(child: &[u32], f: Option<&[u32]>, m: Option<&[u32]>) -> SmallVec<[(u32, u32); 2]> { + let (a, b) = (child[0], child[1]); + let mut opts = SmallVec::new(); + for (x, y) in [(a, b), (b, a)] { + if f.is_none_or(|f| f.contains(&x)) + && m.is_none_or(|m| m.contains(&y)) + && !opts.contains(&(x, y)) + { + opts.push((x, y)); + } + } + opts +} + +/// One sample's column of the current record. +struct Entry<'a> { + gt: Option, + phased: bool, + vals: SmallVec<[&'a [u8]; 8]>, +} + +/// The samples of one record, each split and parsed on first use. +pub struct Calls<'a> { + cols: &'a [&'a [u8]], + gi: usize, + cache: SmallVec<[Option>; 8]>, + /// For messages: the file and line number. + pub path: &'a str, + pub line: u64, +} + +impl<'a> Calls<'a> { + /// `gi`: the index of GT in FORMAT. + pub fn new(cols: &'a [&'a [u8]], gi: usize, path: &'a str, line: u64) -> Self { + let mut cache = SmallVec::new(); + cache.resize_with(cols.len().saturating_sub(9), || None); + Calls { + cols, + gi, + cache, + path, + line, + } + } + + fn entry(&mut self, col: usize) -> Result<&Entry<'a>, Die> { + let k = col - 9; + if k >= self.cache.len() { + return Err(Die::one(format!( + "{} line {}: {} columns, the #CHROM line has more", + self.path, + self.line, + self.cols.len() + ))); + } + if self.cache[k].is_none() { + let vals: SmallVec<[&'a [u8]; 8]> = self.cols[col].split(|&c| c == b':').collect(); + let raw = vals.get(self.gi).copied().unwrap_or(b"."); + let (gt, phased) = parse_gt(raw) + .map_err(|e| Die::one(format!("{} line {}: {e}", self.path, self.line)))?; + self.cache[k] = Some(Entry { gt, phased, vals }); + } + Ok(self.cache[k].as_ref().expect("filled above")) + } + + /// The alleles of the sample in column `col`, `None` when not called. + pub fn gt(&mut self, col: usize) -> Result, Die> { + Ok(self.entry(col)?.gt.clone()) + } + + /// The alleles and whether the GT is phased. + pub fn gt_phased(&mut self, col: usize) -> Result<(Option, bool), Die> { + let e = self.entry(col)?; + Ok((e.gt.clone(), e.phased)) + } + + /// FORMAT value `i` of the sample in column `col`, `None` past its end. + pub fn val(&mut self, col: usize, i: usize) -> Result, Die> { + Ok(self.entry(col)?.vals.get(i).copied()) + } +} + +/// The joint VCF is sorted, as the chromosome map needs: no position goes +/// back on a contig, and no contig comes back after another. +#[derive(Default)] +pub struct Sorted { + prev: Option<(Vec, i64)>, + done: std::collections::HashSet>, +} + +impl Sorted { + /// Checks the next record; the error names the record and the one before. + pub fn next(&mut self, path: &str, chrom: &[u8], pos: i64) -> Result<(), Die> { + let bad = match &self.prev { + Some((pc, pp)) if pc.as_slice() == chrom => pos < *pp, + Some((pc, _)) => { + self.done.insert(pc.clone()); + self.done.contains(chrom) + } + None => false, + }; + if bad { + let (pc, pp) = self.prev.as_ref().expect("bad needs a record before"); + return Err(Die::one(format!( + "{path} is not sorted: {}:{pos} after {}:{pp}", + String::from_utf8_lossy(chrom), + String::from_utf8_lossy(pc) + ))); + } + match &mut self.prev { + Some((pc, pp)) if pc.as_slice() == chrom => *pp = pos, + _ => self.prev = Some((chrom.to_vec(), pos)), + } + Ok(()) + } +} + +/// POS as the Python's `int()` read it. +pub fn pos(raw: &[u8], path: &str, line: u64) -> Result { + std::str::from_utf8(raw) + .ok() + .and_then(|t| pycompat::num::parse_int(t).ok()) + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| { + Die::one(format!( + "{path} line {line}: POS '{}' is not a whole number", + String::from_utf8_lossy(raw) + )) + }) +} + +/// The index of `key` in a FORMAT column. +pub fn format_index(format: &[u8], key: &[u8]) -> Option { + format.split(|&c| c == b':').position(|k| k == key) +} + +/// The sample names of a `#CHROM` line, in column order. +pub fn sample_names(line: &[u8]) -> Vec { + columns(line) + .iter() + .skip(9) + .map(|c| String::from_utf8_lossy(c).into_owned()) + .collect() +} + +/// `{s: 9 + i}`: a name given twice maps to its last column. +pub fn sample_index(names: &[String]) -> std::collections::HashMap { + names + .iter() + .enumerate() + .map(|(i, s)| (s.clone(), 9 + i)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gt_as_python_reads_it() { + assert_eq!( + parse_gt(b"0/1").unwrap(), + (Some(Alleles::from_slice(&[0, 1])), false) + ); + assert_eq!( + parse_gt(b"1|0").unwrap(), + (Some(Alleles::from_slice(&[1, 0])), true) + ); + assert_eq!(parse_gt(b".|1").unwrap(), (None, true)); + assert_eq!(parse_gt(b".").unwrap(), (None, false)); + assert_eq!(parse_gt(b"").unwrap(), (None, false)); + assert_eq!( + parse_gt(b"1").unwrap(), + (Some(Alleles::from_slice(&[1])), false) + ); + assert!(parse_gt(b"A/1").is_err()); + } + + #[test] + fn resolve_as_python() { + assert_eq!( + resolve(&[0, 1], Some(&[0]), Some(&[0, 1])).as_slice(), + &[(0, 1)] + ); + assert_eq!(resolve(&[0, 1], Some(&[0, 1]), Some(&[0, 1])).len(), 2); + assert_eq!(resolve(&[1, 1], None, Some(&[1])).as_slice(), &[(1, 1)]); + assert!(resolve(&[1, 1], Some(&[0]), Some(&[0])).is_empty()); + } + + #[test] + fn sorted_check() { + let mut s = Sorted::default(); + assert!(s.next("v", b"chr1", 5).is_ok()); + assert!(s.next("v", b"chr1", 5).is_ok()); + assert!(s.next("v", b"chr2", 1).is_ok()); + let e = s.next("v", b"chr2", 0).unwrap_err(); + assert_eq!(e.lines, ["v is not sorted: chr2:0 after chr2:1"]); + let mut s = Sorted::default(); + s.next("v", b"chr1", 5).unwrap(); + s.next("v", b"chr2", 1).unwrap(); + let e = s.next("v", b"chr1", 9).unwrap_err(); + assert_eq!(e.lines, ["v is not sorted: chr1:9 after chr2:1"]); + } +} diff --git a/rust/crates/og-steps/src/lib.rs b/rust/crates/og-steps/src/lib.rs index 542697ce7..f8ee292f3 100644 --- a/rust/crates/og-steps/src/lib.rs +++ b/rust/crates/og-steps/src/lib.rs @@ -3,5 +3,8 @@ //! //! - [`constraint`]: `og table constraint-join`, gnomAD v4.1 gene constraint //! columns appended to a table. +//! - [`family`]: `og family plan|prepare|check|report|segments`, the family +//! analysis of step 38 and the Nextflow FAMILY workflow. pub mod constraint; +pub mod family; diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index acc0e4476..cb0fb028b 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -18,7 +18,7 @@ fn tree_hash() -> &'static str { &TREE_MARK["og-tree-hash:".len()..TREE_MARK.len() - 1] } -const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og dev parity-compare KIND ORACLE NEW\n og version\n og --tree-hash\n"; +const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og family plan|prepare|check|report|segments [args]\n og dev parity-compare KIND ORACLE NEW\n og dev remnants [--list FILE] [--root DIR]\n og version\n og --tree-hash\n"; /// argparse's way to refuse a command line: usage and the error on stderr, exit 2. fn usage_error(msg: &str) -> ExitCode { @@ -110,6 +110,7 @@ fn main() -> ExitCode { None => usage_error("the following arguments are required: pgx tool"), }, "dev" => ExitCode::from(og_dev::main(rest)), + "family" => ExitCode::from(og_steps::family::cli(rest)), "chip" => match rest.first() { Some(verb) => ExitCode::from(og_chip::cli( verb, diff --git a/scripts/38-family.sh b/scripts/38-family.sh index 10f4dba30..37efa616f 100755 --- a/scripts/38-family.sh +++ b/scripts/38-family.sh @@ -12,7 +12,7 @@ # a sample's): # pedigree.tsv the pedigree the step read (the family's rows of a # families file, copied beside it as families.tsv) -# family_plan.tsv which family analyses ran and why (family_report.py plan's table) +# family_plan.tsv which family analyses ran and why (og family plan's table) # family.vcf.gz the members joint-called by GLnexus # somalier/ somalier relate over the members' gVCFs # relationships.tsv each pair's declared and measured relatedness, verdict @@ -35,8 +35,8 @@ # # The step stops (exit 1) when a pair's measured relatedness disagrees with # the pedigree, before any phasing: a sample swap or a wrong pedigree line. -# relationships.tsv says which pair and why. The rules are bin/family_report.py's, -# the code the Nextflow FAMILY workflow (--pedigree) runs too. +# relationships.tsv says which pair and why. The rules are og family's +# (rust/crates/og-steps/src/family), the code the Nextflow FAMILY workflow (--pedigree) runs too. # # Env: SOMALIER_SITES sites VCF (default reference/somalier/sites.hg38.vcf.gz), # inside GENOME_DIR @@ -147,7 +147,7 @@ done # A families file (a family column: assets/families.example.tsv) holds several # families: this family's rows become its own ${OUT}/pedigree.tsv, written by -# bin/family_report.py plan (which checks the whole file first), and the step +# og family plan (which checks the whole file first), and the step # reads that. A pedigree without the column is read as it is. if awk -F'\t' '{ sub(/\r+$/, "") } /^[ \t]*$/ { next } { sub(/^#+/, ""); for (i = 1; i <= NF; i++) { f = tolower($i); gsub(/^[ \t]+|[ \t]+$/, "", f); if (f == "family") found = 1 }; exit } @@ -160,7 +160,7 @@ if awk -F'\t' '{ sub(/\r+$/, "") } /^[ \t]*$/ { next } mkdir -p "$OUT" [ "$PEDIGREE" -ef "${OUT}/families.tsv" ] || cp -f "$PEDIGREE" "${OUT}/families.tsv" if ! run_in --rw "$OUT" --cpus 1 --memory 1g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/family_report.py plan --families "${OUT_C}/families.tsv" --family "$FAMILY" --write-pedigree /genome \ + /pgp-bin/og family plan --families "${OUT_C}/families.tsv" --family "$FAMILY" --write-pedigree /genome \ > /dev/null; then echo " (families.tsv is the step's copy of ${PEDIGREE})" >&2 [ -z "$NEW_OUT" ] || rm -rf "$OUT" @@ -172,8 +172,8 @@ fi # The samples the pedigree names (its sample column; the first non-blank # line is the header), and which of them have a gVCF here. Read as -# family_report.py reads it: CRLF line ends, whitespace-only lines and spaces -# around a column name are allowed. family_report.py prepare checks +# og family reads it: CRLF line ends, whitespace-only lines and spaces +# around a column name are allowed. og family prepare checks # everything else. mapfile -t NAMED < <(awk -F'\t' ' { sub(/\r+$/, "") } @@ -230,8 +230,8 @@ echo "Members: ${MEMBERS[*]}" mkdir -p "$OUT" [ "$PEDIGREE" -ef "${OUT}/pedigree.tsv" ] || cp -f "$PEDIGREE" "${OUT}/pedigree.tsv" -PY=(run_in --rw "$OUT" --cpus 2 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets/par_grch38.bed:/pgp/par_grch38.bed:ro" - "$PYTHON_IMAGE" python3 /pgp-bin/family_report.py) +OG=(run_in --rw "$OUT" --cpus 2 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets/par_grch38.bed:/pgp/par_grch38.bed:ro" + "$PYTHON_IMAGE" /pgp-bin/og family) # --- 1. The pedigree, and each gVCF's sample name -------------------------------------- GVCF_ARGS=() @@ -243,7 +243,7 @@ done # family_plan.tsv: the same decision run-all.sh's plan makes, from the members' ClinVar hits and the panel CLINVAR_HAVE=() for s in ${NAMED[@]+"${NAMED[@]}"}; do [ ! -f "${GENOME_DIR}/${s}/clinvar/${s}_clinvar_hits.vcf" ] || CLINVAR_HAVE+=("$s"); done -"${PY[@]}" prepare --pedigree "${OUT_C}/pedigree.tsv" --family "$FAMILY" "${GVCF_ARGS[@]}" \ +"${OG[@]}" prepare --pedigree "${OUT_C}/pedigree.tsv" --family "$FAMILY" "${GVCF_ARGS[@]}" \ --have-clinvar ${CLINVAR_HAVE[@]+"${CLINVAR_HAVE[@]}"} --ibd-panel "$(ibd_panel_ok && echo yes || echo no)" --out-dir "$OUT_C" # --- 2. Joint call (GLnexus) -------------------------------------------------------------- @@ -278,7 +278,7 @@ done run_in --rw "$OUT" --cpus 1 --memory 2g "$SOMALIER_IMAGE" \ somalier relate --ped "${OUT_C}/family.ped" -o "${OUT_C}/somalier/family" "${EXTRACTED[@]}" rm -rf "${OUT}/somalier/extract" -"${PY[@]}" check --members "${OUT_C}/members.tsv" --pairs "${OUT_C}/somalier/family.pairs.tsv" \ +"${OG[@]}" check --members "${OUT_C}/members.tsv" --pairs "${OUT_C}/somalier/family.pairs.tsv" \ --out "${OUT_C}/relationships.tsv" # --- 4. Phasing by inheritance (WhatsHap), complete trios only ------------------------------- @@ -305,7 +305,7 @@ for s in "${MEMBERS[@]}"; do [ -f "$c" ] && CANDS+=(--candidates "${s}=/genome/${s}/clinical/${s}_clinical.vcf.gz") done rm -f "${OUT}"/*_inheritance.tsv.gz "${OUT}"/*_segments.tsv "${OUT}"/*_dnapainter.csv "${OUT}"/*_segments.svg -"${PY[@]}" report --members "${OUT_C}/members.tsv" --relationships "${OUT_C}/relationships.tsv" \ +"${OG[@]}" report --members "${OUT_C}/members.tsv" --relationships "${OUT_C}/relationships.tsv" \ --vcf "${OUT_C}/family.vcf.gz" ${PHASED[@]+"${PHASED[@]}"} ${CANDS[@]+"${CANDS[@]}"} \ --par-bed /pgp/par_grch38.bed --out-dir "$OUT_C" diff --git a/scripts/39-couple.sh b/scripts/39-couple.sh index 60a010b3a..71d1050f2 100755 --- a/scripts/39-couple.sh +++ b/scripts/39-couple.sh @@ -79,7 +79,7 @@ fi # A families file (a family column: assets/families.example.tsv) holds several # families: this family's rows become its own ${OUT}/pedigree.tsv, written by -# bin/family_report.py plan (which checks the whole file first), and the step +# og family plan (which checks the whole file first), and the step # reads that. A pedigree without the column is read as it is. if awk -F'\t' '{ sub(/\r+$/, "") } /^[ \t]*$/ { next } { sub(/^#+/, ""); for (i = 1; i <= NF; i++) { f = tolower($i); gsub(/^[ \t]+|[ \t]+$/, "", f); if (f == "family") found = 1 }; exit } @@ -92,7 +92,7 @@ if awk -F'\t' '{ sub(/\r+$/, "") } /^[ \t]*$/ { next } mkdir -p "$OUT" [ "$PEDIGREE" -ef "${OUT}/families.tsv" ] || cp -f "$PEDIGREE" "${OUT}/families.tsv" if ! run_in --rw "$OUT" --cpus 1 --memory 1g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/family_report.py plan --families "${OUT_C}/families.tsv" --family "$FAMILY" --write-pedigree /genome \ + /pgp-bin/og family plan --families "${OUT_C}/families.tsv" --family "$FAMILY" --write-pedigree /genome \ > /dev/null; then echo " (families.tsv is the step's copy of ${PEDIGREE})" >&2 [ -z "$NEW_OUT" ] || rm -rf "$OUT" diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 2b3f5054e..6c2d7359f 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -77,23 +77,23 @@ cn-engine-1-parity assets/bloodtype_markers.tsv bin/bloodtype_report.py bin/cn_l cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog bin/cn_loci.tsv scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/comphet_phase.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/export_consumer.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/exomiser_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/exomiser_report.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/rfc1_motif.py bin/strchive_convert.py scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv -family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh -followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh +followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz genome-lookup-1 scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh @@ -105,7 +105,7 @@ haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html a hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/export_consumer.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/family_segments.py bin/ibd_report.py scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/ibd_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/mei_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh @@ -144,7 +144,7 @@ prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summar prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-1-sample-qc bin/collect_summary.py bin/mosaic_sex_loss.py bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/mosaic_sex_loss.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/export_consumer.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 700fd09b5..b79c0257a 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -29,7 +29,7 @@ # SOMATIC=true (29), EXTRA_CALLERS=gatk,freebayes,strelka2,octopus (03a-03d), BENCHMARK=true (needs # EXTRA_CALLERS or a second caller VCF); then the HTML report (24) and the text report, also after a failed pipeline. # FAMILIES=families.tsv (who is related to whom, assets/families.example.tsv): when the sample is in a family, the sex -# must agree with the file (exit 2 before the pipeline). After the pipeline, before the reports, bin/family_report.py +# must agree with the file (exit 2 before the pipeline). After the pipeline, before the reports, og family # plan writes GENOME_DIR//family_plan.tsv from what the members have, and step 38 (38-family.sh) and step 39 # (39-couple.sh) run when it says so, or are recorded as skipped with the reason (waiting: a member has no gVCF, or for step 39 no ClinVar hits, yet). # The other members are never run here: run run-all.sh for each, and the last one runs the family steps. @@ -83,10 +83,10 @@ on SKIP_VALIDATION || "${SCRIPT_DIR}/validate-setup.sh" "$SAMPLE" \ || { echo "ERROR: setup validation failed; fix the items above. To bypass: SKIP_VALIDATION=true $0 ${SAMPLE} ${SEX}" >&2; exit 1; } # The sample's family (FAMILIES): the file is checked and the sex must be the one it gives, before anything runs. FAM="" -famplan() { # famplan [run_in options] -- [plan options]: bin/family_report.py plan on the FAMILIES file +famplan() { # famplan [run_in options] -- [plan options]: og family plan on the FAMILIES file local o=(); while [ "$1" != -- ]; do o+=("$1"); shift; done; shift run_in ${o[@]+"${o[@]}"} --cpus 1 --memory 1g -v "${FAMILIES}:/pgp/families.tsv:ro" -v "${PGP_ROOT}/bin:/pgp-bin:ro" \ - "$PYTHON_IMAGE" python3 /pgp-bin/family_report.py plan --families /pgp/families.tsv "$@" + "$PYTHON_IMAGE" /pgp-bin/og family plan --families /pgp/families.tsv "$@" } if [ -n "${FAMILIES:-}" ]; then [ -f "$FAMILIES" ] || { echo "ERROR: FAMILIES=${FAMILIES}: no such file (assets/families.example.tsv shows the format)" >&2; exit 2; } @@ -372,7 +372,7 @@ done # The reports are written after a failed pipeline too: they show what finished and mark older results stale. [ "$rc" -eq 0 ] || { echo "ERROR: the pipeline failed (exit ${rc}): see above and ${S}/nextflow/.nextflow.log. Fix the cause and run the same command; -resume reruns only what did not finish. Writing the reports of what finished." >&2; OPTIN=(); } # Steps 38 and 39 for the sample's family, after the pipeline and the opt-in scripts and before the reports (so they -# show this run's couple table), as family_report.py plan decides from what is under GENOME_DIR now: each member's +# show this run's couple table), as og family plan decides from what is under GENOME_DIR now: each member's # gVCF (with its index) and step 06's ClinVar hits, and step 26b's panel for the shared segments. family_steps() { [ -n "$FAM" ] && [ "$rc" -eq 0 ] || return 0 @@ -385,8 +385,8 @@ family_steps() { famplan --rw "${G}/${FAM}" -- --family "$FAM" --have-gvcf ${HG[@]+"${HG[@]}"} --have-clinvar ${HC[@]+"${HC[@]}"} \ --ibd-panel "$IBD" --out "/genome/${FAM}/family_plan.tsv" || { # never act on an older family_plan.tsv: the family steps wait for a plan that worked - echo " FAILED: family_report.py plan; steps 38 and 39 not run" >&2; failed=1 - printf 'step\t%s\tskipped (family_report.py plan failed)\n' 38 39 >> "$STATUS"; return 0; } + echo " FAILED: og family plan; steps 38 and 39 not run" >&2; failed=1 + printf 'step\t%s\tskipped (og family plan failed)\n' 38 39 >> "$STATUS"; return 0; } # decisionreason of step 38 (the joint call) and step 39 (a couple that runs, else the first couple) d38=$(awk -F'\t' '$2 == "joint_call" {print $4 "\t" $5; exit}' "$FP" 2>/dev/null) d39=$(awk -F'\t' '$2 == "couple_carrier" {r = r ? r : $4 "\t" $5; if ($4 == "runs") {r = $4 "\t" $5; exit}} END {print r}' "$FP" 2>/dev/null) diff --git a/tests/e2e/family-joint-1.sh b/tests/e2e/family-joint-1.sh index 4c52f753a..4ce3adf25 100755 --- a/tests/e2e/family-joint-1.sh +++ b/tests/e2e/family-joint-1.sh @@ -293,7 +293,7 @@ check "FAMILY_CHECK is the task that failed (.nextflow.log)" \ # The pair message, read from the failed FAMILY_CHECK task's own .command.err. FC_ERR="" for d in "${F}"/nfwork/*/*/; do - grep -q 'family_report.py check' "${d}.command.sh" 2>/dev/null || continue + grep -q 'og family check' "${d}.command.sh" 2>/dev/null || continue [ "$(cat "${d}.exitcode" 2>/dev/null)" = 0 ] && continue FC_ERR="${d}.command.err" done diff --git a/tests/fake-docker/run-all-families.sh b/tests/fake-docker/run-all-families.sh index 5cb809bf4..dcb4e2fe5 100755 --- a/tests/fake-docker/run-all-families.sh +++ b/tests/fake-docker/run-all-families.sh @@ -2,7 +2,7 @@ # run-all.sh with FAMILIES=: the file says who is related to # whom, and run-all.sh runs the family steps when their members are ready. # 1. partner_x of family_b (tests/fixtures/families/two_families.tsv) runs -# first: bin/family_report.py plan writes GENOME_DIR/family_b/family_plan.tsv, +# first: og family plan writes GENOME_DIR/family_b/family_plan.tsv, # and steps 38 and 39 are recorded as skipped, waiting for partner_y's # gVCF; neither script runs; # 2. partner_y runs next: both members have a gVCF and ClinVar hits, so @@ -17,8 +17,8 @@ # rows to GENOME_DIR/family_b/pedigree.tsv and hands that file, not the # two-family one, to bin/couple_report.py. # run-all.sh runs from a copy of the repository whose 38-family.sh and -# 39-couple.sh only log their arguments; family_report.py runs for real, with -# the host's python3, through a hook on the fake docker. +# 39-couple.sh only log their arguments; og family runs for real (bin/og, the +# host build), through a hook on the fake docker. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" @@ -33,6 +33,8 @@ cp "${REPO_ROOT}/tests/fixtures/families/two_families.tsv" "$FAMS" R="${CASE_WORK}/repo" mkdir -p "$R" (cd "$REPO_ROOT" && git ls-files -z -co --exclude-standard | tar --null -T - -cf -) | tar -xf - -C "$R" +# og family plans the family: the built og (bin/.og, ignored by git) comes too. +cp -R "${REPO_ROOT}/bin/.og" "${R}/bin/" || fail "bin/.og is missing: build og first (scripts/setup.sh --og)" for n in 38-family 39-couple; do printf '#!/usr/bin/env bash\necho "%s $*" >> "${CASE_WORK}/family-steps.log"\n' "${n%%-*}" > "${R}/scripts/${n}.sh" done @@ -44,8 +46,8 @@ cat > "${CASE_WORK}/tools-hook" <<'HOOK' set -euo pipefail . "${CASE_WORK:?}/host-path.sh" case "${*:2}" in - "python3 /pgp-bin/family_report.py"*) - # The real family_report.py, on the host paths of every /... argument. + "/pgp-bin/og family"*) + # The real og family, on the host paths of every /... argument. shift mapped=() for a in "$@"; do @@ -102,15 +104,17 @@ if status partner_y | grep -q $'\tplanned$'; then fail "a planned row is left: $ [ "$(grep -c '' "$STEPS_LOG")" -eq 2 ] || fail "the family scripts ran $(grep -c '' "$STEPS_LOG") times, want 2" # --- 2b. a plan that fails: the family steps do not run from the older plan ----------------- -chmod 444 "$PLAN" +# og writes the plan atomically (a new file renamed over the old one), so a +# read-only plan would be replaced: the family's folder is what cannot be written. +chmod 555 "${PLAN%/*}" run_rc planfail env FAMILIES="$FAMS" "${R}/scripts/run-all.sh" partner_y female -chmod 644 "$PLAN" +chmod 755 "${PLAN%/*}" [ "$RC" -eq 1 ] || fail "a failed plan exited ${RC}, expected 1" [ "$(grep -c '' "$STEPS_LOG")" -eq 2 ] || fail "a family step ran from the older plan: $(cat "$STEPS_LOG")" -output_has planfail 'FAILED: family_report.py plan; steps 38 and 39 not run' -status partner_y | grep -q $'^step\t38\tskipped (family_report.py plan failed)$' \ +output_has planfail 'FAILED: og family plan; steps 38 and 39 not run' +status partner_y | grep -q $'^step\t38\tskipped (og family plan failed)$' \ || fail "step 38 is not recorded as skipped after the failed plan: $(status partner_y)" -status partner_y | grep -q $'^step\t39\tskipped (family_report.py plan failed)$' \ +status partner_y | grep -q $'^step\t39\tskipped (og family plan failed)$' \ || fail "step 39 is not recorded as skipped after the failed plan: $(status partner_y)" # --- 3. a sex the file contradicts stops before nextflow ------------------------------------- diff --git a/tests/fixtures/families/README.md b/tests/fixtures/families/README.md index 8ac0e5a6c..23cf41b54 100644 --- a/tests/fixtures/families/README.md +++ b/tests/fixtures/families/README.md @@ -1,11 +1,11 @@ # Families fixtures -Invented names only, for `tests/test_family_report.py`. +Invented names only, for `tests/test_family_report.py` (`og family`). - `two_families.tsv`: a families file with the family column, two families (a trio and a couple). - `members.golden.tsv`: the `members.tsv` that `family_report.py prepare` - wrote for `assets/pedigree.example.tsv` before the family column existed. A + (now `og family prepare`) wrote for `assets/pedigree.example.tsv` before the family column existed. A file without the column must still give these bytes. - `stub_families.tsv`: two families for the stub run of `tests/e2e/families-file-1.sh`; family_b holds two samples of diff --git a/tests/parity/README.md b/tests/parity/README.md index 51214192d..9aa9dc93d 100644 --- a/tests/parity/README.md +++ b/tests/parity/README.md @@ -21,7 +21,7 @@ scripts/ci/oracle-diff.sh --self-test # planted differences must each be re OG=path/to/og scripts/ci/oracle-diff.sh # another og binary ``` -Each case stages the same inputs into two directories. `og` runs in one directory. The oracle runs in the other, inside its image with no network. Both get the same arguments. The harness then compares the exit code, stdout, stderr, the outputs the case names and every new file either side wrote. A staged input that a tool changes in place is compared only when the case names it as an output, so name such a file in the `outputs` column. Files use the case's compare kind. The exit code, stdout and stderr compare as bytes. `og dev parity-compare` does each comparison. Its kinds are `bytes`, `gz-content`, `zip-content`, `png-pixels`, `tsv[:keys=A,B][:unordered]`, `kv`, `vcf`, `json[:paths=..][:mask=..][:tol=..]`, `mask:REGEX` and `stderr-first-line`. Its own `--self-test` plants a difference of each kind. +Each case stages the same inputs into two directories. `og` runs in one directory. The oracle runs in the other, inside its image with no network. Both get the same arguments. The harness then compares the exit code, stdout, stderr, the outputs the case names and every new file either side wrote. A staged input that a tool changes in place is compared only when the case names it as an output, so name such a file in the `outputs` column. Files use the case's compare kind. The exit code, stdout and stderr compare as bytes. `og dev parity-compare` does each comparison. Its kinds are `bytes`, `gz-content`, `content` (`gz-content` when the oracle's file is gzip, else `bytes`, for a case whose outputs mix the two), `zip-content`, `png-pixels`, `tsv[:keys=A,B][:unordered]`, `kv`, `vcf`, `json[:paths=..][:mask=..][:tol=..]`, `mask:REGEX` and `stderr-first-line`. Its own `--self-test` plants a difference of each kind. Each difference has one of three outcomes: diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index d71c7f6ae..6de2fabe8 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -118,3 +118,77 @@ chip check-bad-table chip/bad-table.tsv check-table bad-table.tsv - bytes PYTHON chip check-bad-header chip/bad-header.tsv check-table bad-header.tsv - bytes PYTHON_IMAGE chip check-short-row chip/short-row.tsv check-table short-row.tsv - bytes PYTHON_IMAGE chip check-no-genes chip/no-genes.tsv check-table no-genes.tsv - bytes PYTHON_IMAGE +family plan-wait gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample partner_x --have-gvcf partner_x --out plan.tsv plan.tsv content PYTHON_IMAGE +family plan-trio gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --family family_a --have-gvcf child_a father_a mother_a --have-clinvar child_a father_a mother_a --ibd-panel yes --write-pedigree wp --out plan.tsv plan.tsv,wp/family_a/pedigree.tsv content PYTHON_IMAGE +family plan-duo gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --family family_a --have-gvcf child_a mother_a --have-clinvar mother_a --ibd-panel no --out plan.tsv plan.tsv content PYTHON_IMAGE +family plan-partners gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --family family_b --have-gvcf partner_x partner_y --have-clinvar partner_x --out plan.tsv plan.tsv content PYTHON_IMAGE +family plan-couple-no-gvcf gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --family family_b --have-gvcf partner_x --have-clinvar partner_x partner_y --out=plan.tsv plan.tsv content PYTHON_IMAGE +family plan-pedigree gen:family-oracle.sh,family/pedigree.example.tsv plan --families pedigree.example.tsv --have-gvcf child_a father_a --write-pedigree wp - content PYTHON_IMAGE +family plan-pedigree-sample gen:family-oracle.sh,family/pedigree.example.tsv plan --families pedigree.example.tsv --sample child_a --sex male --have-gvcf child_a - content PYTHON_IMAGE +family plan-which gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample partner_y --which - content PYTHON_IMAGE +family plan-which-none gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample nobody --which - content PYTHON_IMAGE +family plan-nobody gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample nobody - content PYTHON_IMAGE +family plan-two-families gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv - content PYTHON_IMAGE +family plan-unknown-family gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --family family_c - content PYTHON_IMAGE +family plan-sex-contradicts gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample partner_x --sex female - content PYTHON_IMAGE +family plan-sex-ok gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample partner_x --sex Male - content PYTHON_IMAGE +family plan-sex-bad gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sample partner_x --sex boy - content PYTHON_IMAGE +family plan-sex-no-sample gen:family-oracle.sh,family/two_families.tsv plan --families two_families.tsv --sex male - content PYTHON_IMAGE +family plan-crlf gen:family-oracle.sh,family/ped-crlf.tsv plan --families ped-crlf.tsv --have-gvcf kid dad mum --out plan.tsv plan.tsv content PYTHON_IMAGE +family plan-family-name-is-sample gen:family-oracle.sh,family/trio.tsv plan --families trio.tsv --family kid - content PYTHON_IMAGE +family plan-bad-twofam gen:family-oracle.sh,family/fam-twofam.tsv plan --families fam-twofam.tsv --family fa - content PYTHON_IMAGE +family plan-bad-famname gen:family-oracle.sh,family/fam-famname.tsv plan --families fam-famname.tsv --family fa - content PYTHON_IMAGE +family plan-bad-famissample gen:family-oracle.sh,family/fam-famissample.tsv plan --families fam-famissample.tsv --family fa - content PYTHON_IMAGE +family plan-bad-parentother gen:family-oracle.sh,family/fam-parentother.tsv plan --families fam-parentother.tsv --family fa - content PYTHON_IMAGE +family plan-bad-partnerother gen:family-oracle.sh,family/fam-partnerother.tsv plan --families fam-partnerother.tsv --family fa - content PYTHON_IMAGE +family plan-bad-partial gen:family-oracle.sh,family/fam-partial.tsv plan --families fam-partial.tsv --family fa - content PYTHON_IMAGE +family plan-all-missing-family gen:family-oracle.sh,family/fam-allmissing.tsv plan --families fam-allmissing.tsv --have-gvcf kid dad mum --out plan.tsv plan.tsv content PYTHON_IMAGE +family plan-ped-nohead gen:family-oracle.sh,family/ped-nohead.tsv plan --families ped-nohead.tsv - content PYTHON_IMAGE +family plan-ped-unknowncol gen:family-oracle.sh,family/ped-unknowncol.tsv plan --families ped-unknowncol.tsv - content PYTHON_IMAGE +family plan-ped-femalefather gen:family-oracle.sh,family/ped-femalefather.tsv plan --families ped-femalefather.tsv - content PYTHON_IMAGE +family plan-ped-ownparent gen:family-oracle.sh,family/ped-ownparent.tsv plan --families ped-ownparent.tsv - content PYTHON_IMAGE +family plan-ped-loop gen:family-oracle.sh,family/ped-loop.tsv plan --families ped-loop.tsv - content PYTHON_IMAGE +family plan-ped-secondrow gen:family-oracle.sh,family/ped-secondrow.tsv plan --families ped-secondrow.tsv - content PYTHON_IMAGE +family plan-ped-samefm gen:family-oracle.sh,family/ped-samefm.tsv plan --families ped-samefm.tsv - content PYTHON_IMAGE +family plan-ped-badsex gen:family-oracle.sh,family/ped-badsex.tsv plan --families ped-badsex.tsv - content PYTHON_IMAGE +family plan-ped-partners gen:family-oracle.sh,family/ped-partners.tsv plan --families ped-partners.tsv - content PYTHON_IMAGE +family plan-ped-toomany gen:family-oracle.sh,family/ped-toomany.tsv plan --families ped-toomany.tsv - content PYTHON_IMAGE +family plan-ped-empty gen:family-oracle.sh,family/ped-empty.tsv plan --families ped-empty.tsv - content PYTHON_IMAGE +family plan-ped-badname gen:family-oracle.sh,family/ped-badname.tsv plan --families ped-badname.tsv - content PYTHON_IMAGE +family plan-missing-file gen:family-oracle.sh plan --families nothere.tsv - content PYTHON_IMAGE +family prepare-trio gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --out-dir out out/members.tsv,out/family.ped,out/trios.ped,out/family_plan.tsv content PYTHON_IMAGE +family prepare-duo gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf mum=mum.g.vcf --declared-sex kid=Male --have-clinvar mum dad --out-dir out out/members.tsv,out/family.ped,out/trios.ped,out/family_plan.tsv content PYTHON_IMAGE +family prepare-declared-contradicts gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --declared-sex mum=male --out-dir out - content PYTHON_IMAGE +family prepare-declared-bad gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --declared-sex mum=x --out-dir out - content PYTHON_IMAGE +family prepare-one-sample gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-wrong-sample gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/hg003.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf dad=hg003.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-no-chrom gen:family-oracle.sh,family/trio.tsv,family/kid.g.vcf,family/nochrom.g.vcf prepare --pedigree trio.tsv --gvcf kid=kid.g.vcf --gvcf dad=nochrom.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-ped-femalefather gen:family-oracle.sh,family/ped-femalefather.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree ped-femalefather.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-ped-loop gen:family-oracle.sh,family/ped-loop.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree ped-loop.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-ped-partners gen:family-oracle.sh,family/ped-partners.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree ped-partners.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-ped-badsex gen:family-oracle.sh,family/ped-badsex.tsv,family/kid.g.vcf,family/dad.g.vcf,family/mum.g.vcf prepare --pedigree ped-badsex.tsv --gvcf kid=kid.g.vcf --gvcf dad=dad.g.vcf --gvcf mum=mum.g.vcf --out-dir out - content PYTHON_IMAGE +family prepare-example gen:family-oracle.sh,family/pedigree.example.tsv,family/child_a.g.vcf,family/father_a.g.vcf,family/mother_a.g.vcf prepare --pedigree pedigree.example.tsv --gvcf child_a=child_a.g.vcf --gvcf father_a=father_a.g.vcf --gvcf mother_a=mother_a.g.vcf --out-dir out out/members.tsv,out/family.ped,out/trios.ped,out/family_plan.tsv content PYTHON_IMAGE +family prepare-families gen:family-oracle.sh,family/two_families.tsv,family/child_a.g.vcf,family/father_a.g.vcf,family/mother_a.g.vcf prepare --pedigree two_families.tsv --family family_a --have-clinvar father_a --ibd-panel yes --pedigree-out ped.tsv --gvcf child_a=child_a.g.vcf --gvcf father_a=father_a.g.vcf --gvcf mother_a=mother_a.g.vcf --out-dir out out/members.tsv,out/family.ped,out/trios.ped,out/family_plan.tsv,ped.tsv content PYTHON_IMAGE +family prepare-families-no-family gen:family-oracle.sh,family/two_families.tsv,family/child_a.g.vcf,family/father_a.g.vcf,family/mother_a.g.vcf prepare --pedigree two_families.tsv --gvcf child_a=child_a.g.vcf --gvcf father_a=father_a.g.vcf --gvcf mother_a=mother_a.g.vcf --out-dir out - content PYTHON_IMAGE +family check-ok gen:family-oracle.sh,family/members-trio.tsv,family/pairs-ok.tsv check --members members-trio.tsv --pairs pairs-ok.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family check-swap gen:family-oracle.sh,family/members-swap.tsv,family/pairs-ok.tsv check --members members-swap.tsv --pairs pairs-ok.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family check-few gen:family-oracle.sh,family/members-trio.tsv,family/pairs-few.tsv check --members members-trio.tsv --pairs pairs-few.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family check-sib gen:family-oracle.sh,family/members-trio.tsv,family/pairs-sib.tsv check --members members-trio.tsv --pairs pairs-sib.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family check-warn gen:family-oracle.sh,family/members-trio.tsv,family/pairs-warn.tsv check --members members-trio.tsv --pairs pairs-warn.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family check-lost gen:family-oracle.sh,family/members-trio.tsv,family/pairs-lost.tsv check --members members-trio.tsv --pairs pairs-lost.tsv --out rel.tsv - content PYTHON_IMAGE +family check-duo gen:family-oracle.sh,family/members-duo.tsv,family/pairs-duo.tsv check --members members-duo.tsv --pairs pairs-duo.tsv --out rel.tsv rel.tsv content PYTHON_IMAGE +family report-trio gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/trio.vcf,family/kid_clinical.vcf,family/mum_ann.vcf,family/par.bed report --members members-trio.tsv --relationships rel.tsv --vcf trio.vcf --candidates kid=kid_clinical.vcf --candidates mum=mum_ann.vcf --candidates nobody=kid_clinical.vcf --par-bed par.bed --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-unsorted gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/unsorted.vcf report --members members-trio.tsv --relationships rel.tsv --vcf unsorted.vcf --out-dir out - content PYTHON_IMAGE +family report-trio-gz-no-par gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/trio.vcf.gz report --members members-trio.tsv --relationships rel.tsv --vcf trio.vcf.gz --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-low gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/low.vcf,family/par.bed report --members members-trio.tsv --relationships rel.tsv --vcf low.vcf --par-bed par.bed --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-phased gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/trio.vcf,family/phased.vcf,family/par.bed report --members members-trio.tsv --relationships rel.tsv --vcf trio.vcf --phased phased.vcf --par-bed par.bed --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-duo gen:family-oracle.sh,family/members-duo.tsv,family/rel-duo.tsv,family/duo.vcf,family/par.bed report --members members-duo.tsv --relationships rel-duo.tsv --vcf duo.vcf --par-bed par.bed --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-daughter-x gen:family-oracle.sh,family/members-daughter.tsv,family/rel.tsv,family/daughter.vcf,family/par.bed report --members members-daughter.tsv --relationships rel.tsv --vcf daughter.vcf --par-bed par.bed --out-dir out out/de_novo.tsv,out/compound_het.tsv,out/family_report.md,out/kid_inheritance.tsv.gz content PYTHON_IMAGE +family report-no-column gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/nocol.vcf report --members members-trio.tsv --relationships rel.tsv --vcf nocol.vcf --out-dir out - content PYTHON_IMAGE +family report-nochrom gen:family-oracle.sh,family/members-trio.tsv,family/rel.tsv,family/nochrom.vcf report --members members-trio.tsv --relationships rel.tsv --vcf nochrom.vcf --out-dir out - content PYTHON_IMAGE +family report-3gen gen:family-oracle.sh,family/members-3gen.tsv,family/rel-empty.tsv,gen:family-3gen.sh,family/par.bed report --members members-3gen.tsv --relationships rel-empty.tsv --vcf 3gen.vcf.gz --par-bed par.bed --out-dir out out/family_report.md,out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg,out/dad_segments.tsv,out/kid_inheritance.tsv.gz,out/dad_inheritance.tsv.gz content PYTHON_IMAGE +family report-3gen-duo gen:family-oracle.sh,family/members-3gen-duo.tsv,family/rel-empty.tsv,gen:family-3gen.sh,family/par.bed report --members members-3gen-duo.tsv --relationships rel-empty.tsv --vcf 3gen.vcf --par-bed par.bed --out-dir out out/family_report.md,out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg content PYTHON_IMAGE +family segments-3gen gen:family-oracle.sh,family/members-3gen.tsv,gen:family-3gen.sh,family/par.bed segments --members members-3gen.tsv --vcf 3gen.vcf --par-bed par.bed --out-dir out out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg,out/dad_segments.tsv content PYTHON_IMAGE +family segments-3gen-no-par gen:family-oracle.sh,family/members-3gen.tsv,gen:family-3gen.sh segments --members members-3gen.tsv --vcf 3gen.vcf.gz --out-dir out out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg content PYTHON_IMAGE +family segments-no-child gen:family-oracle.sh,family/members-duo.tsv,family/duo.vcf segments --members members-duo.tsv --vcf duo.vcf --out-dir out - content PYTHON_IMAGE +family report-bench gen:family-oracle.sh,family/members-bench.tsv,family/rel-empty.tsv,gen:family-bench.sh,family/par.bed report --members members-bench.tsv --relationships rel-empty.tsv --vcf bench.vcf.gz --par-bed par.bed --out-dir out out/family_report.md,out/KID_segments.tsv,out/KID_dnapainter.csv,out/KID_segments.svg,out/DAD_segments.tsv,out/KID_inheritance.tsv.gz,out/DAD_inheritance.tsv.gz content PYTHON_IMAGE diff --git a/tests/parity/gen/family-3gen.sh b/tests/parity/gen/family-3gen.sh new file mode 100644 index 000000000..5e10e51f8 --- /dev/null +++ b/tests/parity/gen/family-3gen.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# A three-generation family for og family's chromosome map (the fixture of +# tests/test_family_segments.py): grandparents gpa and gma, their son dad, +# mum, and their son kid. chr1 has a crossover in the father's meiosis at +# 20 Mb and three planted genotype errors, chr2 a 10 Mb gap with no site, +# chrX the son's single X from his mother. Writes 3gen.vcf (its genotypes +# drawn with awk's rand, so they differ between awk builds; both sides of a +# case read the same file) and 3gen.vcf.gz. +set -euo pipefail +awk 'BEGIN { + srand(38); OFS = "\t" + print "##fileformat=VCFv4.2" + print "##contig=" + print "##contig=" + print "##contig=" + print "##FORMAT=" + print "#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", "gpa", "gma", "dad", "mum", "kid" + span("chr1", 1000000, 41000000); span("chr2", 1000000, 11000000); span("chr2", 21000000, 31000000) + span("chrX", 5000000, 15000000) +} +function gt(a, b) { return a <= b ? a "/" b : b "/" a } +function r() { return rand() < 0.5 ? 0 : 1 } +function span(chrom, lo, hi, pos, gpa1, gpa2, gma1, gma2, mum1, mum2, planted, from_dad) { + for (pos = lo; pos < hi; pos += 2000) { + gpa1 = r(); gpa2 = r(); gma1 = r(); gma2 = r(); mum1 = r(); mum2 = r() + planted = chrom == "chr1" && (pos == 5000000 || pos == 9000000 || pos == 13000000) + if (planted) { gpa1 = 0; gpa2 = 0; gma1 = 1; gma2 = 1; mum1 = 0; mum2 = 0 } + if (chrom == "chrX") { + print chrom, pos, ".", "A", "G", "50", "PASS", ".", "GT", gpa1, gt(gma1, gma2), gma1, gt(mum1, mum2), mum1 + continue + } + from_dad = (chrom == "chr1" && pos < 20000000) ? gpa1 : gma1 + if (planted) from_dad = gma1 + print chrom, pos, ".", "A", "G", "50", "PASS", ".", "GT", gt(gpa1, gpa2), gt(gma1, gma2), gt(gpa1, gma1), gt(mum1, mum2), gt(from_dad, mum1) + } +}' > 3gen.vcf +gzip -n -c 3gen.vcf > 3gen.vcf.gz diff --git a/tests/parity/gen/family-bench.sh b/tests/parity/gen/family-bench.sh new file mode 100644 index 000000000..d016e06c6 --- /dev/null +++ b/tests/parity/gen/family-bench.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# A synthetic joint VCF for og family report at scale, five samples: GF and +# GM, their son DAD, MOM, and their daughter KID, so two children are +# mapped, KID's father's copy told apart by the grandparents. FAMILY_SITES +# sites (default 200,000) over chr1-22 and chrX, every 100 bases; KID's +# grandparental copy switches at random (about one site in 25), which gives +# the smoothing many short runs to absorb. Genotypes come from awk's rand +# (seed 1), so they differ between awk builds; both sides of a case read the +# same file. Writes bench.vcf.gz. +set -euo pipefail +awk -v n="${FAMILY_SITES:-200000}" 'BEGIN { + srand(1); OFS = "\t" + per = int(n / 23) + print "##fileformat=VCFv4.2" + for (c = 1; c <= 23; c++) printf "##contig=\n", name(c), per * 100 + 1000 + print "#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", "GF", "GM", "DAD", "MOM", "KID" + for (c = 1; c <= 23; c++) { + copy = 0 + for (i = 1; i <= per; i++) { + for (k = 0; k < 8; k++) h[k] = rand() < 0.4 ? 1 : 0 + if (rand() < 0.04) copy = 1 - copy + d0 = h[0]; d1 = h[3] + kd = copy ? d1 : d0 + print name(c), i * 100, ".", "A", "G", "50", "PASS", ".", "GT:DP:GQ:AD", g(h[0], h[1]), g(h[2], h[3]), g(d0, d1), g(h[4], h[5]), g(kd, h[5]) + } + } +} +function name(c) { return c == 23 ? "chrX" : "chr" c } +function g(a, b) { return a "/" b ":30:50:15,15" }' | gzip -n -1 > bench.vcf.gz diff --git a/tests/parity/gen/family-oracle.sh b/tests/parity/gen/family-oracle.sh new file mode 100644 index 000000000..198de3c3a --- /dev/null +++ b/tests/parity/gen/family-oracle.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# The oracle of og family: bin/family_report.py imports bin/family_segments.py, +# and oracle-diff.sh materialises one file per tool. This stages the second +# file, at the commit tests/parity/oracle.ref names, and oracle.sh, which +# puts both beside each other and runs the one og's verb names. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../../.." && pwd) +sha=$(awk -F'\t' '$1 == "family" { print $2 }' "${ROOT}/tests/parity/oracle.ref") +[ -n "$sha" ] || { echo "family-oracle.sh: no family row in oracle.ref" >&2; exit 1; } +git -C "$ROOT" cat-file -e "${sha}^{commit}" 2> /dev/null || git -C "$ROOT" fetch -q --depth=1 origin "$sha" +git -C "$ROOT" show "${sha}:bin/family_segments.py" > family_segments.py +cat > oracle.sh <<'SH' +# sh ./oracle.sh PROG ARGS: PROG is family_report.py; family_segments.py is staged here. +prog=$1 +shift +d=$(mktemp -d) +cp "$prog" "${d}/family_report.py" +cp ./family_segments.py "${d}/family_segments.py" +script=family_report.py +if [ "${1:-}" = segments ]; then + shift + script=family_segments.py +fi +rc=0 +PYTHONDONTWRITEBYTECODE=1 python3 "${d}/${script}" "$@" || rc=$? +rm -rf "$d" +exit "$rc" +SH diff --git a/tests/parity/golden/family/plan-missing-file/exit b/tests/parity/golden/family/plan-missing-file/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/family/plan-missing-file/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/family/plan-missing-file/stderr b/tests/parity/golden/family/plan-missing-file/stderr new file mode 100644 index 000000000..9f312fe2c --- /dev/null +++ b/tests/parity/golden/family/plan-missing-file/stderr @@ -0,0 +1 @@ +ERROR: cannot read nothere.tsv: No such file or directory (os error 2) diff --git a/tests/parity/golden/family/report-no-column/out/kid_inheritance.tsv.gz.absent b/tests/parity/golden/family/report-no-column/out/kid_inheritance.tsv.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-nochrom/out/kid_inheritance.tsv.gz.absent b/tests/parity/golden/family/report-nochrom/out/kid_inheritance.tsv.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/exit b/tests/parity/golden/family/report-unsorted/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/family/report-unsorted/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/family/report-unsorted/out/compound_het.tsv.absent b/tests/parity/golden/family/report-unsorted/out/compound_het.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/de_novo.tsv.absent b/tests/parity/golden/family/report-unsorted/out/de_novo.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/family_report.md.absent b/tests/parity/golden/family/report-unsorted/out/family_report.md.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/kid_dnapainter.csv.absent b/tests/parity/golden/family/report-unsorted/out/kid_dnapainter.csv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/kid_inheritance.tsv.gz.absent b/tests/parity/golden/family/report-unsorted/out/kid_inheritance.tsv.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/kid_segments.svg.absent b/tests/parity/golden/family/report-unsorted/out/kid_segments.svg.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/out/kid_segments.tsv.absent b/tests/parity/golden/family/report-unsorted/out/kid_segments.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/family/report-unsorted/stderr b/tests/parity/golden/family/report-unsorted/stderr new file mode 100644 index 000000000..6ddc3e780 --- /dev/null +++ b/tests/parity/golden/family/report-unsorted/stderr @@ -0,0 +1 @@ +ERROR: unsorted.vcf is not sorted: chr1:100 after chr1:300 diff --git a/tests/parity/golden/family/report-unsorted/stdout b/tests/parity/golden/family/report-unsorted/stdout new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/family/child_a.g.vcf b/tests/parity/inputs/family/child_a.g.vcf new file mode 100644 index 000000000..8c678cb87 --- /dev/null +++ b/tests/parity/inputs/family/child_a.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT child_a +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/dad.g.vcf b/tests/parity/inputs/family/dad.g.vcf new file mode 100644 index 000000000..b2e661414 --- /dev/null +++ b/tests/parity/inputs/family/dad.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/daughter.vcf b/tests/parity/inputs/family/daughter.vcf new file mode 100644 index 000000000..812e9ba1c --- /dev/null +++ b/tests/parity/inputs/family/daughter.vcf @@ -0,0 +1,7 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid mum +chrX 5000000 . A G 50 . . GT:DP:AD:GQ 1/1:30:15,15:50 1/1:30:15,15:50 0/0:30:15,15:50 +chrX 5000100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:14,16:50 0/0:30:15,15:50 +chrX 5000200 . A G 50 . . GT:DP:AD:GQ 1/1:30:15,15:50 0/1:30:15,15:50 0/0:30:15,15:50 diff --git a/tests/parity/inputs/family/duo.vcf b/tests/parity/inputs/family/duo.vcf new file mode 100644 index 000000000..f8fbd19ef --- /dev/null +++ b/tests/parity/inputs/family/duo.vcf @@ -0,0 +1,7 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT kid mum +chr1 100 . A G 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/0:30:15,15:50 +chr1 200 . A G 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 300 . A G 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 1/1:30:15,15:50 diff --git a/tests/parity/inputs/family/fam-allmissing.tsv b/tests/parity/inputs/family/fam-allmissing.tsv new file mode 100644 index 000000000..12b142f6b --- /dev/null +++ b/tests/parity/inputs/family/fam-allmissing.tsv @@ -0,0 +1,4 @@ +family sample sex father mother partner +. kid male dad mum . +. dad male . . mum +. mum female . . dad diff --git a/tests/parity/inputs/family/fam-famissample.tsv b/tests/parity/inputs/family/fam-famissample.tsv new file mode 100644 index 000000000..9d491b475 --- /dev/null +++ b/tests/parity/inputs/family/fam-famissample.tsv @@ -0,0 +1,3 @@ +family sample sex father mother partner +kid kid male . . . +kid mum female . . . diff --git a/tests/parity/inputs/family/fam-famname.tsv b/tests/parity/inputs/family/fam-famname.tsv new file mode 100644 index 000000000..aabc42cbc --- /dev/null +++ b/tests/parity/inputs/family/fam-famname.tsv @@ -0,0 +1,2 @@ +family sample sex father mother partner +f/a kid male . . . diff --git a/tests/parity/inputs/family/fam-parentother.tsv b/tests/parity/inputs/family/fam-parentother.tsv new file mode 100644 index 000000000..ce821aa60 --- /dev/null +++ b/tests/parity/inputs/family/fam-parentother.tsv @@ -0,0 +1,3 @@ +family sample sex father mother partner +fa kid male dad . . +fb dad male . . . diff --git a/tests/parity/inputs/family/fam-partial.tsv b/tests/parity/inputs/family/fam-partial.tsv new file mode 100644 index 000000000..5ec0f1e4d --- /dev/null +++ b/tests/parity/inputs/family/fam-partial.tsv @@ -0,0 +1,3 @@ +family sample sex father mother partner +fa kid male . . . +. mum female . . . diff --git a/tests/parity/inputs/family/fam-partnerother.tsv b/tests/parity/inputs/family/fam-partnerother.tsv new file mode 100644 index 000000000..175cba262 --- /dev/null +++ b/tests/parity/inputs/family/fam-partnerother.tsv @@ -0,0 +1,3 @@ +family sample sex father mother partner +fa dad male . . mum +fb mum female . . dad diff --git a/tests/parity/inputs/family/fam-twofam.tsv b/tests/parity/inputs/family/fam-twofam.tsv new file mode 100644 index 000000000..49f2ddfed --- /dev/null +++ b/tests/parity/inputs/family/fam-twofam.tsv @@ -0,0 +1,4 @@ +family sample sex father mother partner +fa kid male . . . +fb kid male . . . +fb mum female . . . diff --git a/tests/parity/inputs/family/father_a.g.vcf b/tests/parity/inputs/family/father_a.g.vcf new file mode 100644 index 000000000..2e68e7dc5 --- /dev/null +++ b/tests/parity/inputs/family/father_a.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT father_a +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/hg003.g.vcf b/tests/parity/inputs/family/hg003.g.vcf new file mode 100644 index 000000000..f6be1489e --- /dev/null +++ b/tests/parity/inputs/family/hg003.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG003 +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/kid.g.vcf b/tests/parity/inputs/family/kid.g.vcf new file mode 100644 index 000000000..da09dc30d --- /dev/null +++ b/tests/parity/inputs/family/kid.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT kid +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/kid_clinical.vcf b/tests/parity/inputs/family/kid_clinical.vcf new file mode 100644 index 000000000..7d7e41c1d --- /dev/null +++ b/tests/parity/inputs/family/kid_clinical.vcf @@ -0,0 +1,8 @@ +##fileformat=VCFv4.2 +##INFO= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT kid +chr2 100 . A G 50 PASS CSQ=G|missense_variant|MODERATE|G1 GT 0/1 +chr2 200 . C T 50 PASS CSQ=T|stop_gained|HIGH|G1,T|x|y|G3 GT 0/1 +chr3 100 . A G 50 PASS CSQ=G|missense_variant|MODERATE|G2 GT 0/1 +chr3 200 . C T 50 PASS CSQ=T|missense_variant|MODERATE|G2 GT 0/1 +chr1 100 . A G 50 PASS CSQ=G|missense_variant|MODERATE|G3 GT 0/1 diff --git a/tests/parity/inputs/family/low.vcf b/tests/parity/inputs/family/low.vcf new file mode 100644 index 000000000..fd30017b5 --- /dev/null +++ b/tests/parity/inputs/family/low.vcf @@ -0,0 +1,5 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid mum +chr1 300 . G A 50 . . GT:DP:AD:GQ 0/0:5:5,0:10 0/1:30:14,16:50 0/0:30:30,0:50 diff --git a/tests/parity/inputs/family/members-3gen-duo.tsv b/tests/parity/inputs/family/members-3gen-duo.tsv new file mode 100644 index 000000000..05ee2d425 --- /dev/null +++ b/tests/parity/inputs/family/members-3gen-duo.tsv @@ -0,0 +1,6 @@ +sample sex father mother partner in_pedigree sequenced +kid male dad mum yes yes +dad male gpa gma mum yes yes +mum female dad yes no +gpa male gma yes yes +gma female gpa yes yes diff --git a/tests/parity/inputs/family/members-3gen.tsv b/tests/parity/inputs/family/members-3gen.tsv new file mode 100644 index 000000000..d8c025550 --- /dev/null +++ b/tests/parity/inputs/family/members-3gen.tsv @@ -0,0 +1,6 @@ +sample sex father mother partner in_pedigree sequenced +kid male dad mum yes yes +dad male gpa gma mum yes yes +mum female dad yes yes +gpa male gma yes yes +gma female gpa yes yes diff --git a/tests/parity/inputs/family/members-bench.tsv b/tests/parity/inputs/family/members-bench.tsv new file mode 100644 index 000000000..59a5e3c35 --- /dev/null +++ b/tests/parity/inputs/family/members-bench.tsv @@ -0,0 +1,6 @@ +sample sex father mother partner in_pedigree sequenced +DAD male GF GM MOM yes yes +GF male yes yes +GM female yes yes +KID female DAD MOM yes yes +MOM female DAD yes yes diff --git a/tests/parity/inputs/family/members-daughter.tsv b/tests/parity/inputs/family/members-daughter.tsv new file mode 100644 index 000000000..2418abd6b --- /dev/null +++ b/tests/parity/inputs/family/members-daughter.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner in_pedigree sequenced +dad male grandpa mum yes yes +grandpa male no no +kid female dad mum yes yes +mum female dad yes yes diff --git a/tests/parity/inputs/family/members-duo.tsv b/tests/parity/inputs/family/members-duo.tsv new file mode 100644 index 000000000..91943ee91 --- /dev/null +++ b/tests/parity/inputs/family/members-duo.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner in_pedigree sequenced +dad male grandpa mum yes no +grandpa male no no +kid male dad mum yes yes +mum female dad yes yes diff --git a/tests/parity/inputs/family/members-swap.tsv b/tests/parity/inputs/family/members-swap.tsv new file mode 100644 index 000000000..80e359e41 --- /dev/null +++ b/tests/parity/inputs/family/members-swap.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner in_pedigree sequenced +dad male kid mum yes yes +grandpa male no no +kid male grandpa mum yes yes +mum female kid yes yes diff --git a/tests/parity/inputs/family/members-trio.tsv b/tests/parity/inputs/family/members-trio.tsv new file mode 100644 index 000000000..9eb6298af --- /dev/null +++ b/tests/parity/inputs/family/members-trio.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner in_pedigree sequenced +dad male grandpa mum yes yes +grandpa male no no +kid male dad mum yes yes +mum female dad yes yes diff --git a/tests/parity/inputs/family/mother_a.g.vcf b/tests/parity/inputs/family/mother_a.g.vcf new file mode 100644 index 000000000..6b423efd1 --- /dev/null +++ b/tests/parity/inputs/family/mother_a.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT mother_a +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/mum.g.vcf b/tests/parity/inputs/family/mum.g.vcf new file mode 100644 index 000000000..0bdd66e65 --- /dev/null +++ b/tests/parity/inputs/family/mum.g.vcf @@ -0,0 +1,3 @@ +##fileformat=VCFv4.2 +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT mum +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/mum_ann.vcf b/tests/parity/inputs/family/mum_ann.vcf new file mode 100644 index 000000000..2042bb6f7 --- /dev/null +++ b/tests/parity/inputs/family/mum_ann.vcf @@ -0,0 +1,4 @@ +##fileformat=VCFv4.2 +##INFO= +#CHROM POS ID REF ALT QUAL FILTER INFO +chr1 100 . A G 50 PASS ANN=G|missense|MODERATE|GA diff --git a/tests/parity/inputs/family/nochrom.g.vcf b/tests/parity/inputs/family/nochrom.g.vcf new file mode 100644 index 000000000..b46727823 --- /dev/null +++ b/tests/parity/inputs/family/nochrom.g.vcf @@ -0,0 +1,2 @@ +##fileformat=VCFv4.2 +chr1 1 . A <*> 0 . END=10 GT:GQ 0/0:50 diff --git a/tests/parity/inputs/family/nochrom.vcf b/tests/parity/inputs/family/nochrom.vcf new file mode 100644 index 000000000..c204596fa --- /dev/null +++ b/tests/parity/inputs/family/nochrom.vcf @@ -0,0 +1 @@ +##fileformat=VCFv4.2 diff --git a/tests/parity/inputs/family/nocol.vcf b/tests/parity/inputs/family/nocol.vcf new file mode 100644 index 000000000..2d050de92 --- /dev/null +++ b/tests/parity/inputs/family/nocol.vcf @@ -0,0 +1,6 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid +chr1 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 200 . C T 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 diff --git a/tests/parity/inputs/family/pairs-duo.tsv b/tests/parity/inputs/family/pairs-duo.tsv new file mode 100644 index 000000000..3c8fe10c0 --- /dev/null +++ b/tests/parity/inputs/family/pairs-duo.tsv @@ -0,0 +1,2 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +kid mum 0.5 1 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 diff --git a/tests/parity/inputs/family/pairs-few.tsv b/tests/parity/inputs/family/pairs-few.tsv new file mode 100644 index 000000000..e033a4fdf --- /dev/null +++ b/tests/parity/inputs/family/pairs-few.tsv @@ -0,0 +1,4 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +dad kid 0.49 2 100 0.9 1 1 1 1 1 1 1 300 0 0 -1 +kid mum 0.51 1 100 0.9 1 1 1 1 1 1 1 300 0 0 -1 +dad mum 0.01 600 100 0.9 1 1 1 1 1 1 1 300 0 0 -1 diff --git a/tests/parity/inputs/family/pairs-lost.tsv b/tests/parity/inputs/family/pairs-lost.tsv new file mode 100644 index 000000000..51020b9b4 --- /dev/null +++ b/tests/parity/inputs/family/pairs-lost.tsv @@ -0,0 +1,3 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +dad kid 0.49 2 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +kid mum 0.51 1 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 diff --git a/tests/parity/inputs/family/pairs-ok.tsv b/tests/parity/inputs/family/pairs-ok.tsv new file mode 100644 index 000000000..ac9214a95 --- /dev/null +++ b/tests/parity/inputs/family/pairs-ok.tsv @@ -0,0 +1,4 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +dad kid 0.49 2 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +kid mum 0.51 1 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +dad mum 0.01 600 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 diff --git a/tests/parity/inputs/family/pairs-sib.tsv b/tests/parity/inputs/family/pairs-sib.tsv new file mode 100644 index 000000000..7354738b0 --- /dev/null +++ b/tests/parity/inputs/family/pairs-sib.tsv @@ -0,0 +1,4 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +dad kid 0.5 400 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +kid mum 0.51 1 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +dad mum 0.01 600 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 diff --git a/tests/parity/inputs/family/pairs-warn.tsv b/tests/parity/inputs/family/pairs-warn.tsv new file mode 100644 index 000000000..f47cd25a1 --- /dev/null +++ b/tests/parity/inputs/family/pairs-warn.tsv @@ -0,0 +1,4 @@ +#sample_a sample_b relatedness ibs0 ibs2 concordance hets_a hets_b hets_ab shared_hets hom_alts_a hom_alts_b shared_hom_alts n x_ibs0 x_ibs2 expected_relatedness +dad kid 0.49 2 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +kid mum 0.51 1 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 +dad mum 0.2 6 100 0.9 1 1 1 1 1 1 1 5000 0 0 -1 diff --git a/tests/parity/inputs/family/par.bed b/tests/parity/inputs/family/par.bed new file mode 100644 index 000000000..4d40829a9 --- /dev/null +++ b/tests/parity/inputs/family/par.bed @@ -0,0 +1,4 @@ +chrX 10000 2781479 +chrX 155701382 156030895 +chrY 10000 2781479 +chrY 56887902 57217415 diff --git a/tests/parity/inputs/family/ped-badname.tsv b/tests/parity/inputs/family/ped-badname.tsv new file mode 100644 index 000000000..adf261f54 --- /dev/null +++ b/tests/parity/inputs/family/ped-badname.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner +kid male dad mum . +dad male grandpa . mum +mum female . . dad +-bad male . . . diff --git a/tests/parity/inputs/family/ped-badsex.tsv b/tests/parity/inputs/family/ped-badsex.tsv new file mode 100644 index 000000000..747d6bda9 --- /dev/null +++ b/tests/parity/inputs/family/ped-badsex.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid boy dad mum . +dad male grandpa . mum +mum female . . dad diff --git a/tests/parity/inputs/family/ped-crlf.tsv b/tests/parity/inputs/family/ped-crlf.tsv new file mode 100644 index 000000000..c8cb413de --- /dev/null +++ b/tests/parity/inputs/family/ped-crlf.tsv @@ -0,0 +1,7 @@ +#Sample Sex Father Mother Partner + + +kid male dad mum . +dad male grandpa . mum +mum female . . dad +# a comment diff --git a/tests/parity/inputs/family/ped-empty.tsv b/tests/parity/inputs/family/ped-empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/family/ped-femalefather.tsv b/tests/parity/inputs/family/ped-femalefather.tsv new file mode 100644 index 000000000..ba4259741 --- /dev/null +++ b/tests/parity/inputs/family/ped-femalefather.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male dad mum . +dad female grandpa . mum +mum female . . dad diff --git a/tests/parity/inputs/family/ped-loop.tsv b/tests/parity/inputs/family/ped-loop.tsv new file mode 100644 index 000000000..26aedc3d6 --- /dev/null +++ b/tests/parity/inputs/family/ped-loop.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male dad mum . +dad male kid . mum +mum female . . dad diff --git a/tests/parity/inputs/family/ped-nohead.tsv b/tests/parity/inputs/family/ped-nohead.tsv new file mode 100644 index 000000000..8065f1556 --- /dev/null +++ b/tests/parity/inputs/family/ped-nohead.tsv @@ -0,0 +1 @@ +kid male dad mum . diff --git a/tests/parity/inputs/family/ped-ownparent.tsv b/tests/parity/inputs/family/ped-ownparent.tsv new file mode 100644 index 000000000..a7326090d --- /dev/null +++ b/tests/parity/inputs/family/ped-ownparent.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male kid mum . +dad male grandpa . mum +mum female . . dad diff --git a/tests/parity/inputs/family/ped-partners.tsv b/tests/parity/inputs/family/ped-partners.tsv new file mode 100644 index 000000000..e05e1e8a0 --- /dev/null +++ b/tests/parity/inputs/family/ped-partners.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male dad mum . +dad male grandpa . mum +mum female . . kid diff --git a/tests/parity/inputs/family/ped-samefm.tsv b/tests/parity/inputs/family/ped-samefm.tsv new file mode 100644 index 000000000..cd690ffa0 --- /dev/null +++ b/tests/parity/inputs/family/ped-samefm.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male dad dad . +dad male grandpa . mum +mum female . . dad diff --git a/tests/parity/inputs/family/ped-secondrow.tsv b/tests/parity/inputs/family/ped-secondrow.tsv new file mode 100644 index 000000000..6bb72e45e --- /dev/null +++ b/tests/parity/inputs/family/ped-secondrow.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner +kid male dad mum . +dad male grandpa . mum +mum female . . dad +kid male . . . diff --git a/tests/parity/inputs/family/ped-toomany.tsv b/tests/parity/inputs/family/ped-toomany.tsv new file mode 100644 index 000000000..817583524 --- /dev/null +++ b/tests/parity/inputs/family/ped-toomany.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner +kid male dad mum . +dad male grandpa . mum +mum female . . dad +x male . . . extra diff --git a/tests/parity/inputs/family/ped-unknowncol.tsv b/tests/parity/inputs/family/ped-unknowncol.tsv new file mode 100644 index 000000000..ee760c47a --- /dev/null +++ b/tests/parity/inputs/family/ped-unknowncol.tsv @@ -0,0 +1,2 @@ +sample sex dad +kid male x diff --git a/tests/parity/inputs/family/pedigree.example.tsv b/tests/parity/inputs/family/pedigree.example.tsv new file mode 100644 index 000000000..814ddccba --- /dev/null +++ b/tests/parity/inputs/family/pedigree.example.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +child_a female father_a mother_a . +father_a male . . mother_a +mother_a female . . father_a diff --git a/tests/parity/inputs/family/phased.vcf b/tests/parity/inputs/family/phased.vcf new file mode 100644 index 000000000..a061faa0b --- /dev/null +++ b/tests/parity/inputs/family/phased.vcf @@ -0,0 +1,7 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid mum +chr2 100 . A G 50 . . GT:DP:AD:GQ:PS 0|1:30:15,15:50:1 1|0:30:15,15:50:1 0|0:30:15,15:50:1 +chr2 200 . C T 50 . . GT:DP:AD:GQ:PS 0|0:30:15,15:50:1 0|1:30:15,15:50:1 0|1:30:15,15:50:1 +chr2 300 . C T 50 . . GT:DP:AD:GQ:PS 0|0:30:15,15:50:1 1|0:30:15,15:50:1 0|1:30:15,15:50:1 diff --git a/tests/parity/inputs/family/rel-duo.tsv b/tests/parity/inputs/family/rel-duo.tsv new file mode 100644 index 000000000..88e4dce37 --- /dev/null +++ b/tests/parity/inputs/family/rel-duo.tsv @@ -0,0 +1,2 @@ +sample_a sample_b declared expected_relatedness relatedness ibs0 ibs0_rate ibs2 sites verdict reason +kid mum parent-child 0.500 0.500 1 0.0002 100 5000 ok first degree, as declared diff --git a/tests/parity/inputs/family/rel-empty.tsv b/tests/parity/inputs/family/rel-empty.tsv new file mode 100644 index 000000000..18d56d2c5 --- /dev/null +++ b/tests/parity/inputs/family/rel-empty.tsv @@ -0,0 +1 @@ +sample_a sample_b declared expected_relatedness relatedness ibs0 ibs0_rate ibs2 sites verdict reason diff --git a/tests/parity/inputs/family/rel.tsv b/tests/parity/inputs/family/rel.tsv new file mode 100644 index 000000000..cd763a10c --- /dev/null +++ b/tests/parity/inputs/family/rel.tsv @@ -0,0 +1,4 @@ +sample_a sample_b declared expected_relatedness relatedness ibs0 ibs0_rate ibs2 sites verdict reason +dad kid parent-child 0.500 0.490 2 0.0004 100 5000 ok first degree, as declared +dad mum partners 0.000 0.010 600 0.1200 100 5000 ok unrelated, as declared +kid mum parent-child 0.500 0.510 1 0.0002 100 5000 ok first degree, as declared diff --git a/tests/parity/inputs/family/trio.tsv b/tests/parity/inputs/family/trio.tsv new file mode 100644 index 000000000..46afd7b74 --- /dev/null +++ b/tests/parity/inputs/family/trio.tsv @@ -0,0 +1,4 @@ +sample sex father mother partner +kid male dad mum . +dad male grandpa . mum +mum female . . dad diff --git a/tests/parity/inputs/family/trio.vcf b/tests/parity/inputs/family/trio.vcf new file mode 100644 index 000000000..250b5666e --- /dev/null +++ b/tests/parity/inputs/family/trio.vcf @@ -0,0 +1,27 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid mum +chr1 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 200 . C T 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 300 . G A 50 . . GT:DP:AD:GQ 0/0:30:30,0:50 0/1:30:14,16:50 0/0:30:30,0:50 +chr1 400 . T C 50 . . GT:DP:AD:GQ 1/1:30:15,15:50 0/0:30:15,15:50 0/0:30:15,15:50 +chr1 500 . A C 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 1/1:30:15,15:50 0/1:30:15,15:50 +chr1 600 . A C,T 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 1/2:30:10,10,10:50 0/1:30:15,15:50 +chr1 700 . A C 50 . . GT:DP:AD:GQ ./.:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 800 . A C 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 ./.:30:15,15:50 0/1:30:15,15:50 +chr1 900 . A C 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 1:30:15,15:50 0/1:30:15,15:50 +chr2 100 . A G 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 0/0:30:15,15:50 +chr2 200 . C T 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr3 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr3 200 . C T 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 1/1:30:15,15:50 +chr3 300 . C T 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:3:.:7 0/0:30:15,15:50 +chrX 500000 . C T 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 0/0:30:15,15:50 +chrX 74000000 . G A 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 1/1:30:15,15:50 0/1:30:15,15:50 +chrX 74000100 . G A 50 . . GT:DP:AD:GQ 1/1:30:15,15:50 0/0:30:15,15:50 0/1:30:15,15:50 +chrX 74000200 . G A 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chrX 74000300 . G A 50 . . GT:DP:AD:GQ 0:30:15,15:50 1:30:15,15:50 0/0:30:15,15:50 +chrY 5000000 . C T 50 . . GT:DP:AD:GQ 1:30:15,15:50 1:30:15,15:50 ./.:30:15,15:50 +chrY 5000100 . C T 50 . . GT:DP:AD:GQ 0:30:15,15:50 1:30:15,15:50 ./.:30:15,15:50 +chrM 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/0:30:15,15:50 +chrUn_x 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/0:30:15,15:50 diff --git a/tests/parity/inputs/family/two_families.tsv b/tests/parity/inputs/family/two_families.tsv new file mode 100644 index 000000000..1893c148c --- /dev/null +++ b/tests/parity/inputs/family/two_families.tsv @@ -0,0 +1,7 @@ +family sample sex father mother partner +# Two invented families: a trio and a couple. +family_a child_a female father_a mother_a . +family_a father_a male . . mother_a +family_a mother_a female . . father_a +family_b partner_x male . . partner_y +family_b partner_y female . . partner_x diff --git a/tests/parity/inputs/family/unsorted.vcf b/tests/parity/inputs/family/unsorted.vcf new file mode 100644 index 000000000..d3897bfe6 --- /dev/null +++ b/tests/parity/inputs/family/unsorted.vcf @@ -0,0 +1,8 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT dad kid mum +chr1 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 200 . C T 50 . . GT:DP:AD:GQ 0/1:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 +chr1 300 . G A 50 . . GT:DP:AD:GQ 0/0:30:30,0:50 0/1:30:14,16:50 0/0:30:30,0:50 +chr1 100 . A G 50 . . GT:DP:AD:GQ 0/0:30:15,15:50 0/1:30:15,15:50 0/1:30:15,15:50 diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index ad4c6688b..464e1c07e 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -12,3 +12,4 @@ sv-inv-filter-py3 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/manta_inv_fi pgx-refblock-fill 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_refblock_fill.py python3 pgx refblock-fill pgx-mito-merge 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_mito_merge.py python3 pgx mito-merge chip 6f2407f9f66cc6bee43ae13c8abae5874b53e1cf file:bin/chip_report.py python3 chip +family 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/family_report.py sh ./oracle.sh family diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv new file mode 100644 index 000000000..a5770abc1 --- /dev/null +++ b/tests/parity/remnants.tsv @@ -0,0 +1,6 @@ +# Python files kept after their tool moved to og, only because other Python +# still imports names from them. `og dev remnants` checks that each holds the +# names listed here and nothing else, that every name has a user, and that +# the importers are exactly these (docs/rust.md). +# filenamesimporterslane +bin/family_report.py COLUMNS,MISSING,SEX,NAME,BINS,die,missing,read_pedigree,build_members,bad_name,kinship_table,declared_relationship,degree,read_par,in_par bin/annotsv_postfilter.py,bin/cnvpytor_to_vcf.py,bin/couple_report.py,bin/ibd_report.py,bin/pedigree_svg.py,tests/test_family_report.py R4 diff --git a/tests/test_family_report.py b/tests/test_family_report.py index be0e49100..54d279915 100644 --- a/tests/test_family_report.py +++ b/tests/test_family_report.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 -"""bin/family_report.py (step 38 and the Nextflow FAMILY workflow) on -synthetic pedigrees, somalier tables and joint VCFs. +"""og family (step 38 and the Nextflow FAMILY workflow) on synthetic +pedigrees, somalier tables and joint VCFs, and the pedigree rules other +Python tools still import from bin/family_report.py. + +Without og (OG unset) only check 2 runs, on bin/family_report.py. +With OG naming the og binary, every check runs. Checks: 1. prepare: a trio with a grandparent named but not sequenced writes @@ -21,9 +25,10 @@ outside the PARs taken from the mother only, inside a PAR from both, the allele of an unsequenced father inferred from a duo, compound heterozygotes in trans and in cis, WhatsHap's phase compared with the - genotype origin, and the same bytes on a second run. + genotype origin, and the same bytes on a second run; an unsorted joint + VCF stops it with exit 2 and writes nothing. -Run: python3 tests/test_family_report.py +Run: OG=rust/target/release/og python3 tests/test_family_report.py """ import gzip import os @@ -32,7 +37,7 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "family_report.py") +OG = os.environ.get("OG", "") PAR = os.path.join(REPO, "assets", "par_grch38.bed") sys.path.insert(0, os.path.join(REPO, "bin")) import family_report as fr # noqa: E402 @@ -47,7 +52,7 @@ def check(desc, ok, detail=""): def run(*args): - p = subprocess.run([sys.executable, SCRIPT] + list(args), capture_output=True, text=True) + p = subprocess.run([OG, "family"] + list(args), capture_output=True, text=True) return p.returncode, p.stdout + p.stderr @@ -245,9 +250,9 @@ def test_report(): ("chr2", 200, "C", "T", ["0/0", "0/1", "0/1"]), # gene G1, from mum -> in trans ("chr3", 100, "A", "G", ["0/0", "0/1", "0/1"]), # gene G2, from mum ("chr3", 200, "C", "T", ["0/0", "0/1", "1/1"]), # gene G2, from mum -> in cis + ("chrX", 500000, "C", "T", ["0/1", "0/1", "0/0"]), # PAR1: autosomal, from dad ("chrX", 74000000, "G", "A", ["0/0", "1/1", "0/1"]), # son's X: from mum only ("chrX", 74000100, "G", "A", ["1/1", "0/0", "0/1"]), # son's X: dad's allele does not count - ("chrX", 500000, "C", "T", ["0/1", "0/1", "0/0"]), # PAR1: autosomal, from dad ("chrM", 100, "A", "G", ["0/0", "0/1", "0/0"]), # left out ] with tempfile.TemporaryDirectory() as d: @@ -315,6 +320,14 @@ def got(k): "WhatsHap phased 2 heterozygous sites of kid by inheritance. At 2 of them the genotypes also give the " "origin, and the phase agrees with it at 2 (100.0%)" in md, md) + # An unsorted joint VCF would split the chromosome map wrongly: it stops, writing nothing + bad = joint(os.path.join(d, "unsorted.vcf.gz"), [rows[1], rows[0]]) + rc, log = run("report", "--members", members, "--relationships", rel, "--vcf", bad, "--out-dir", + os.path.join(d, "unsorted")) + check("an unsorted joint VCF stops the report (exit 2), naming the two records", + rc == 2 and "is not sorted: chr1:100 after chr1:200" in log + and not os.path.exists(os.path.join(d, "unsorted", "kid_inheritance.tsv.gz")), "exit %d: %s" % (rc, log)) + # A duo: the father is not sequenced; his allele is inferred with tempfile.TemporaryDirectory() as d: prepare(d, TRIO, samples=("kid", "mum")) @@ -398,7 +411,7 @@ def test_families_file(): check("plan --write-pedigree: %s gets its own pedigree (exit %d)" % (fam, rc), rc == 0 and [r["sample"] for r in rows] == want, out + str(rows)) check("%s's pedigree has the pedigree columns, no family column" % fam, - rows and list(rows[0]) == list(fr.PED_COLUMNS), str(rows[:1])) + rows and list(rows[0]) == ["sample", "sex", "father", "mother", "partner"], str(rows[:1])) rows, errors = fr.read_pedigree(os.path.join(d, "family_a", "pedigree.tsv")) members, more = fr.build_members(rows, {}, "x") check("the written pedigree reads back: child_a's parents are father_a and mother_a", @@ -469,7 +482,7 @@ def test_plan(): rc, log = plan("--families", TWO, "--sample", "partner_x", "--have-gvcf", "partner_x", "--out", out) p = by_analysis(out) if rc == 0 else {} check("plan writes the columns family, analysis, members, decision, reason", - rc == 0 and open(out).readline().rstrip("\n").split("\t") == list(fr.PLAN_COLUMNS), log) + rc == 0 and open(out).readline().rstrip("\n").split("\t") == ["family", "analysis", "members", "decision", "reason"], log) check("one member with a gVCF: the joint call waits for partner_y", p.get("joint_call", [{}])[0].get("decision") == "waiting" and p["joint_call"][0]["reason"] == "partner_y has no gVCF yet: run run-all.sh partner_y first", str(p)) @@ -546,8 +559,11 @@ def test_plan(): if __name__ == "__main__": - test_prepare() test_kinship() + if not OG: + print("skip: OG is not set (OG=rust/target/release/og): only the kinship check ran") + sys.exit(1 if FAILS or os.environ.get("OG_REQUIRED") else 0) + test_prepare() test_check() test_report() test_daughter_x() diff --git a/tests/test_family_segments.py b/tests/test_family_segments.py index 690cd089c..073e47e36 100755 --- a/tests/test_family_segments.py +++ b/tests/test_family_segments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/family_segments.py (step 38's chromosome map) on a synthetic +"""og family segments (step 38's chromosome map) on a synthetic three-generation family: two grandparents, their son (the father), the mother, and a grandchild. Genotypes are drawn from known haplotypes, so the truth is known. @@ -20,11 +20,13 @@ 3. the DNA Painter CSV has DNA Painter's own header, a row per segment, chromosome names without 'chr', and the grandparent as Ancestor; 4. the SVG is well-formed XML with a bar per segment; - 5. family_report.py report writes the files and a "Chromosome map" + 5. og family report writes the files and a "Chromosome map" section in family_report.md that counts the crossover; a second run writes the same bytes. -Run: python3 tests/test_family_segments.py +Needs og: without OG it skips (a failure with OG_REQUIRED set). + +Run: OG=rust/target/release/og python3 tests/test_family_segments.py """ import csv import os @@ -36,10 +38,8 @@ import xml.etree.ElementTree as ET REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -REPORT = os.path.join(REPO, "bin", "family_report.py") -SEGMENTS = os.path.join(REPO, "bin", "family_segments.py") +OG = os.environ.get("OG", "") PAR = os.path.join(REPO, "assets", "par_grch38.bed") -sys.path.insert(0, os.path.join(REPO, "bin")) FAILS = [] CROSSOVER = 20_000_000 @@ -116,8 +116,8 @@ def read_tsv(path): return [dict(zip(head, l.rstrip("\n").split("\t"))) for l in f] -def run(script, *args): - p = subprocess.run([sys.executable, script] + list(args), capture_output=True, text=True) +def run(verb, *args): + p = subprocess.run([OG, "family", verb] + list(args), capture_output=True, text=True) return p.returncode, p.stdout + p.stderr @@ -132,8 +132,8 @@ def test_trio(d): check("three genotype errors were planted on chr1", len(errors) == 3, errors) members = members_tsv(os.path.join(d, "members.tsv")) out = os.path.join(d, "trio") - rc, log = run(SEGMENTS, "--members", members, "--vcf", vcf, "--par-bed", PAR, "--out-dir", out) - check("family_segments.py exits 0", rc == 0, log) + rc, log = run("segments", "--members", members, "--vcf", vcf, "--par-bed", PAR, "--out-dir", out) + check("og family segments exits 0", rc == 0, log) path = os.path.join(out, "kid_segments.tsv") check("kid_segments.tsv written", os.path.exists(path), log) if not os.path.exists(path): @@ -198,7 +198,7 @@ def test_trio(d): def test_duo(d, vcf): members = members_tsv(os.path.join(d, "members_duo.tsv"), mum_sequenced=False) out = os.path.join(d, "duo") - rc, log = run(SEGMENTS, "--members", members, "--vcf", vcf, "--par-bed", PAR, "--out-dir", out) + rc, log = run("segments", "--members", members, "--vcf", vcf, "--par-bed", PAR, "--out-dir", out) check("duo: exits 0", rc == 0, log) path = os.path.join(out, "kid_segments.tsv") if not os.path.exists(path): @@ -223,7 +223,7 @@ def test_duo(d, vcf): def test_no_par(d, vcf, members): out = os.path.join(d, "nopar") - rc, log = run(SEGMENTS, "--members", members, "--vcf", vcf, "--out-dir", out) + rc, log = run("segments", "--members", members, "--vcf", vcf, "--out-dir", out) check("no --par-bed: exits 0", rc == 0, log) path = os.path.join(out, "kid_segments.tsv") rows = read_tsv(path) if os.path.exists(path) else [] @@ -237,7 +237,7 @@ def test_report(d, vcf, members): outs = [] for k in (1, 2): out = os.path.join(d, "report%d" % k) - rc, log = run(REPORT, "report", "--members", members, "--relationships", rel, "--vcf", vcf, + rc, log = run("report", "--members", members, "--relationships", rel, "--vcf", vcf, "--par-bed", PAR, "--out-dir", out) check("report run %d exits 0" % k, rc == 0, log[-2000:]) outs.append(out) @@ -256,6 +256,9 @@ def test_report(d, vcf, members): def main(): + if not OG: + print("skip: OG is not set (OG=rust/target/release/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) with tempfile.TemporaryDirectory() as d: res = test_trio(d) if res: From b18ab558f0bb338af14c364645d64d156ab1aa8a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:15 +0100 Subject: [PATCH 19/53] test(family): a parity case where the smoothing tie-break decides the map --- tests/parity/cases.tsv | 1 + tests/parity/inputs/family/ties.vcf | 59 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/parity/inputs/family/ties.vcf diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 6de2fabe8..300fcff4a 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -190,5 +190,6 @@ family report-3gen gen:family-oracle.sh,family/members-3gen.tsv,family/rel-empty family report-3gen-duo gen:family-oracle.sh,family/members-3gen-duo.tsv,family/rel-empty.tsv,gen:family-3gen.sh,family/par.bed report --members members-3gen-duo.tsv --relationships rel-empty.tsv --vcf 3gen.vcf --par-bed par.bed --out-dir out out/family_report.md,out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg content PYTHON_IMAGE family segments-3gen gen:family-oracle.sh,family/members-3gen.tsv,gen:family-3gen.sh,family/par.bed segments --members members-3gen.tsv --vcf 3gen.vcf --par-bed par.bed --out-dir out out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg,out/dad_segments.tsv content PYTHON_IMAGE family segments-3gen-no-par gen:family-oracle.sh,family/members-3gen.tsv,gen:family-3gen.sh segments --members members-3gen.tsv --vcf 3gen.vcf.gz --out-dir out out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg content PYTHON_IMAGE +family segments-ties gen:family-oracle.sh,family/members-3gen.tsv,family/ties.vcf segments --members members-3gen.tsv --vcf ties.vcf --out-dir out out/kid_segments.tsv,out/kid_dnapainter.csv,out/kid_segments.svg content PYTHON_IMAGE family segments-no-child gen:family-oracle.sh,family/members-duo.tsv,family/duo.vcf segments --members members-duo.tsv --vcf duo.vcf --out-dir out - content PYTHON_IMAGE family report-bench gen:family-oracle.sh,family/members-bench.tsv,family/rel-empty.tsv,gen:family-bench.sh,family/par.bed report --members members-bench.tsv --relationships rel-empty.tsv --vcf bench.vcf.gz --par-bed par.bed --out-dir out out/family_report.md,out/KID_segments.tsv,out/KID_dnapainter.csv,out/KID_segments.svg,out/DAD_segments.tsv,out/KID_inheritance.tsv.gz,out/DAD_inheritance.tsv.gz content PYTHON_IMAGE diff --git a/tests/parity/inputs/family/ties.vcf b/tests/parity/inputs/family/ties.vcf new file mode 100644 index 000000000..38b90c0a8 --- /dev/null +++ b/tests/parity/inputs/family/ties.vcf @@ -0,0 +1,59 @@ +##fileformat=VCFv4.2 +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT gpa gma dad mum kid +chr1 30000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 60000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 90000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 120000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 150000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 180000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 210000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 240000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 270000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 300000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 330000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 360000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 390000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 420000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 450000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 480000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 510000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 540000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 570000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 600000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 630000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 660000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 690000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 720000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 750000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 780000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 810000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 840000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 870000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 900000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 930000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/0 +chr1 960000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 990000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1020000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1050000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1080000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1110000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1140000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1170000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1200000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1230000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1260000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1290000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1320000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1350000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1380000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1410000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1440000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1470000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1500000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1530000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1560000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1590000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1620000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1650000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 +chr1 1680000 . A G 50 . . GT 0/0 1/1 0/1 0/0 0/1 From d619d2233d47dda5912d00fdc2314c1b1f3e17bd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:15 +0100 Subject: [PATCH 20/53] test(family): plant a tie-break fault and an extra remnant name (red-first) The weakest-run tie now goes to the last run in the block, and family_report.py defines a name no importer uses. The smoothing tests, the segments-ties parity case and og dev remnants must each go red. Reverted in the next commit. --- bin/family_report.py | 4 ++++ rust/crates/og-steps/src/family/segments.rs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/family_report.py b/bin/family_report.py index 044940c32..e7a62d43d 100755 --- a/bin/family_report.py +++ b/bin/family_report.py @@ -228,3 +228,7 @@ def read_par(path): def in_par(par, chrom, pos): # BED: 0-based start, end exclusive; pos is 1-based return any(s < pos <= e for s, e in par.get(chrom, ())) + + +def read_members(path): + return path diff --git a/rust/crates/og-steps/src/family/segments.rs b/rust/crates/og-steps/src/family/segments.rs index 593a00281..7420d8978 100644 --- a/rust/crates/og-steps/src/family/segments.rs +++ b/rust/crates/og-steps/src/family/segments.rs @@ -487,10 +487,10 @@ fn merge(lo: &Run, mid: &[&Run], hi: &Run, i_is: impl Fn(usize) -> bool, keep: u /// The tie-break of the weakest run: fewest sites, then shortest span, /// then the first in the block (`order` grows along it). -type Key = (usize, i64, usize); +type Key = (usize, i64, Reverse); fn key(r: &Run, order: usize) -> Key { - (r.agree, r.end - r.start, order) + (r.agree, r.end - r.start, Reverse(order)) } /// The smoothed runs of one block of (position, label): repeatedly absorb From dc77be81bcfb071fa9bdc8452b26462e3263fb46 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:16 +0100 Subject: [PATCH 21/53] feat(rust): og reads the seven opt-in step reports og vntyper report, og locityper report, og mocha report, og mei report and annotsv-input, og viral report, og y str-bed and str-report, and og exomiser prepare and report, in the new crate og-reports. Each reads its tool's output as the Python did and writes the same tables, through a temporary file and a rename. Bead: pgp-bs9u.17 --- rust/Cargo.lock | 12 + rust/Cargo.toml | 3 +- rust/crates/og-reports/Cargo.toml | 17 + rust/crates/og-reports/src/exomiser.rs | 1008 +++++++++++++++++++++++ rust/crates/og-reports/src/lib.rs | 265 ++++++ rust/crates/og-reports/src/locityper.rs | 467 +++++++++++ rust/crates/og-reports/src/mei.rs | 676 +++++++++++++++ rust/crates/og-reports/src/mocha.rs | 618 ++++++++++++++ rust/crates/og-reports/src/viral.rs | 600 ++++++++++++++ rust/crates/og-reports/src/vntyper.rs | 483 +++++++++++ rust/crates/og-reports/src/ystr.rs | 704 ++++++++++++++++ rust/og/Cargo.toml | 1 + rust/og/src/main.rs | 16 +- 13 files changed, 4868 insertions(+), 2 deletions(-) create mode 100644 rust/crates/og-reports/Cargo.toml create mode 100644 rust/crates/og-reports/src/exomiser.rs create mode 100644 rust/crates/og-reports/src/lib.rs create mode 100644 rust/crates/og-reports/src/locityper.rs create mode 100644 rust/crates/og-reports/src/mei.rs create mode 100644 rust/crates/og-reports/src/mocha.rs create mode 100644 rust/crates/og-reports/src/viral.rs create mode 100644 rust/crates/og-reports/src/vntyper.rs create mode 100644 rust/crates/og-reports/src/ystr.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 11c5e36b1..4437f10a7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -559,6 +559,7 @@ dependencies = [ "og-intervals", "og-io", "og-map", + "og-reports", "og-runlog", "og-steps", "og-vcf", @@ -642,6 +643,17 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-reports" +version = "0.1.0" +dependencies = [ + "indexmap", + "og-filters", + "og-io", + "og-tsv", + "pycompat", +] + [[package]] name = "og-runlog" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c9c55d60f..133558914 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-chip", "crates/og-reports", "og"] [workspace.package] version = "0.1.0" @@ -22,6 +22,7 @@ og-nf = { path = "crates/og-nf" } og-runlog = { path = "crates/og-runlog" } og-steps = { path = "crates/og-steps" } og-chip = { path = "crates/og-chip" } +og-reports = { path = "crates/og-reports" } indexmap = "2" flate2 = "1" memchr = "2" diff --git a/rust/crates/og-reports/Cargo.toml b/rust/crates/og-reports/Cargo.toml new file mode 100644 index 000000000..43d45f085 --- /dev/null +++ b/rust/crates/og-reports/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "og-reports" +description = "The opt-in step reports: og vntyper, locityper, mocha, mei, viral and exomiser, and og y str-bed/str-report" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +indexmap.workspace = true +og-filters.workspace = true +og-io.workspace = true +og-tsv.workspace = true +pycompat.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-reports/src/exomiser.rs b/rust/crates/og-reports/src/exomiser.rs new file mode 100644 index 000000000..1308d20b7 --- /dev/null +++ b/rust/crates/og-reports/src/exomiser.rs @@ -0,0 +1,1008 @@ +//! `og exomiser prepare` and `og exomiser report` (step 48): the +//! symptom-driven ranking of Exomiser. +//! +//! ```text +//! og exomiser prepare --hpo HP:0001156,HP:0001363 --sample S --vcf S.vcf.gz +//! --assembly hg38 --out-dir DIR --analysis S_analysis.yml +//! --terms S_exomiser_terms.tsv [--obo hp.obo] +//! [--pedigree pedigree.tsv --ped S_exomiser.ped +//! --affected a,b --unaffected c,d] +//! og exomiser report --sample S --variants S_exomiser.variants.tsv +//! --terms S_exomiser_terms.tsv +//! --out S_exomiser_top.tsv --text S_exomiser.txt [--top 10] +//! ``` +//! +//! `prepare` checks the HPO terms (HP: and seven digits; with `--obo`, each +//! must be a current term of the ontology Exomiser's data holds: an +//! alternative id is read as its term, an obsolete one stops the step with +//! its replacement) and writes the analysis file (JSON, which YAML reads): +//! the VCF, the terms, Exomiser's exome preset and the output options. With +//! `--pedigree` (step 38's pedigree.tsv) the VCF is the family's joint call +//! and `--ped` gets the PED file of its members: the proband and the +//! `--affected` are affected, the `--unaffected` are not, everyone else is +//! unknown. +//! +//! `report` writes the top `--top` ranks (a gene under one mode of +//! inheritance is one rank) with each variant that contributes to the +//! gene's score, in a table and in words. In a family run the PED beside +//! the variants names each member's genotype. +//! +//! What changes on purpose: the summary `report` prints on stdout is the +//! "Symptoms searched" line and one line per rank kept; the Python counted +//! variant rows, so a rank with several variants also printed the closing +//! note (stdout only; the text file is the same). + +use crate::{Error, Outcome}; +use indexmap::IndexMap; +use og_filters::args::Spec; +use og_io::Utf8; +use og_tsv::{Options, Quoting}; +use pycompat::json::{Dump, Value}; +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::path::Path; + +pub const TERMS_HEADER: [&str; 3] = ["hpo_id", "label", "given_as"]; +pub const OUT_HEADER: [&str; 16] = [ + "rank", + "gene", + "moi", + "exomiser_score", + "phenotype_score", + "variant_score", + "p_value", + "variant", + "genotype", + "consequence", + "hgvs", + "acmg", + "acmg_evidence", + "disease", + "clinvar", + "max_frequency", +]; +/// Columns of Exomiser's variants TSV (TsvVariantResultsWriter, 15.x). +pub const NEED: [&str; 23] = [ + "#RANK", + "GENE_SYMBOL", + "MOI", + "P-VALUE", + "EXOMISER_GENE_COMBINED_SCORE", + "EXOMISER_GENE_PHENO_SCORE", + "EXOMISER_GENE_VARIANT_SCORE", + "CONTRIBUTING_VARIANT", + "CONTIG", + "START", + "REF", + "ALT", + "GENOTYPE", + "FUNCTIONAL_CLASS", + "HGVS", + "EXOMISER_ACMG_CLASSIFICATION", + "EXOMISER_ACMG_EVIDENCE", + "EXOMISER_ACMG_DISEASE_ID", + "EXOMISER_ACMG_DISEASE_NAME", + "CLINVAR_PRIMARY_INTERPRETATION", + "CLINVAR_STAR_RATING", + "MAX_FREQ_SOURCE", + "MAX_FREQ", +]; + +fn moi_words(m: &str) -> &str { + match m { + "AD" => "autosomal dominant", + "AR" => "autosomal recessive", + "XD" => "X-linked dominant", + "XR" => "X-linked recessive", + "MT" => "mitochondrial", + "ANY" => "any mode of inheritance", + o => o, + } +} + +/// Exomiser's exome preset (examples/preset-exome-analysis.yml of 15.1.0). +const INHERITANCE: [(&str, f64); 7] = [ + ("AUTOSOMAL_DOMINANT", 0.1), + ("AUTOSOMAL_RECESSIVE_HOM_ALT", 0.1), + ("AUTOSOMAL_RECESSIVE_COMP_HET", 2.0), + ("X_DOMINANT", 0.1), + ("X_RECESSIVE_HOM_ALT", 0.1), + ("X_RECESSIVE_COMP_HET", 2.0), + ("MITOCHONDRIAL", 0.2), +]; +const FREQUENCY_SOURCES: [&str; 11] = [ + "UK10K", + "GNOMAD_E_AFR", + "GNOMAD_E_AMR", + "GNOMAD_E_EAS", + "GNOMAD_E_NFE", + "GNOMAD_E_SAS", + "GNOMAD_G_AFR", + "GNOMAD_G_AMR", + "GNOMAD_G_EAS", + "GNOMAD_G_NFE", + "GNOMAD_G_SAS", +]; +const PATHOGENICITY_SOURCES: [&str; 4] = ["REVEL", "MVP", "ALPHA_MISSENSE", "SPLICE_AI"]; +const REMOVED_EFFECTS: [&str; 11] = [ + "FIVE_PRIME_UTR_EXON_VARIANT", + "FIVE_PRIME_UTR_INTRON_VARIANT", + "THREE_PRIME_UTR_EXON_VARIANT", + "THREE_PRIME_UTR_INTRON_VARIANT", + "NON_CODING_TRANSCRIPT_EXON_VARIANT", + "NON_CODING_TRANSCRIPT_INTRON_VARIANT", + "CODING_TRANSCRIPT_INTRON_VARIANT", + "UPSTREAM_GENE_VARIANT", + "DOWNSTREAM_GENE_VARIANT", + "INTERGENIC_VARIANT", + "REGULATORY_REGION_VARIANT", +]; +/// Genes written to Exomiser's own files: the report shows `--top` of them. +const OUTPUT_GENES: i128 = 50; + +pub const NOTE: &str = "A ranking to bring to a clinical geneticist, not a diagnosis. A high score means a rare variant, predicted to be damaging, in a gene linked to symptoms like the ones given; it does not show that the variant causes them. A low score or an empty list does not rule a genetic cause out: this search reads coding and splice variants against the terms given, and not structural variants, repeat expansions, mitochondrial heteroplasmy or variants outside genes."; + +/// `re.split(r"[,\s]+", text)` without the empty items. +fn split_list(text: &str) -> Vec<&str> { + text.split(|c: char| c == ',' || pycompat::text::is_space(c)) + .filter(|x| !x.is_empty()) + .collect() +} + +/// `HP:` and seven ASCII digits. +fn is_term(t: &str) -> bool { + t.strip_prefix("HP:") + .is_some_and(|d| d.len() == 7 && d.bytes().all(|b| b.is_ascii_digit())) +} + +/// An HPO term of the OBO file. +#[derive(Debug, Clone, Default)] +struct Term { + name: String, + obsolete: bool, + replaced_by: String, + consider: Vec, +} + +/// The terms of an OBO file by id, and its alternative ids (alternative id to term id). +type Obo = (HashMap, HashMap); + +/// Reads an OBO file's `[Term]` stanzas. +fn read_obo(path: &str) -> Result { + #[derive(Default)] + struct Open { + id: Option, + term: Term, + alt: Vec, + } + let (mut terms, mut alt) = (HashMap::new(), HashMap::new()); + let mut close = |cur: Option| { + if let Some(c) = cur + && let Some(id) = c.id.filter(|i| i.starts_with("HP:")) + { + for a in c.alt { + alt.insert(a, id.clone()); + } + terms.insert(id, c.term); + } + }; + let mut cur: Option = None; + for raw in crate::read_lines(path, Utf8::Replace)? { + let line = raw.strip_suffix('\n').unwrap_or(&raw); + if line.starts_with('[') { + close(cur.take()); + cur = (line == "[Term]").then(Open::default); + continue; + } + let Some(c) = cur.as_mut() else { continue }; + let Some((k, v)) = line.split_once(": ") else { + continue; + }; + let v = crate::strip(v.split(" ! ").next().unwrap_or(v)).to_owned(); + match k { + "id" => c.id = Some(v), + "name" => c.term.name = v, + "replaced_by" => c.term.replaced_by = v, + "is_obsolete" => c.term.obsolete = v == "true", + "alt_id" => c.alt.push(v), + "consider" => c.term.consider.push(v), + _ => {} + } + } + close(cur); + Ok((terms, alt)) +} + +/// `(id, label, given_as)` of the given terms, in order, once each. +fn check_terms(given: &[&str], obo: Option<&str>) -> Result, Error> { + if given.is_empty() { + return Err(Error("no HPO term given: name the symptoms as HPO ids, e.g. HPO=HP:0001250,HP:0001263 (find them at https://hpo.jax.org)".into())); + } + let bad: Vec<&str> = given.iter().copied().filter(|t| !is_term(t)).collect(); + if !bad.is_empty() { + return Err(Error(format!( + "not an HPO id: {}. An HPO id is HP: and seven digits, e.g. HP:0001250 (Seizure)", + bad.join(", ") + ))); + } + let (terms, alt) = match obo { + Some(p) => read_obo(p)?, + None => Default::default(), + }; + let (mut out, mut seen, mut problems) = (Vec::new(), HashSet::new(), Vec::new()); + for &t in given { + let tid = alt.get(t).map_or(t, String::as_str); + let mut name = String::new(); + if let Some(path) = obo { + let Some(term) = terms.get(tid) else { + problems.push(format!("{t} is not a term of the HPO release in {path}")); + continue; + }; + if term.obsolete { + let hint = if !term.replaced_by.is_empty() { + format!(" use {} instead", term.replaced_by) + } else if !term.consider.is_empty() { + format!(" consider {}", term.consider.join(", ")) + } else { + " pick a current term".into() + }; + problems.push(format!( + "{t} ({}) is obsolete in this HPO release;{hint}", + term.name + )); + continue; + } + name.clone_from(&term.name); + } + if seen.insert(tid.to_owned()) { + out.push([tid.to_owned(), name, t.to_owned()]); + } + } + if !problems.is_empty() { + return Err(Error( + problems.join("; ") + ". Look the terms up at https://hpo.jax.org", + )); + } + Ok(out) +} + +/// The sample names of a VCF's `#CHROM` line. +fn vcf_samples(path: &str) -> Result, Error> { + let file = og_io::open(path).map_err(|e| Error::io(&e, path))?; + let mut lines = og_io::Lines::new(file, Utf8::Replace); + let mut line = String::new(); + while lines + .read_line(&mut line) + .map_err(|e| Error::io(&e, path))? + { + if line.starts_with("#CHROM") { + let l = line.strip_suffix('\n').unwrap_or(&line); + return Ok(l.split('\t').skip(9).map(str::to_owned).collect()); + } + if !line.starts_with('#') { + break; + } + } + Err(Error(format!("{path} has no #CHROM header line"))) +} + +/// Rows of step 38's pedigree TSV by sample, each a column-to-value map. +fn read_pedigree(path: &str) -> Result>, Error> { + let mut rows = HashMap::new(); + let mut header: Option> = None; + for raw in crate::lines(path, Utf8::Replace)? { + let raw = raw?; + let line = raw.trim_end_matches(['\r', '\n']); + if crate::strip(line).is_empty() { + continue; + } + let Some(h) = &header else { + let h: Vec = line + .trim_start_matches('#') + .split('\t') + .map(|c| pycompat::text::lower(crate::strip(c))) + .collect(); + if !h.iter().any(|c| c == "sample") { + return Err(Error(format!( + "{path}: the first line must be a header with a 'sample' column" + ))); + } + header = Some(h); + continue; + }; + if line.starts_with('#') { + continue; + } + let mut vals: Vec = line + .split('\t') + .map(|v| crate::strip(v).to_owned()) + .collect(); + if vals.len() < h.len() { + vals.resize(h.len(), String::new()); + } + let r: HashMap = h.iter().cloned().zip(vals).collect(); + rows.insert(r.get("sample").cloned().unwrap_or_default(), r); + } + if header.is_none() { + return Err(Error(format!("{path} is empty"))); + } + Ok(rows) +} + +/// The PED file Exomiser reads: one line per member of the VCF. +fn ped_text( + pedigree: &str, + members: &[String], + proband: &str, + affected: &[&str], + unaffected: &[&str], +) -> Result { + let rows = read_pedigree(pedigree)?; + let has = |n: &str| members.iter().any(|m| m == n); + let missing: Vec<&str> = members + .iter() + .map(String::as_str) + .filter(|m| !rows.contains_key(*m)) + .collect(); + if !missing.is_empty() { + return Err(Error(format!( + "the family VCF holds {}, which {pedigree} does not name", + missing.join(", ") + ))); + } + if !has(proband) { + return Err(Error(format!( + "{proband} is not a sample of the family VCF ({})", + members.join(", ") + ))); + } + for (label, names) in [("--affected", affected), ("--unaffected", unaffected)] { + let stray: Vec<&str> = names.iter().copied().filter(|n| !has(n)).collect(); + if !stray.is_empty() { + return Err(Error(format!( + "{label} names {}, not a member of the family VCF ({})", + stray.join(", "), + members.join(", ") + ))); + } + } + let mut both: Vec<&str> = affected + .iter() + .copied() + .filter(|a| unaffected.contains(a)) + .collect::>() + .into_iter() + .collect(); + both.sort_unstable(); + if !both.is_empty() { + return Err(Error(format!( + "{} is named both affected and unaffected", + both.join(", ") + ))); + } + if unaffected.contains(&proband) { + return Err(Error(format!( + "{proband} is the proband, the person the symptoms are of: it cannot be unaffected" + ))); + } + let mut out = String::new(); + for m in members { + let r = &rows[m]; + let get = |k: &str| r.get(k).map_or("", String::as_str); + let parent = |k: &str| if has(get(k)) { get(k) } else { "0" }; + let sex = match pycompat::text::lower(get("sex")).as_str() { + "male" => "1", + "female" => "2", + _ => "0", + }; + let status = if m == proband || affected.contains(&m.as_str()) { + "2" + } else if unaffected.contains(&m.as_str()) { + "1" + } else { + "0" + }; + out.push_str(&["family", m, parent("father"), parent("mother"), sex, status].join("\t")); + out.push('\n'); + } + Ok(out) +} + +fn s(v: &str) -> Value { + Value::Str(v.into()) +} + +fn obj(items: Vec<(&str, Value)>) -> Value { + Value::Object(items.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +fn strs(v: &[&str]) -> Value { + Value::Array(v.iter().map(|x| s(x)).collect()) +} + +/// The arguments of `prepare`. +pub struct Prepare<'a> { + pub hpo: &'a str, + pub obo: Option<&'a str>, + pub sample: &'a str, + pub vcf: &'a str, + pub assembly: &'a str, + pub out_dir: &'a str, + pub analysis: &'a str, + pub terms: &'a str, + pub pedigree: Option<&'a str>, + pub ped: Option<&'a str>, + pub affected: &'a str, + pub unaffected: &'a str, +} + +/// `og exomiser prepare`: writes the files; the stdout line. +pub fn prepare(a: &Prepare<'_>) -> Result { + let terms = check_terms(&split_list(a.hpo), a.obo)?; + let empty = || IndexMap::new(); + let steps = Value::Array( + [ + ("geneBlacklistFilter", Value::Object(empty())), + ("failedVariantFilter", Value::Object(empty())), + ("alleleBalanceFilter", Value::Object(empty())), + ( + "variantEffectFilter", + obj(vec![("remove", strs(&REMOVED_EFFECTS))]), + ), + ( + "frequencyFilter", + obj(vec![("maxFrequency", Value::Float(2.0))]), + ), + ( + "pathogenicityFilter", + obj(vec![("keepNonPathogenic", Value::Bool(true))]), + ), + ("inheritanceFilter", Value::Object(empty())), + ("omimPrioritiser", Value::Object(empty())), + ("hiPhivePrioritiser", Value::Object(empty())), + ] + .into_iter() + .map(|(k, v)| obj(vec![(k, v)])) + .collect(), + ); + let mut analysis = vec![ + ("genomeAssembly", s(a.assembly)), + ("vcf", s(a.vcf)), + ( + "hpoIds", + Value::Array(terms.iter().map(|t| s(&t[0])).collect()), + ), + ( + "inheritanceModes", + obj(INHERITANCE + .iter() + .map(|(k, v)| (*k, Value::Float(*v))) + .collect()), + ), + ("analysisMode", s("PASS_ONLY")), + ("frequencySources", strs(&FREQUENCY_SOURCES)), + ("pathogenicitySources", strs(&PATHOGENICITY_SOURCES)), + ("steps", steps), + ]; + let (affected, unaffected) = (split_list(a.affected), split_list(a.unaffected)); + let mut ped = None; + if let Some(pedigree) = a.pedigree { + let Some(out) = a.ped else { + return Err(Error( + "--pedigree needs --ped, the PED file to write".into(), + )); + }; + let text = ped_text( + pedigree, + &vcf_samples(a.vcf)?, + a.sample, + &affected, + &unaffected, + )?; + ped = Some((out, text)); + analysis.push(("ped", s(out))); + analysis.push(("proband", s(a.sample))); + } else if !affected.is_empty() || !unaffected.is_empty() { + return Err(Error("--affected and --unaffected name family members: they need --pedigree and the family VCF".into())); + } else { + let n = vcf_samples(a.vcf)?.len(); + if n != 1 { + return Err(Error(format!( + "{} has {n} samples; one person's VCF has one (a family VCF goes with --pedigree)", + a.vcf + ))); + } + } + let doc = obj(vec![ + ("analysis", obj(analysis)), + ( + "outputOptions", + obj(vec![ + ("outputContributingVariantsOnly", Value::Bool(false)), + ("numGenes", Value::Int(OUTPUT_GENES)), + ("outputDirectory", s(a.out_dir)), + ("outputFileName", s(&format!("{}_exomiser", a.sample))), + ("outputFormats", strs(&["TSV_VARIANT", "TSV_GENE", "HTML"])), + ]), + ), + ]); + let dump = Dump { + indent: Some(2), + ..Dump::default() + }; + let json = pycompat::json::dumps(&doc, &dump).map_err(|e| Error(e.to_string()))? + "\n"; + let mut tsv = TERMS_HEADER.join("\t") + "\n"; + for t in &terms { + tsv.push_str(&t.join("\t")); + tsv.push('\n'); + } + if let Some((path, text)) = &ped { + crate::write_atomic(path, text.as_bytes())?; + } + crate::write_atomic(a.analysis, json.as_bytes())?; + crate::write_atomic(a.terms, tsv.as_bytes())?; + let shown: Vec = terms + .iter() + .map(|t| { + if t[1].is_empty() { + t[0].clone() + } else { + format!("{} ({})", t[0], t[1]) + } + }) + .collect(); + Ok(format!("HPO terms: {}\n", shown.join(", "))) +} + +/// `(hpo_id, label)` of the terms TSV `prepare` wrote; the label is `None` +/// on a row that ends before it. +fn read_terms(path: &str) -> Result)>, Error> { + let (header, rows) = crate::read_table(path, Options::default().quoting(Quoting::None))?; + if rows.is_empty() || TERMS_HEADER.iter().any(|k| !header.iter().any(|h| h == k)) { + return Err(Error(format!( + "{path} has no term rows or lacks a column of {}", + TERMS_HEADER.join(", ") + ))); + } + Ok(rows + .iter() + .map(|r| { + ( + r.get("hpo_id").unwrap_or("None").to_owned(), + r.get("label").map(str::to_owned), + ) + }) + .collect()) +} + +/// `"%.3f" % float(v)`, "." when it is no number. +fn number(v: &str) -> String { + crate::float(v).map_or(".".into(), |x| crate::fixed(x, 3)) +} + +fn genotype_word(gt: &str) -> String { + match gt.replace('|', "/").as_str() { + "0/1" | "1/0" => "heterozygous".into(), + "1/1" => "homozygous".into(), + "1" => "hemizygous".into(), + "0/0" | "0" => "does not carry it".into(), + _ => gt.into(), + } +} + +/// One person's genotype in a word; a family's as each member's, named. +fn genotype_text(gt: &str, members: &[String]) -> String { + let gts: Vec<&str> = gt.split(':').collect(); + if members.len() > 1 && gts.len() == members.len() { + let each: Vec = members + .iter() + .zip(>s) + .map(|(m, g)| format!("{m} {}", genotype_word(g))) + .collect(); + return format!("[{}]", each.join(", ")); + } + genotype_word(gt) +} + +/// The members of the PED `prepare` wrote, in its line order; none when it +/// cannot be read. +fn ped_members(path: &Path) -> Vec { + crate::read_lines(&path.display().to_string(), Utf8::Replace) + .map(|lines| { + lines + .iter() + .filter_map(|l| { + let mut c = l.strip_suffix('\n').unwrap_or(l).split('\t'); + c.nth(1).map(str::to_owned) + }) + .collect() + }) + .unwrap_or_default() +} + +/// Exomiser's variants TSV: its rows, each checked to have one cell per +/// column. +fn read_variants(path: &str) -> Result, Error> { + let opts = Options::default() + .quoting(Quoting::None) + .utf8(Utf8::Replace); + let (header, rows) = crate::read_table(path, opts)?; + if rows.is_empty() && header.first().map(String::as_str) != Some("#RANK") { + return Err(Error(format!( + "{path} is not an Exomiser variants TSV (no #RANK header)" + ))); + } + let missing: Vec<&str> = NEED + .iter() + .copied() + .filter(|k| !header.iter().any(|h| h == k)) + .collect(); + if !missing.is_empty() { + return Err(Error(format!( + "{path} lacks the column{} {}: is it Exomiser 15's TSV_VARIANT output?", + if missing.len() > 1 { "s" } else { "" }, + missing.join(", ") + ))); + } + if let Some(i) = rows + .iter() + .position(|r| r.dict().extra().is_some() || NEED.iter().any(|k| r.get(k).is_none())) + { + return Err(Error(format!( + "{path} line {} does not have one field per column", + i + 2 + ))); + } + Ok(rows) +} + +/// One row of the top table, by [`OUT_HEADER`]. +type TopRow = [String; 16]; + +/// The contributing variants of the first `top` ranks, in rank order. +fn top_rows(rows: &[og_tsv::Row], top: i64) -> Result, Error> { + let (mut out, mut ranks) = (Vec::new(), Vec::::new()); + for r in rows { + let g = |k: &str| r.get(k).unwrap_or(""); + let or_dot = |k: &str| { + if g(k).is_empty() { + ".".to_owned() + } else { + g(k).to_owned() + } + }; + let rank = crate::int(g("#RANK")) + .ok_or_else(|| Error(format!("rank '{}' is not a number", g("#RANK"))))?; + if g("CONTRIBUTING_VARIANT") != "1" { + continue; + } + if !ranks.contains(&rank) { + if ranks.len() as i64 >= top { + continue; + } + ranks.push(rank); + } + let (name, id) = ( + g("EXOMISER_ACMG_DISEASE_NAME"), + g("EXOMISER_ACMG_DISEASE_ID"), + ); + let disease = match (name.is_empty(), id.is_empty()) { + (_, true) => name.to_owned(), + (false, false) => format!("{name} ({id})"), + (true, false) => id.to_owned(), + }; + let cv = g("CLINVAR_PRIMARY_INTERPRETATION"); + let stars = g("CLINVAR_STAR_RATING"); + let clinvar = if cv.is_empty() || cv == "NOT_PROVIDED" { + ".".to_owned() + } else { + format!( + "{cv} ({} star{})", + if stars.is_empty() { "0" } else { stars }, + if stars == "1" { "" } else { "s" } + ) + }; + let freq = if g("MAX_FREQ").is_empty() { + "not seen".to_owned() + } else { + format!("{} {}%", g("MAX_FREQ_SOURCE"), g("MAX_FREQ")) + }; + out.push([ + rank.to_string(), + g("GENE_SYMBOL").into(), + g("MOI").into(), + number(g("EXOMISER_GENE_COMBINED_SCORE")), + number(g("EXOMISER_GENE_PHENO_SCORE")), + number(g("EXOMISER_GENE_VARIANT_SCORE")), + or_dot("P-VALUE"), + format!("{}:{} {}>{}", g("CONTIG"), g("START"), g("REF"), g("ALT")), + or_dot("GENOTYPE"), + or_dot("FUNCTIONAL_CLASS"), + or_dot("HGVS"), + or_dot("EXOMISER_ACMG_CLASSIFICATION"), + or_dot("EXOMISER_ACMG_EVIDENCE"), + if disease.is_empty() { + ".".into() + } else { + disease + }, + clinvar, + freq, + ]); + } + Ok(out) +} + +/// The text file's lines: what was searched, each rank in a sentence, then +/// the note. Also the number of rank lines. +fn words( + rows: &[TopRow], + terms: &[(String, Option)], + members: &[String], +) -> (Vec, usize) { + let named: Vec = terms + .iter() + .map(|(id, label)| match label.as_deref() { + Some(l) if !l.is_empty() => format!("{l} ({id})"), + _ => id.clone(), + }) + .collect(); + let mut lines = vec![format!("Symptoms searched: {}.", named.join(", "))]; + if rows.is_empty() { + lines.push( + "No gene passed Exomiser's filters with a variant that fits a mode of inheritance." + .into(), + ); + } + let mut by_rank: IndexMap<&str, Vec<&TopRow>> = IndexMap::new(); + for r in rows { + by_rank.entry(r[0].as_str()).or_default().push(r); + } + for (rank, vs) in &by_rank { + let g = vs[0]; + let head = format!( + "{rank}. {} ({}): Exomiser score {} (symptom match {}, variant {}).", + g[1], + moi_words(&g[2]), + g[3], + g[4], + g[5] + ); + let parts: Vec = vs + .iter() + .map(|v| { + let mut p = format!( + "{} {}, {} {}, ACMG {}", + v[7], + genotype_text(&v[8], members), + v[9], + v[10], + pycompat::text::lower(&v[11].replace('_', " ")) + ); + if v[12] != "." { + p.push_str(&format!(" ({})", v[12])); + } + if v[13] != "." { + p.push_str(&format!(", linked disease {}", v[13])); + } + if v[14] != "." { + p.push_str(&format!( + ", ClinVar {}", + pycompat::text::lower(&v[14].replace('_', " ")) + )); + } + p.push_str(&format!(", highest population frequency {}", v[15])); + p + }) + .collect(); + lines.push(format!("{head} {}.", parts.join("; "))); + } + lines.push(NOTE.into()); + (lines, by_rank.len()) +} + +/// The arguments of `report`. +pub struct Report<'a> { + pub sample: &'a str, + pub variants: &'a str, + pub terms: &'a str, + pub out: &'a str, + pub text: &'a str, + pub top: i64, +} + +/// `og exomiser report`: writes the table and the text; the stdout lines. +pub fn report(a: &Report<'_>) -> Result { + let terms = read_terms(a.terms)?; + let rows = top_rows(&read_variants(a.variants)?, a.top)?; + let mut tsv = OUT_HEADER.join("\t") + "\n"; + for r in &rows { + tsv.push_str( + &r.iter() + .map(|c| c.replace('\t', " ")) + .collect::>() + .join("\t"), + ); + tsv.push('\n'); + } + crate::write_atomic(a.out, tsv.as_bytes())?; + let dir = std::path::absolute(a.variants) + .ok() + .and_then(|p| p.parent().map(Path::to_path_buf)) + .unwrap_or_default(); + let members = ped_members(&dir.join(format!("{}_exomiser.ped", a.sample))); + let (text, ranks) = words(&rows, &terms, &members); + crate::write_atomic(a.text, (text.join("\n") + "\n").as_bytes())?; + // The searched line and one line per rank kept. + let shown = if rows.is_empty() { 1 } else { 1 + ranks }; + Ok(text[..shown].join("\n") + "\n") +} + +const PREPARE: Spec = Spec { + prog: "og exomiser prepare", + usage: "usage: og exomiser prepare [-h] --hpo HPO [--obo OBO] --sample SAMPLE --vcf VCF [--assembly {hg38,hg19}] --out-dir OUT_DIR --analysis ANALYSIS --terms TERMS [--pedigree PEDIGREE] [--ped PED] [--affected AFFECTED] [--unaffected UNAFFECTED]\n", + options: &[ + "--hpo", + "--obo", + "--sample", + "--vcf", + "--assembly", + "--out-dir", + "--analysis", + "--terms", + "--pedigree", + "--ped", + "--affected", + "--unaffected", + ], + positionals: &[], + required: &[ + "--hpo", + "--sample", + "--vcf", + "--out-dir", + "--analysis", + "--terms", + ], +}; + +const REPORT: Spec = Spec { + prog: "og exomiser report", + usage: "usage: og exomiser report [-h] --sample SAMPLE --variants VARIANTS --terms TERMS --out OUT --text TEXT [--top TOP]\n", + options: &[ + "--sample", + "--variants", + "--terms", + "--out", + "--text", + "--top", + ], + positionals: &[], + required: &["--sample", "--variants", "--terms", "--out", "--text"], +}; + +/// `og exomiser VERB ARGS`: the exit code (1 for a problem, 2 for a +/// refused command line). +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let fail = |r: Result| -> Outcome { r.map_err(|e| (e, 1)) }; + match verb { + Some("prepare") => crate::run(&PREPARE, rest, out, err, |a| { + let assembly = a.get("--assembly").unwrap_or("hg38"); + if assembly != "hg38" && assembly != "hg19" { + return Err(og_filters::args::ArgError::Usage(format!( + "argument --assembly: invalid choice: {} (choose from 'hg38', 'hg19')", + og_filters::py_repr(assembly) + ))); + } + Ok(fail(prepare(&Prepare { + hpo: crate::opt(a, "--hpo"), + obo: a.get("--obo"), + sample: crate::opt(a, "--sample"), + vcf: crate::opt(a, "--vcf"), + assembly, + out_dir: crate::opt(a, "--out-dir"), + analysis: crate::opt(a, "--analysis"), + terms: crate::opt(a, "--terms"), + pedigree: a.get("--pedigree").filter(|p| !p.is_empty()), + ped: a.get("--ped").filter(|p| !p.is_empty()), + affected: a.get("--affected").unwrap_or(""), + unaffected: a.get("--unaffected").unwrap_or(""), + }))) + }), + Some("report") => crate::run(&REPORT, rest, out, err, |a| { + let top = crate::int_opt(a, "--top", 10)?; + Ok(fail(report(&Report { + sample: crate::opt(a, "--sample"), + variants: crate::opt(a, "--variants"), + terms: crate::opt(a, "--terms"), + out: crate::opt(a, "--out"), + text: crate::opt(a, "--text"), + top, + }))) + }), + other => crate::unknown_verb("exomiser", "prepare|report", other, err), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + const OBO: &str = "format-version: 1.2\n\n[Term]\nid: HP:0001250\nname: Seizure\nalt_id: HP:0002279\n\n[Term]\nid: HP:0000001\nname: Old\nis_obsolete: true\nreplaced_by: HP:0001250 ! Seizure\n\n[Typedef]\nid: part_of\n"; + + #[test] + fn terms_against_the_ontology() { + let d = Dir::new("exo-obo"); + let obo = d.put("hp.obo", OBO); + let t = check_terms(&["HP:0002279", "HP:0001250"], Some(&obo)).unwrap(); + assert_eq!( + t, + [[ + "HP:0001250".to_owned(), + "Seizure".into(), + "HP:0002279".into() + ]] + ); + let e = check_terms(&["HP:0000001"], Some(&obo)).unwrap_err(); + assert_eq!( + e.0, + "HP:0000001 (Old) is obsolete in this HPO release; use HP:0001250 instead. Look the terms up at https://hpo.jax.org" + ); + assert!( + check_terms(&["HP:123"], None) + .unwrap_err() + .0 + .starts_with("not an HPO id: HP:123.") + ); + assert_eq!(split_list(" HP:1,HP:2\tHP:3 ,, "), ["HP:1", "HP:2", "HP:3"]); + } + + const VHEAD: &str = "#RANK\tID\tGENE_SYMBOL\tENTREZ_GENE_ID\tMOI\tP-VALUE\tEXOMISER_GENE_COMBINED_SCORE\tEXOMISER_GENE_PHENO_SCORE\tEXOMISER_GENE_VARIANT_SCORE\tEXOMISER_VARIANT_SCORE\tCONTRIBUTING_VARIANT\tWHITELIST_VARIANT\tFUNCTIONAL_CLASS\tHGVS\tEXOMISER_ACMG_CLASSIFICATION\tEXOMISER_ACMG_EVIDENCE\tEXOMISER_ACMG_DISEASE_ID\tEXOMISER_ACMG_DISEASE_NAME\tCLINVAR_VARIANT_ID\tCLINVAR_PRIMARY_INTERPRETATION\tCLINVAR_STAR_RATING\tGENE_CONSTRAINT_LOEUF\tGENE_CONSTRAINT_LOEUF_LOWER\tGENE_CONSTRAINT_LOEUF_UPPER\tMAX_FREQ_SOURCE\tMAX_FREQ\tALL_FREQ\tMAX_PATH_SOURCE\tMAX_PATH\tALL_PATH\tCONTIG\tSTART\tEND\tREF\tALT\tGENOTYPE\n"; + + fn v(rank: u32, gene: &str, contributing: &str, start: u32) -> String { + format!( + "{rank}\tid\t{gene}\t1\tAD\t0.01\t0.9\t0.8\t0.7\t0.7\t{contributing}\t0\tmissense_variant\t{gene}:c.1A>G\tLIKELY_PATHOGENIC\tPM2\tOMIM:1\tA disease\t1\tPATHOGENIC\t2\t.\t.\t.\tGNOMAD_E_NFE\t0.01\t.\t.\t.\t.\t1\t{start}\t{start}\tA\tG\t0/1\n" + ) + } + + /// Red first: a variant Exomiser marks 0 does not contribute. + #[test] + fn red_first_a_non_contributing_variant_is_left_out() { + let d = Dir::new("exo-top"); + let p = d.put( + "v.tsv", + &format!( + "{VHEAD}{}{}{}{}", + v(1, "A", "1", 10), + v(1, "A", "1", 20), + v(1, "A", "0", 30), + v(2, "B", "1", 40) + ), + ); + let rows = top_rows(&read_variants(&p).unwrap(), 10).unwrap(); + assert_eq!( + rows.iter().map(|r| r[7].as_str()).collect::>(), + ["1:10 A>G", "1:20 A>G", "1:40 A>G"] + ); + assert_eq!(rows[0][14], "PATHOGENIC (2 stars)"); + assert_eq!(rows[0][15], "GNOMAD_E_NFE 0.01%"); + let rows1 = top_rows(&read_variants(&p).unwrap(), 1).unwrap(); + assert_eq!(rows1.len(), 2); + let (text, ranks) = words(&rows, &[("HP:0001250".into(), Some("Seizure".into()))], &[]); + assert_eq!(ranks, 2); + assert_eq!(text[0], "Symptoms searched: Seizure (HP:0001250)."); + assert!(text[1].starts_with("1. A (autosomal dominant): Exomiser score 0.900 (symptom match 0.800, variant 0.700). 1:10 A>G heterozygous, missense_variant A:c.1A>G, ACMG likely pathogenic (PM2), linked disease A disease (OMIM:1), ClinVar pathogenic (2 stars), highest population frequency GNOMAD_E_NFE 0.01%; 1:20")); + assert_eq!(text.len(), 4); + } + + #[test] + fn family_genotypes_are_named() { + let m = ["kid".to_owned(), "mum".into()]; + assert_eq!( + genotype_text("0/1:0/0", &m), + "[kid heterozygous, mum does not carry it]" + ); + assert_eq!(genotype_text("1|1", &[]), "homozygous"); + } +} diff --git a/rust/crates/og-reports/src/lib.rs b/rust/crates/og-reports/src/lib.rs new file mode 100644 index 000000000..f92ddc157 --- /dev/null +++ b/rust/crates/og-reports/src/lib.rs @@ -0,0 +1,265 @@ +//! The opt-in step reports: each one reads one third-party tool's output +//! and writes OwnGenome's table of it. +//! +//! - [`vntyper`] (`og vntyper report`, step 52): VNtyper's MUC1 VNTR call. +//! - [`locityper`] (`og locityper report`, step 53): Locityper's genotypes. +//! - [`mocha`] (`og mocha report`, step 51): MoChA's mosaic alterations. +//! - [`mei`] (`og mei report`, `og mei annotsv-input`, step 50): MEGAnE's +//! mobile element insertions and the known disease insertions. +//! - [`viral`] (`og viral report`, step 47): viral DNA in the unmapped reads. +//! - [`ystr`] (`og y str-bed`, `og y str-report`, step 45): Y-STR counts. +//! - [`exomiser`] (`og exomiser prepare`, `og exomiser report`, step 48). +//! +//! Each one writes what the Python it replaced wrote, cell for cell +//! (`tests/parity`), except where its module names a fix. What every tool +//! changes on purpose: an input it cannot read is one `ERROR:` line on +//! stderr and the tool's exit code, never a traceback, and each output is +//! written whole, through a temporary file and a rename, or not at all. + +pub mod exomiser; +pub mod locityper; +pub mod mei; +pub mod mocha; +pub mod viral; +pub mod vntyper; +pub mod ystr; + +use og_filters::args::{self, Parsed, Spec}; +use og_io::Utf8; +use std::fmt; +use std::io::{self, Write}; + +/// Why a tool stopped: the message after `ERROR: `. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub String); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl From for Error { + /// A file that cannot be opened is worded as Python's `OSError`; any + /// other error as og-tsv words it. + fn from(e: og_tsv::Error) -> Self { + match &e { + og_tsv::Error::Io { + path, + line: None, + source, + } => Error::io(source, path), + _ => Error(e.to_string()), + } + } +} + +impl Error { + /// An I/O error on `path`, worded as Python's `OSError` + /// (`[Errno 2] No such file or directory: 'x.tsv'`). + pub fn io(e: &io::Error, path: &str) -> Error { + if e.kind() == io::ErrorKind::InvalidData { + Error(format!("{path}: {e}")) + } else { + Error(og_filters::os_error(e, path)) + } + } +} + +/// The lines of `path` as Python's `open()` iterates them, one at a time: +/// universal newlines, each line with its `\n` (none on a last line without +/// one). Plain or gzip. A read error ends the iteration with that error. +pub(crate) fn lines(path: &str, utf8: Utf8) -> Result { + let file = og_io::open(path).map_err(|e| Error::io(&e, path))?; + Ok(LineIter { + lines: Some(og_io::Lines::new(file, utf8)), + path: path.to_owned(), + }) +} + +/// [`lines`]: the lines of one file. +pub(crate) struct LineIter { + lines: Option>>, + path: String, +} + +impl Iterator for LineIter { + type Item = Result; + + fn next(&mut self) -> Option { + let lines = self.lines.as_mut()?; + let mut line = String::new(); + match lines.read_line(&mut line) { + Ok(true) => Some(Ok(line)), + Ok(false) => { + self.lines = None; + None + } + Err(e) => { + self.lines = None; + Some(Err(Error::io(&e, &self.path))) + } + } + } +} + +/// Every line of a small file, as [`lines`] reads them. +pub(crate) fn read_lines(path: &str, utf8: Utf8) -> Result, Error> { + lines(path, utf8)?.collect() +} + +/// A header-keyed TSV, read whole: its header (empty for an empty file) +/// and its rows. +pub(crate) fn read_table( + path: &str, + opts: og_tsv::Options, +) -> Result<(Vec, Vec), Error> { + let mut t = opts.open(path)?; + let header = t.header()?.map(<[String]>::to_vec).unwrap_or_default(); + let rows = t.collect::, _>>()?; + Ok((header, rows)) +} + +/// Writes `body` to `path` whole, through a temporary file and a rename. +pub(crate) fn write_atomic(path: &str, body: &[u8]) -> Result<(), Error> { + let fail = |e: io::Error| Error::io(&e, path); + let mut f = og_io::AtomicFile::create(path).map_err(fail)?; + f.write_all(body).map_err(fail)?; + f.commit().map_err(fail) +} + +/// Python's `s.strip()`. +pub(crate) fn strip(s: &str) -> &str { + pycompat::text::strip(s, None) +} + +/// `f"{x:.{prec}f}"`. +pub(crate) fn fixed(x: f64, prec: usize) -> String { + pycompat::fmt::fixed(x, prec) +} + +/// `s.isdigit()` for ASCII digits only. Python's also takes other Unicode +/// digits (`²`), which its `int()` then refuses with a traceback; here they +/// are not digits. +pub(crate) fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) +} + +/// `int(s)` for a string [`digits`] accepted. +pub(crate) fn int(s: &str) -> Option { + pycompat::num::parse_int(s).ok() +} + +/// `float(s)`, or `None` where Python raises `ValueError`. +pub(crate) fn float(s: &str) -> Option { + pycompat::num::parse_float(s).ok() +} + +/// What a tool's run gave: its stdout text, or the message and exit code it +/// stops with. +pub(crate) type Outcome = Result; + +/// Parses `rest` by `spec` and runs `body`, printing its stdout or its +/// `ERROR:` line; the exit code. +pub(crate) fn run( + spec: &Spec, + rest: &[String], + out: &mut dyn Write, + err: &mut dyn Write, + body: impl FnOnce(&Parsed) -> Result, +) -> u8 { + let a = match args::parse(spec, rest) { + Ok(a) => a, + Err(e) => return args::report(spec, &e), + }; + match body(&a) { + Err(e) => args::report(spec, &e), + Ok(Ok(text)) => match out.write_all(text.as_bytes()).and_then(|()| out.flush()) { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "ERROR: cannot write stdout: {e}"); + 1 + } + }, + Ok(Err((e, code))) => { + let _ = writeln!(err, "ERROR: {e}"); + code + } + } +} + +/// `--name` as argparse's `type=int` reads it, or `default`. +pub(crate) fn int_opt(a: &Parsed, name: &str, default: i64) -> Result { + a.typed(name, default, "int", |s| { + args::int(s).and_then(|v| i64::try_from(v).ok()) + }) +} + +/// The value of a required option (the parser has checked it is there). +pub(crate) fn opt<'a>(a: &'a Parsed, name: &str) -> &'a str { + a.get(name).unwrap_or_default() +} + +/// The usage line of an area with several verbs, and its refusal. +pub(crate) fn unknown_verb(area: &str, verbs: &str, verb: Option<&str>, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "usage: og {area} {verbs} [args]"); + match verb { + Some(v) => { + let _ = writeln!(err, "og {area}: error: unknown command '{v}'"); + } + None => { + let _ = writeln!( + err, + "og {area}: error: the following arguments are required: command" + ); + } + } + 2 +} + +#[cfg(test)] +pub(crate) mod testutil { + use std::path::{Path, PathBuf}; + + /// A scratch directory removed when dropped. + pub struct Dir(pub PathBuf); + + impl Dir { + pub fn new(tag: &str) -> Dir { + let p = std::env::temp_dir().join(format!( + "og-reports-{tag}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + Dir(p) + } + + /// Writes `text` to `name` (parents made) and returns its path. + pub fn put(&self, name: &str, text: &str) -> String { + let p = self.0.join(name); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, text).unwrap(); + p.display().to_string() + } + + pub fn path(&self, name: &str) -> String { + self.0.join(name).display().to_string() + } + + pub fn read(&self, name: &str) -> String { + std::fs::read_to_string(self.0.join(name)).unwrap() + } + + pub fn root(&self) -> &Path { + &self.0 + } + } + + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } +} diff --git a/rust/crates/og-reports/src/locityper.rs b/rust/crates/og-reports/src/locityper.rs new file mode 100644 index 000000000..c61f82505 --- /dev/null +++ b/rust/crates/og-reports/src/locityper.rs @@ -0,0 +1,467 @@ +//! `og locityper report` (step 53): one row per locus from what Locityper +//! genotype wrote. +//! +//! ```text +//! og locityper report --gt-dir DIR --loci BED --exit-code N --out TSV +//! ``` +//! +//! DIR is the output folder of `locityper genotype` (`DIR/loci// +//! res.json.gz`, with a `success` file beside each finished locus). BED is +//! the locus list the database was built from (`assets/locityper_loci.bed`: +//! chrom, start, end, name; `#` lines are comments). One row per BED locus, +//! in the BED's order: locus, region, status (called, or not called with +//! the reason), genotype and its two haplotypes, Locityper's quality, the +//! reads used and unexplained, the weighted distance, Locityper's warnings, +//! and the filter of the authors' HLA and KIR benchmark (Prodanov et al. +//! 2025, Zenodo 15879519 README): no warning, a weighted distance of 30 or +//! less, and fewer than 1000 or fewer than 20% unexplained reads. +//! +//! A locus Locityper did not finish is "not called", never a genotype. Exit +//! 1 only when the BED cannot be read. +//! +//! What changes on purpose: a `res.json.gz` whose JSON is not an object, +//! or whose genotype is not text, is "not called" with that reason (the +//! Python stopped with a traceback), and a BED coordinate or a read count +//! must be ASCII digits (the Python's `isdigit` also took `²`, then +//! crashed on it). + +use crate::{Error, Outcome}; +use og_filters::args::Spec; +use og_io::Utf8; +use pycompat::json::Value; +use std::io::{Read, Write}; +use std::path::Path; + +pub const COLUMNS: [&str; 13] = [ + "locus", + "region", + "status", + "genotype", + "haplotype_1", + "haplotype_2", + "quality", + "total_reads", + "unexplained_reads", + "weighted_distance", + "warnings", + "filter", + "reason", +]; + +const MAX_WEIGHT_DIST: f64 = 30.0; +const MAX_UNEXPLAINED: u128 = 1000; +const MAX_UNEXPLAINED_SHARE: f64 = 0.2; + +/// The loci of the BED: (name, `chrom:start+1-end`). +pub fn read_loci(path: &str) -> Result, Error> { + let mut loci = Vec::new(); + for (n, line) in crate::read_lines(path, Utf8::Strict)?.iter().enumerate() { + if crate::strip(line).is_empty() || line.starts_with('#') { + continue; + } + let c: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(line) + .split('\t') + .collect(); + let start = (c.len() >= 4 && crate::digits(c[1]) && crate::digits(c[2])) + .then(|| crate::int(c[1])) + .flatten(); + let Some(start) = start else { + return Err(Error(format!( + "{path}:{}: want chrom, start, end, name (tab separated)", + n + 1 + ))); + }; + loci.push((c[3].to_owned(), format!("{}:{}-{}", c[0], start + 1, c[2]))); + } + if loci.is_empty() { + return Err(Error(format!("{path} lists no locus"))); + } + Ok(loci) +} + +/// `float(v)` of a JSON value, `None` where Python raises. +fn as_float(v: Option<&Value>) -> Option { + match v? { + Value::Bool(b) => Some(f64::from(u8::from(*b))), + Value::Int(i) => Some(*i as f64), + Value::Float(x) => Some(*x), + Value::Str(s) => crate::float(s), + _ => None, + } +} + +/// A number of `res.json.gz` with `digits` decimals; "." when it is absent, +/// no number, or NaN. +fn fmt(v: Option<&Value>, digits: usize) -> String { + match as_float(v) { + Some(x) if !x.is_nan() => crate::fixed(x, digits), + _ => ".".into(), + } +} + +/// A read count as a whole number, `None` when it is absent or not one. +fn count(res: &Value, key: &str) -> Option { + match pycompat::value::get(res, key).ok()?? { + Value::Int(i) if *i >= 0 => Some(i.to_string()), + Value::Float(x) if x.is_finite() && x.fract() == 0.0 && *x >= 0.0 => { + Some(pycompat::fmt::int_str(*x)) + } + Value::Str(s) if crate::digits(crate::strip(s)) => crate::int(s).map(|v| v.to_string()), + _ => None, + } +} + +/// Locityper's warnings: none for null, "", "*" or []; a string split at +/// `;` and `,`; each item of a list. +fn warnings_of(res: &Value) -> Result, String> { + match pycompat::value::get(res, "warnings").ok().flatten() { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Str(s)) if s.is_empty() || s == "*" => Ok(Vec::new()), + Some(Value::Str(s)) => Ok(s + .replace(';', ",") + .split(',') + .filter(|x| !crate::strip(x).is_empty()) + .map(str::to_owned) + .collect()), + Some(Value::Array(a)) => a + .iter() + .map(|x| pycompat::value::py_str(x).map_err(|e| e.to_string())) + .collect(), + Some(Value::Object(o)) => Ok(o.keys().cloned().collect()), + Some(_) => Err("its warnings are neither text nor a list".into()), + } +} + +/// The authors' filter: pass, or fail with the reasons. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Filter { + Pass, + Fail(Vec), +} + +fn judge(res: &Value, warns: &[String]) -> Filter { + let mut why = Vec::new(); + if !warns.is_empty() { + why.push(format!("Locityper warned: {}", warns.join(", "))); + } + let wd = as_float(pycompat::value::get(res, "weight_dist").ok().flatten()).unwrap_or(f64::NAN); + if wd.is_nan() { + why.push("no weighted distance (the database has no haplotype alignments: setup.sh runs locityper augment)".into()); + } else if wd > MAX_WEIGHT_DIST { + why.push(format!( + "weighted distance {} is above {}", + crate::fixed(wd, 2), + pycompat::fmt::general(MAX_WEIGHT_DIST, 6) + )); + } + let n = |k| count(res, k).and_then(|s| s.parse::().ok()); + match (n("total_reads"), n("unexpl_reads")) { + (Some(total), Some(unexpl)) => { + let share = MAX_UNEXPLAINED_SHARE * total as f64; + if !(unexpl < MAX_UNEXPLAINED || (total > 0 && (unexpl as f64) < share)) { + why.push(format!( + "{unexpl} of {total} reads unexplained (1000 or more, and 20% or more)" + )); + } + } + _ => why.push("Locityper gave no read counts (total_reads, unexpl_reads), so the unexplained-read rule cannot be checked".into()), + } + if why.is_empty() { + Filter::Pass + } else { + Filter::Fail(why) + } +} + +/// One row of the table, by [`COLUMNS`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LocusRow { + NotCalled { + locus: String, + region: String, + reason: String, + }, + Called { + locus: String, + region: String, + haps: [String; 2], + quality: String, + total: String, + unexplained: String, + weight_dist: String, + warnings: Vec, + filter: Filter, + }, +} + +impl LocusRow { + fn cells(&self) -> Vec { + let dot = |s: Option| s.unwrap_or_else(|| ".".into()); + match self { + LocusRow::NotCalled { + locus, + region, + reason, + } => { + let mut c = vec![locus.clone(), region.clone(), "not called".into()]; + c.extend(std::iter::repeat_n(".".to_owned(), 9)); + c.push(reason.clone()); + c + } + LocusRow::Called { + locus, + region, + haps, + quality, + total, + unexplained, + weight_dist, + warnings, + filter, + } => { + let (f, why) = match filter { + Filter::Pass => ("pass", None), + Filter::Fail(w) => ("fail", Some(w.join("; "))), + }; + vec![ + locus.clone(), + region.clone(), + "called".into(), + haps.join(","), + haps[0].clone(), + haps[1].clone(), + quality.clone(), + total.clone(), + unexplained.clone(), + weight_dist.clone(), + dot((!warnings.is_empty()).then(|| warnings.join(","))), + f.into(), + dot(why), + ] + } + } + } +} + +/// `res.json.gz` as JSON, or why it cannot be read. +fn read_res(path: &Path) -> Result { + let raw = std::fs::read(path).map_err(|e| e.to_string())?; + if raw.len() < 2 || raw[..2] != [0x1f, 0x8b] { + return Err(format!( + "Not a gzipped file ({})", + bytes_repr(&raw[..raw.len().min(2)]) + )); + } + let mut text = Vec::new(); + og_io::reader(std::io::Cursor::new(raw)) + .and_then(|mut r| r.read_to_end(&mut text)) + .map_err(|e| e.to_string())?; + let text = String::from_utf8(text).map_err(|e| e.to_string())?; + pycompat::json::loads(&text).map_err(|e| e.to_string()) +} + +/// Python's `repr()` of a short bytes value. +fn bytes_repr(b: &[u8]) -> String { + let mut s = String::from("b'"); + for &c in b { + match c { + b'\'' => s.push_str("\\'"), + b'\\' => s.push_str("\\\\"), + b'\n' => s.push_str("\\n"), + b'\r' => s.push_str("\\r"), + b'\t' => s.push_str("\\t"), + 0x20..=0x7e => s.push(c as char), + _ => s.push_str(&format!("\\x{c:02x}")), + } + } + s.push('\''); + s +} + +/// The row of one locus. +pub fn row_for(name: &str, region: &str, gt_dir: &str, exit_code: i64) -> LocusRow { + let not_called = |reason: String| LocusRow::NotCalled { + locus: name.into(), + region: region.into(), + reason, + }; + let d = Path::new(gt_dir).join("loci").join(name); + let res_path = d.join("res.json.gz"); + if !d.join("success").is_file() || !res_path.is_file() { + return not_called(if exit_code != 0 { + format!("Locityper stopped before this locus (exit code {exit_code})") + } else { + "Locityper wrote no result for this locus".into() + }); + } + let res = match read_res(&res_path) { + Ok(v @ Value::Object(_)) => v, + Ok(_) => return not_called("res.json.gz is not a JSON object".into()), + Err(e) => return not_called(format!("res.json.gz cannot be read: {e}")), + }; + let gt = match pycompat::value::get(&res, "genotype").ok().flatten() { + Some(Value::Str(s)) => crate::strip(s).to_owned(), + Some(v) if pycompat::value::truthy(v) => { + return not_called("Locityper gave a genotype that is not text".into()); + } + _ => String::new(), + }; + let haps: Vec<&str> = gt.split(',').filter(|h| !h.is_empty()).collect(); + let [h1, h2] = haps[..] else { + let shown = if gt.is_empty() { "empty" } else { > }; + return not_called(format!("Locityper gave no diploid genotype ({shown})")); + }; + let warnings = match warnings_of(&res) { + Ok(w) => w, + Err(e) => return not_called(format!("res.json.gz cannot be read: {e}")), + }; + let filter = judge(&res, &warnings); + let get = |k| pycompat::value::get(&res, k).ok().flatten(); + LocusRow::Called { + locus: name.into(), + region: region.into(), + haps: [h1.into(), h2.into()], + quality: fmt(get("quality"), 1), + total: count(&res, "total_reads").unwrap_or_else(|| ".".into()), + unexplained: count(&res, "unexpl_reads").unwrap_or_else(|| ".".into()), + weight_dist: fmt(get("weight_dist"), 3), + warnings, + filter, + } +} + +const SPEC: Spec = Spec { + prog: "og locityper report", + usage: "usage: og locityper report [-h] --gt-dir GT_DIR --loci LOCI [--exit-code EXIT_CODE] --out OUT\n", + options: &["--gt-dir", "--loci", "--exit-code", "--out"], + positionals: &[], + required: &["--gt-dir", "--loci", "--out"], +}; + +/// `og locityper VERB ARGS`: the exit code. +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if verb != Some("report") { + return crate::unknown_verb("locityper", "report", verb, err); + } + crate::run(&SPEC, rest, out, err, |a| { + let exit_code = crate::int_opt(a, "--exit-code", 0)?; + Ok(report( + crate::opt(a, "--gt-dir"), + crate::opt(a, "--loci"), + exit_code, + crate::opt(a, "--out"), + )) + }) +} + +fn report(gt_dir: &str, loci: &str, exit_code: i64, out: &str) -> Outcome { + let loci = read_loci(loci).map_err(|e| (e, 1))?; + let rows: Vec = loci + .iter() + .map(|(n, r)| row_for(n, r, gt_dir, exit_code)) + .collect(); + let mut w = pycompat::csv::Writer::new(Vec::new(), '\t'); + let io = |e: std::io::Error| (Error::io(&e, out), 1); + w.write_row(COLUMNS).map_err(io)?; + for r in &rows { + w.write_row(r.cells()).map_err(io)?; + } + crate::write_atomic(out, &w.into_inner()).map_err(|e| (e, 1))?; + let called = rows + .iter() + .filter(|r| matches!(r, LocusRow::Called { .. })) + .count(); + let pass = rows + .iter() + .filter(|r| { + matches!( + r, + LocusRow::Called { + filter: Filter::Pass, + .. + } + ) + }) + .count(); + Ok(format!( + "{called} of {} loci called, {pass} pass the authors' filter\n", + rows.len() + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use pycompat::json::loads; + + fn j(s: &str) -> Value { + loads(s).unwrap() + } + + #[test] + fn counts_as_python_reads_them() { + let r = j( + r#"{"a": 3, "b": 3.0, "c": " 12 ", "d": true, "e": -1, "f": 2.5, "g": "²", "h": 1e20}"#, + ); + assert_eq!(count(&r, "a").as_deref(), Some("3")); + assert_eq!(count(&r, "b").as_deref(), Some("3")); + assert_eq!(count(&r, "c").as_deref(), Some("12")); + assert_eq!(count(&r, "d"), None); + assert_eq!(count(&r, "e"), None); + assert_eq!(count(&r, "f"), None); + assert_eq!(count(&r, "g"), None); + assert_eq!(count(&r, "h").as_deref(), Some("100000000000000000000")); + assert_eq!(count(&r, "zz"), None); + } + + #[test] + fn filter_rules() { + let r = j(r#"{"weight_dist": 30.0, "total_reads": 4000, "unexpl_reads": 999}"#); + assert_eq!(judge(&r, &[]), Filter::Pass); + let r = j(r#"{"weight_dist": 30.5, "total_reads": 10000, "unexpl_reads": 1999}"#); + assert_eq!( + judge(&r, &["W".into()]), + Filter::Fail(vec![ + "Locityper warned: W".into(), + "weighted distance 30.50 is above 30".into() + ]) + ); + let r = j(r#"{"weight_dist": NaN, "total_reads": 4000, "unexpl_reads": 1000}"#); + let Filter::Fail(w) = judge(&r, &[]) else { + panic!() + }; + assert!(w[0].starts_with("no weighted distance")); + assert_eq!( + w[1], + "1000 of 4000 reads unexplained (1000 or more, and 20% or more)" + ); + } + + /// Red first: a weighted distance of exactly 30 passes. + #[test] + fn red_first_a_weighted_distance_of_30_passes() { + let r = j(r#"{"weight_dist": "30", "total_reads": 10, "unexpl_reads": 0}"#); + assert_eq!(judge(&r, &[]), Filter::Pass); + } + + #[test] + fn warnings() { + assert_eq!( + warnings_of(&j(r#"{"warnings": "*"}"#)).unwrap(), + Vec::::new() + ); + assert_eq!( + warnings_of(&j(r#"{"warnings": "a; b,,c"}"#)).unwrap(), + ["a", " b", "c"] + ); + assert_eq!( + warnings_of(&j(r#"{"warnings": ["x", 1, null]}"#)).unwrap(), + ["x", "1", "None"] + ); + assert_eq!(fmt(Some(&j("1e400")), 1), "inf"); + assert_eq!(fmt(Some(&j("true")), 1), "1.0"); + assert_eq!(fmt(Some(&j("\"x\"")), 1), "."); + } +} diff --git a/rust/crates/og-reports/src/mei.rs b/rust/crates/og-reports/src/mei.rs new file mode 100644 index 000000000..23ec79728 --- /dev/null +++ b/rust/crates/og-reports/src/mei.rs @@ -0,0 +1,676 @@ +//! `og mei report` and `og mei annotsv-input` (step 50): the Alu, L1 and +//! SVA insertions MEGAnE calls, and the known disease insertions. +//! +//! ```text +//! og mei report --sample S --vcf S_megane.vcf --sites assets/mei_sites.tsv +//! --depth S_mei_depth.tsv [--annotsv S_mei_annotsv.tsv] +//! --out-calls S_mei.tsv --out-sites S_mei_sites.tsv +//! og mei annotsv-input --sample S --vcf S_megane.vcf --out S_mei_annotsv_input.vcf +//! ``` +//! +//! MEGAnE (Kojima et al. 2023, Nat Genet, PMID 37169872) writes one record +//! per insertion: ALT ``, the repeat class in INFO SVTYPE, +//! MEPRED=PASS when the inserted sequence was identified, the subfamily in +//! INFO MEI (or MEI_lbp and MEI_rbp) and the copy number in FORMAT CN. +//! +//! `--out-calls`: one row per Alu, L1 or SVA insertion with MEPRED=PASS +//! and at least one copy, sorted as a karyotype is read; with `--annotsv` +//! (AnnotSV's table of the `annotsv-input` VCF) its genes, location, benign +//! allele frequency and ACMG class. `--out-sites`: every row of the catalog +//! with its status: `seen` (an insertion of the same element inside the +//! window), `not seen` (none, with a mean depth of at least 10 over the +//! window), or `not callable`. +//! +//! `annotsv-input` writes the calls the table lists as the VCF AnnotSV +//! reads (``, `SVTYPE=INS`, `END=POS`, and the ID the table is +//! joined on). +//! +//! What changes on purpose: the depth of a catalog row is the bedcov line +//! of its own window (contig, start - 1, end), not the last line of the +//! same gene and contig, so two rows of one gene on one contig no longer +//! share a depth; the window lookup is a binary search over the sorted +//! calls; a POS, a summed depth or a catalog coordinate that is not a +//! number, and a catalog or AnnotSV row that ends early, stop with an +//! `ERROR:` line (exit 1) instead of a traceback; problems start `ERROR:`, +//! not `mei_report.py:`. + +use crate::Error; +use og_filters::args::Spec; +use og_io::Utf8; +use og_tsv::{Options, Quoting}; +use std::collections::HashMap; +use std::io::Write; + +/// Mean depth over a catalog window before "not seen" is said. +const MIN_DEPTH: f64 = 10.0; +pub const CALL_HEADER: [&str; 12] = [ + "chrom", + "pos", + "end", + "element", + "subfamily", + "copies", + "filter", + "confidence", + "genes", + "location", + "benign_af", + "acmg_class", +]; +const ANNOTSV_COLUMNS: [&str; 4] = ["ID", "Annotation_mode", "Gene_name", "ACMG_class"]; +pub const SITE_COLUMNS: [&str; 10] = [ + "gene", + "chrom", + "start", + "end", + "site", + "element", + "inheritance", + "condition", + "clinvar", + "pmid", +]; + +/// The active element classes this report keeps. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Element { + Alu, + L1, + Sva, +} + +impl Element { + fn from_svtype(s: &str) -> Option { + match s { + "SINE/Alu" => Some(Element::Alu), + "LINE/L1" => Some(Element::L1), + "Retroposon/SVA" => Some(Element::Sva), + _ => None, + } + } + + fn name(self) -> &'static str { + match self { + Element::Alu => "Alu", + Element::L1 => "L1", + Element::Sva => "SVA", + } + } +} + +/// AnnotSV's annotation of one call. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Annotation { + genes: String, + location: Vec, + benign_af: String, + acmg_class: String, +} + +/// One kept insertion. +#[derive(Debug, Clone)] +pub struct Call { + chrom: String, + pos: i128, + end: String, + element: Element, + subfamily: String, + copies: String, + filter: String, + confidence: String, + id: String, + ann: Option, +} + +fn die(msg: String) -> Error { + Error(msg) +} + +/// INFO as a dict: the last of a repeated key wins. +fn info_dict(text: &str) -> HashMap<&str, &str> { + text.split(';') + .filter(|x| !x.is_empty()) + .map(|x| x.split_once('=').unwrap_or((x, ""))) + .collect() +} + +/// The first subfamily MEGAnE names: from MEI, else from either breakpoint. +fn subfamily(info: &HashMap<&str, &str>) -> String { + for key in ["MEI", "MEI_lbp", "MEI_rbp"] { + let v = info.get(key).copied().unwrap_or(""); + let name = v + .split('|') + .next() + .unwrap_or("") + .split(',') + .next() + .unwrap_or(""); + if !name.is_empty() && name != "NA" && name != "pA" { + return name.into(); + } + } + ".".into() +} + +/// The calls the table lists, and the `##contig` lines of the VCF. +pub fn read_calls(path: &str) -> Result<(Vec, Vec), Error> { + let mut calls = Vec::new(); + let mut contigs = Vec::new(); + let mut header = false; + for (i, line) in crate::lines(path, Utf8::Strict)?.enumerate() { + let line = &line?; + let n = i + 1; + if line.starts_with("##") { + if line.starts_with("##contig=") { + contigs.push(line.strip_suffix('\n').unwrap_or(line).to_owned()); + } + continue; + } + if line.starts_with("#CHROM") { + header = true; + continue; + } + if crate::strip(line).is_empty() { + continue; + } + if !header { + return Err(die(format!( + "{path}: a record before the #CHROM line (line {n}): not a VCF" + ))); + } + let f: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(line) + .split('\t') + .collect(); + if f.len() < 10 { + return Err(die(format!( + "{path} line {n}: {} columns, a VCF with one sample has 10", + f.len() + ))); + } + let info = info_dict(f[7]); + let (Some(svtype), Some(mepred)) = (info.get("SVTYPE"), info.get("MEPRED")) else { + return Err(die(format!( + "{path} line {n}: no SVTYPE or MEPRED in INFO: not MEGAnE's VCF" + ))); + }; + let fmt: HashMap<&str, &str> = f[8].split(':').zip(f[9].split(':')).collect(); + let cn = fmt.get("CN").copied().unwrap_or("."); + let Some(element) = Element::from_svtype(svtype) else { + continue; + }; + if *mepred != "PASS" || !crate::digits(cn) || crate::int(cn).is_none_or(|c| c < 1) { + continue; + } + let Some(pos) = crate::int(f[1]) else { + return Err(die(format!( + "{path} line {n}: POS {} is not a whole number", + og_filters::py_repr(f[1]) + ))); + }; + calls.push(Call { + chrom: f[0].into(), + pos, + end: info.get("MEI_rpos").copied().unwrap_or(f[1]).into(), + element, + subfamily: subfamily(&info), + copies: cn.into(), + filter: f[6].into(), + confidence: info.get("CN_conf").copied().unwrap_or(".").into(), + id: String::new(), + ann: None, + }); + } + if !header { + return Err(die(format!("{path}: no #CHROM line: not a VCF"))); + } + let mut seen: HashMap = HashMap::new(); + for c in &mut calls { + let key = format!("{}_{}_{}", c.chrom, c.pos, c.element.name()); + let k = seen.entry(key.clone()).or_default(); + *k += 1; + c.id = if *k == 1 { key } else { format!("{key}_{k}") }; + } + Ok((calls, contigs)) +} + +/// The calls as the VCF AnnotSV reads: one `` record each, END=POS. +fn annotsv_input(sample: &str, calls: &[Call], contigs: &[String]) -> String { + let mut out = String::from("##fileformat=VCFv4.2\n"); + for c in contigs { + out.push_str(c); + out.push('\n'); + } + out.push_str("##ALT=\n##INFO=\n##INFO=\n##FORMAT=\n"); + out.push_str(&format!( + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}\n" + )); + for c in calls { + let gt = if c.copies == "1" { "0/1" } else { "1/1" }; + out.push_str(&format!( + "{}\t{}\t{}\tN\t\t.\t{}\tSVTYPE=INS;END={}\tGT\t{gt}\n", + c.chrom, c.pos, c.id, c.filter, c.pos + )); + } + out +} + +/// AnnotSV's table of the calls, by ID. An empty file (AnnotSV had nothing +/// to annotate) is an empty map. +fn read_annotsv(path: &str) -> Result, Error> { + let (header, rows) = crate::read_table(path, Options::default().quoting(Quoting::None))?; + if rows.is_empty() { + return Ok(HashMap::new()); + } + if ANNOTSV_COLUMNS + .iter() + .any(|c| !header.iter().any(|h| h == c)) + { + return Err(die(format!( + "{path}: not AnnotSV's table (a column of {} is missing)", + ANNOTSV_COLUMNS.join(", ") + ))); + } + let mut out: HashMap = HashMap::new(); + for r in &rows { + let id = r.text("ID")?; + let a = out.entry(id.to_owned()).or_insert_with(|| Annotation { + genes: ".".into(), + location: Vec::new(), + benign_af: ".".into(), + acmg_class: ".".into(), + }); + let mode = r.text("Annotation_mode")?; + if mode == "full" { + let genes: Vec<&str> = r + .text("Gene_name")? + .split(';') + .filter(|g| !g.is_empty()) + .collect(); + a.genes = if genes.is_empty() { + ".".into() + } else { + genes.join(",") + }; + let b = crate::strip(r.get("B_ins_AFmax").unwrap_or("")); + a.benign_af = if b.is_empty() { ".".into() } else { b.into() }; + let c = crate::strip(r.text("ACMG_class")?); + a.acmg_class = if c.is_empty() { ".".into() } else { c.into() }; + } else if mode == "split" { + let gene = r.text("Gene_name")?; + let loc = crate::strip(r.get("Location").unwrap_or("")); + if !gene.is_empty() && !loc.is_empty() { + a.location.push(format!("{gene}:{loc}")); + } + } + } + Ok(out) +} + +/// The catalog rows: each one's cells by [`SITE_COLUMNS`]. +fn read_sites(path: &str) -> Result>, Error> { + let opts = Options::default().skip(|l| crate::strip(l).is_empty() || l.starts_with('#')); + let (header, rows) = crate::read_table(path, opts)?; + if rows.is_empty() || SITE_COLUMNS.iter().any(|c| !header.iter().any(|h| h == c)) { + return Err(die(format!( + "{path}: no rows, or a column of {} is missing", + SITE_COLUMNS.join(", ") + ))); + } + rows.iter() + .map(|r| { + SITE_COLUMNS + .iter() + .map(|k| r.text(k).map(str::to_owned).map_err(Error::from)) + .collect() + }) + .collect() +} + +/// `samtools bedcov -c` over the catalog windows: the mean depth of each +/// window, keyed by (contig, 0-based start, end). +fn read_depth(path: &str) -> Result, Error> { + let mut depth = HashMap::new(); + for (i, line) in crate::lines(path, Utf8::Strict)?.enumerate() { + let line = &line?; + let n = i + 1; + if crate::strip(line).is_empty() { + continue; + } + let f: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(line) + .split('\t') + .collect(); + let bad = || { + die(format!( + "{path} line {n}: not samtools bedcov -c output of a 4-column BED" + )) + }; + if f.len() < 6 || !crate::digits(f[1]) || !crate::digits(f[2]) { + return Err(bad()); + } + let (start, end) = ( + crate::int(f[1]).ok_or_else(bad)?, + crate::int(f[2]).ok_or_else(bad)?, + ); + let sum = crate::float(f[4]).ok_or_else(bad)?; + let length = end - start; + let mean = if length > 0 { sum / length as f64 } else { 0.0 }; + depth.insert((f[0].to_owned(), start, end), mean); + } + Ok(depth) +} + +/// chr1 to chr22, then chrX, chrY and the rest. +fn chrom_rank(chrom: &str) -> i128 { + let c = chrom.strip_prefix("chr").unwrap_or(chrom); + if crate::digits(c) { + return crate::int(c).unwrap_or(i128::MAX); + } + match c { + "X" => 23, + "Y" => 24, + _ => 25, + } +} + +/// The status of a catalog row: (status, mean depth, call). +fn site_status( + site: &[String], + by_chrom: &HashMap<&str, Vec<&Call>>, + depth: &HashMap<(String, i128, i128), f64>, + sites_path: &str, +) -> Result<[String; 3], Error> { + let coord = |k: usize| { + crate::int(&site[k]).ok_or_else(|| { + die(format!( + "{sites_path}: {} {} is not a whole number ({})", + SITE_COLUMNS[k], + og_filters::py_repr(&site[k]), + site[0] + )) + }) + }; + let (chrom, lo, hi) = (&site[1], coord(2)?, coord(3)?); + let Some(&mean) = depth.get(&(chrom.clone(), lo - 1, hi)) else { + return Ok(["not callable".into(), ".".into(), ".".into()]); + }; + let shown = crate::fixed(mean, 1); + let calls = by_chrom.get(chrom.as_str()).map_or(&[][..], Vec::as_slice); + let first = calls.partition_point(|c| c.pos < lo); + let hit = calls[first..] + .iter() + .take_while(|c| c.pos <= hi) + .find(|c| c.element.name() == site[5]); + if let Some(c) = hit { + let cop = if c.copies == "1" { "y" } else { "ies" }; + return Ok([ + "seen".into(), + shown, + format!( + "{}:{} {} {} cop{cop} {}", + c.chrom, c.pos, c.subfamily, c.copies, c.filter + ), + ]); + } + if mean < MIN_DEPTH { + return Ok(["not callable".into(), shown, ".".into()]); + } + Ok(["not seen".into(), shown, ".".into()]) +} + +/// The files of `og mei report`. +pub struct Files<'a> { + pub vcf: &'a str, + pub sites: &'a str, + pub depth: &'a str, + pub annotsv: Option<&'a str>, + pub out_calls: &'a str, + pub out_sites: &'a str, +} + +/// `og mei report`: writes both tables; the stdout lines. +pub fn report(sample: &str, f: &Files<'_>) -> Result { + let (mut calls, _) = read_calls(f.vcf)?; + let ann = match f.annotsv { + Some(p) => Some(read_annotsv(p)?), + None => None, + }; + for c in &mut calls { + c.ann = ann.as_ref().and_then(|a| a.get(&c.id).cloned()); + } + let sites = read_sites(f.sites)?; + let depth = read_depth(f.depth)?; + calls.sort_by(|a, b| { + (chrom_rank(&a.chrom), &a.chrom, a.pos).cmp(&(chrom_rank(&b.chrom), &b.chrom, b.pos)) + }); + let mut body = CALL_HEADER.join("\t") + "\n"; + for c in &calls { + let (genes, location, benign, acmg) = match &c.ann { + Some(a) => ( + a.genes.clone(), + if a.location.is_empty() { + ".".into() + } else { + a.location.join(",") + }, + a.benign_af.clone(), + a.acmg_class.clone(), + ), + None => (".".into(), ".".into(), ".".into(), ".".into()), + }; + let cells = [ + c.chrom.clone(), + c.pos.to_string(), + c.end.clone(), + c.element.name().into(), + c.subfamily.clone(), + c.copies.clone(), + c.filter.clone(), + c.confidence.clone(), + genes, + location, + benign, + acmg, + ]; + body.push_str(&cells.join("\t")); + body.push('\n'); + } + let mut by_chrom: HashMap<&str, Vec<&Call>> = HashMap::new(); + for c in &calls { + by_chrom.entry(c.chrom.as_str()).or_default().push(c); + } + let mut sbody = SITE_COLUMNS.join("\t") + "\tstatus\tmean_depth\tcall\n"; + for s in &sites { + let st = site_status(s, &by_chrom, &depth, f.sites)?; + sbody.push_str(&s.join("\t")); + for x in &st { + sbody.push('\t'); + sbody.push_str(x); + } + sbody.push('\n'); + } + crate::write_atomic(f.out_calls, body.as_bytes())?; + crate::write_atomic(f.out_sites, sbody.as_bytes())?; + let mut by: Vec<(Element, usize)> = Vec::new(); + for c in &calls { + match by.iter_mut().find(|(e, _)| *e == c.element) { + Some((_, n)) => *n += 1, + None => by.push((c.element, 1)), + } + } + by.sort_by_key(|(e, _)| e.name()); + let tally = if by.is_empty() { + "none".to_owned() + } else { + by.iter() + .map(|(e, n)| format!("{} {n}", e.name())) + .collect::>() + .join(", ") + }; + let mut text = format!("{sample}: {} insertions ({tally})\n", calls.len()); + if let Some(a) = &ann { + let annotated = calls.iter().filter(|c| a.contains_key(&c.id)).count(); + let in_gene = calls + .iter() + .filter(|c| c.ann.as_ref().is_some_and(|x| x.genes != ".")) + .count(); + text.push_str(&format!( + "{sample}: AnnotSV annotated {annotated} of them, {in_gene} inside a gene\n" + )); + } + Ok(text) +} + +const REPORT: Spec = Spec { + prog: "og mei report", + usage: "usage: og mei report [-h] --sample SAMPLE --vcf VCF --sites SITES --depth DEPTH [--annotsv ANNOTSV] --out-calls OUT_CALLS --out-sites OUT_SITES\n", + options: &[ + "--sample", + "--vcf", + "--sites", + "--depth", + "--annotsv", + "--out-calls", + "--out-sites", + ], + positionals: &[], + required: &[ + "--sample", + "--vcf", + "--sites", + "--depth", + "--out-calls", + "--out-sites", + ], +}; + +const INPUT: Spec = Spec { + prog: "og mei annotsv-input", + usage: "usage: og mei annotsv-input [-h] --sample SAMPLE --vcf VCF --out OUT\n", + options: &["--sample", "--vcf", "--out"], + positionals: &[], + required: &["--sample", "--vcf", "--out"], +}; + +/// `og mei VERB ARGS`: the exit code (1 for a problem, 2 for a refused +/// command line). +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + match verb { + Some("report") => crate::run(&REPORT, rest, out, err, |a| { + let files = Files { + vcf: crate::opt(a, "--vcf"), + sites: crate::opt(a, "--sites"), + depth: crate::opt(a, "--depth"), + annotsv: a.get("--annotsv"), + out_calls: crate::opt(a, "--out-calls"), + out_sites: crate::opt(a, "--out-sites"), + }; + Ok(report(crate::opt(a, "--sample"), &files).map_err(|e| (e, 1))) + }), + Some("annotsv-input") => crate::run(&INPUT, rest, out, err, |a| { + let sample = crate::opt(a, "--sample"); + Ok((|| { + let (calls, contigs) = read_calls(crate::opt(a, "--vcf"))?; + let body = annotsv_input(sample, &calls, &contigs); + crate::write_atomic(crate::opt(a, "--out"), body.as_bytes())?; + Ok(format!( + "{sample}: {} insertions written for AnnotSV\n", + calls.len() + )) + })() + .map_err(|e| (e, 1))) + }), + other => crate::unknown_verb("mei", "report|annotsv-input", other, err), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + const HEAD: &str = "##fileformat=VCFv4.1\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; + + fn rec(chrom: &str, pos: u64, svtype: &str, cn: &str) -> String { + format!( + "{chrom}\t{pos}\t.\tN\t\t.\tPASS\tSVTYPE={svtype};MEPRED=PASS;MEI=SVA_E,x|y;CN_conf=H\tGT:CN\t0/1:{cn}\n" + ) + } + + const SITES: &str = "# catalog\ngene\tchrom\tstart\tend\tsite\telement\tinheritance\tcondition\tclinvar\tpmid\nFKTN\tchr4\t105620000\t105620600\t3' UTR\tSVA\tAR\tFukuyama\t.\t1\nFKTN\tchr4\t105630000\t105630600\tother\tSVA\tAR\tmade up\t.\t1\n"; + const DEPTH: &str = "chr4\t105619999\t105620600\tFKTN\t18030\t500\nchr4\t105629999\t105630600\tFKTN\t3005\t20\n"; + + fn run(vcf: &str) -> (String, String) { + let d = Dir::new("mei"); + let f = Files { + vcf: &d.put("m.vcf", vcf), + sites: &d.put("sites.tsv", SITES), + depth: &d.put("depth.tsv", DEPTH), + annotsv: None, + out_calls: &d.path("calls.tsv"), + out_sites: &d.path("sites_out.tsv"), + }; + report("S", &f).unwrap(); + (d.read("calls.tsv"), d.read("sites_out.tsv")) + } + + #[test] + fn a_planted_sva_is_seen_and_each_window_has_its_own_depth() { + let (calls, sites) = run(&format!( + "{HEAD}{}{}", + rec("chr4", 105620300, "Retroposon/SVA", "1"), + rec("chr4", 105620300, "Retroposon/SVA", "2") + )); + assert_eq!(calls.lines().count(), 3); + let rows: Vec<&str> = sites.lines().collect(); + assert!( + rows[1].ends_with("\tseen\t30.0\tchr4:105620300 SVA_E 1 copy PASS"), + "{}", + rows[1] + ); + // The second FKTN window has its own bedcov line: 5x, too few reads. + assert!(rows[2].ends_with("\tnot callable\t5.0\t."), "{}", rows[2]); + } + + /// Red first: a mean depth of exactly 10 is enough to say "not seen". + #[test] + fn red_first_a_depth_of_ten_is_callable() { + let d = Dir::new("mei-ten"); + let depth: HashMap<_, _> = [(("chr4".to_owned(), 99, 200), 10.0)].into(); + let site: Vec = ["G", "chr4", "100", "200", "s", "Alu", "AD", "c", ".", "1"] + .map(String::from) + .to_vec(); + let st = site_status(&site, &HashMap::new(), &depth, &d.path("x")).unwrap(); + assert_eq!(st[0], "not seen"); + } + + #[test] + fn ids_and_annotsv_input() { + let d = Dir::new("mei-ids"); + let p = d.put( + "m.vcf", + &format!( + "{HEAD}{}{}{}", + rec("chr4", 5, "SINE/Alu", "1"), + rec("chr4", 5, "SINE/Alu", "2"), + rec("chr4", 6, "DNA/hAT", "1") + ), + ); + let (calls, contigs) = read_calls(&p).unwrap(); + assert_eq!( + calls.iter().map(|c| c.id.as_str()).collect::>(), + ["chr4_5_Alu", "chr4_5_Alu_2"] + ); + let v = annotsv_input("S", &calls, &contigs); + assert!(v.contains("##contig=\n")); + assert!( + v.ends_with("chr4\t5\tchr4_5_Alu_2\tN\t\t.\tPASS\tSVTYPE=INS;END=5\tGT\t1/1\n") + ); + assert_eq!(chrom_rank("chrX"), 23); + assert_eq!(chrom_rank("chr10"), 10); + assert_eq!(chrom_rank("chrUn_x"), 25); + } +} diff --git a/rust/crates/og-reports/src/mocha.rs b/rust/crates/og-reports/src/mocha.rs new file mode 100644 index 000000000..53212a023 --- /dev/null +++ b/rust/crates/og-reports/src/mocha.rs @@ -0,0 +1,618 @@ +//! `og mocha report` (step 51): the mosaic chromosomal alterations MoChA +//! calls, with MoChA's own UK Biobank filters and this pipeline's floor. +//! +//! A mosaic chromosomal alteration (mCA) is a large gain, loss or +//! copy-neutral loss of heterozygosity carried by a share of the blood +//! cells. They rise with age and go with a higher risk of a later blood +//! cancer (Loh et al. 2018, PMID 29995854). MoChA calls them from a phased +//! VCF: in the cells that carry the event one parental haplotype is lost or +//! gained, so the allele balance leans the same way along the haplotype. +//! +//! ```text +//! og mocha report --sample S --calls CALLS.tsv --stats STATS.tsv --sites SITES.tsv \ +//! --phasing beagle|input --out-calls S_mocha.tsv --out-summary S_mocha_summary.tsv +//! ``` +//! +//! A call is reported when it is on chr1 to chr22, MoChA's filter of the +//! UK Biobank callset keeps it (not a germline CNP; not likely +//! constitutional; not weak; not a likely germline duplication), and its +//! cell fraction is at or above the floor: the larger of 0.05 and +//! 6 / sqrt(D x H), D MoChA's median depth and H the heterozygous sites per +//! 10 Mb. The sample is not assessed when MoChA's sample filter fails +//! (call rate under 0.97, phased BAF autocorrelation over 0.03) or cannot +//! be applied. Exit 0 whatever the result; 2 on unreadable input. +//! +//! NaN is kept as Python has it: every comparison with NaN is false, so a +//! call with no BAF deviation, no N50 and no concordance LOD is dropped as +//! likely constitutional, as MoChA's own filter expression does. +//! +//! What changes on purpose: `calls_mocha` counts the calls of this sample +//! (the Python counted every row of a multi-sample table, other samples' +//! included; a pipeline run gives MoChA one sample, so its value does not +//! change), and a calls row that ends before a needed column, or a sites +//! value that is not ASCII digits, is an input error (exit 2), not a +//! traceback. + +use crate::{Error, Outcome}; +use og_filters::args::Spec; +use og_io::Utf8; +use og_tsv::{Options, Quoting}; +use std::io::Write; + +const FLOOR_MIN: f64 = 0.05; +const FLOOR_Z: f64 = 3.0; +const FLOOR_MB: f64 = 10.0; +const CALL_RATE_MIN: f64 = 0.97; +const BAF_AUTO_MAX: f64 = 0.03; + +pub const CALL_COLUMNS: [&str; 14] = [ + "chrom", + "start", + "end", + "length_mb", + "arms", + "type", + "cell_fraction", + "bdev", + "rel_cov", + "n_hets", + "lod_baf_phase", + "lod_lrr_baf", + "reported", + "reason", +]; +const NEED_CALLS: [&str; 14] = [ + "chrom", + "length", + "p_arm", + "q_arm", + "n50_hets", + "bdev", + "bdev_se", + "rel_cov", + "lod_baf_phase", + "lod_lrr_baf", + "lod_baf_conc", + "n_hets", + "type", + "cf", +]; +const NEED_STATS: [&str; 5] = ["sample_id", "call_rate", "cov_median", "baf_auto", "n_hets"]; + +/// chr1 to chr22, with or without the prefix. +fn autosome(c: &str) -> bool { + let n = c.strip_prefix("chr").unwrap_or(c); + crate::digits(n) && !n.starts_with('0') && n.parse::().is_ok_and(|v| (1..=22).contains(&v)) +} + +/// MoChA's number: NaN for `nan`, `.`, an empty or a missing cell. +fn num(x: Option<&str>) -> f64 { + x.and_then(crate::float).unwrap_or(f64::NAN) +} + +/// `f"{x:.{d}f}"`, "." for NaN. +fn fmt(x: f64, d: usize) -> String { + if x.is_nan() { + ".".into() + } else { + crate::fixed(x, d) + } +} + +/// MoChA's call type, as the table writes it. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Kind { + Loss, + Gain, + CnLoh, + Undetermined, + Other(String), +} + +impl Kind { + fn read(s: &str) -> Kind { + match s { + "Loss" => Kind::Loss, + "Gain" => Kind::Gain, + "CN-LOH" => Kind::CnLoh, + "Undetermined" => Kind::Undetermined, + o => Kind::Other(o.into()), + } + } + + fn word(&self) -> &str { + match self { + Kind::Loss => "loss", + Kind::Gain => "gain", + Kind::CnLoh => "CN-LOH", + Kind::Undetermined => "undetermined", + Kind::Other(s) => s, + } + } +} + +/// One chromosome of the sites table. +#[derive(Debug, Clone)] +pub struct Sites { + chrom: String, + first: i128, + last: i128, + hets: i128, + phased: i128, +} + +/// A table of MoChA's, checked for the columns it needs. +fn read_table(path: &str, need: &[&str]) -> Result, Error> { + let opts = Options::default() + .quoting(Quoting::None) + .utf8(Utf8::Replace); + let (header, rows) = crate::read_table(path, opts)?; + let missing: Vec<&str> = need + .iter() + .copied() + .filter(|k| !header.iter().any(|h| h == k)) + .collect(); + if !missing.is_empty() { + return Err(Error(format!( + "{path} lacks the column{} {}", + if missing.len() > 1 { "s" } else { "" }, + missing.join(", ") + ))); + } + Ok(rows) +} + +fn read_sites(path: &str) -> Result, Error> { + let mut out = Vec::new(); + for (n, line) in crate::read_lines(path, Utf8::Replace)?.iter().enumerate() { + if crate::strip(line).is_empty() { + continue; + } + let c: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(line) + .split('\t') + .collect(); + let v: Option> = (c.len() == 6) + .then(|| { + c[1..] + .iter() + .map(|x| crate::digits(x).then(|| crate::int(x)).flatten()) + .collect() + }) + .flatten(); + let Some(v) = v else { + return Err(Error(format!( + "{path} line {}: want chrom, first, last, sites, hets, phased hets", + n + 1 + ))); + }; + out.push(Sites { + chrom: c[0].into(), + first: v[0], + last: v[1], + hets: v[3], + phased: v[4], + }); + } + Ok(out) +} + +/// The read-sampling floor: 6 / sqrt(D x H); NaN without data. +fn floor_read(depth: f64, het_per_10mb: f64) -> f64 { + if !(depth > 0.0 && het_per_10mb > 0.0) { + return f64::NAN; + } + 4.0 * FLOOR_Z * 0.5 / (depth * het_per_10mb).sqrt() +} + +/// `20p`, `20q` or `20p and 20q`, with the telomere it reaches. +fn arms(chrom: &str, p_arm: Option<&str>, q_arm: Option<&str>) -> String { + let c = chrom.strip_prefix("chr").unwrap_or(chrom); + let both = [("p", p_arm), ("q", q_arm)]; + let on: Vec<&str> = both + .iter() + .filter(|(_, v)| matches!(v, Some("Y" | "C" | "T"))) + .map(|(a, _)| *a) + .collect(); + let tel: Vec<&str> = both + .iter() + .filter(|(_, v)| *v == Some("T")) + .map(|(a, _)| *a) + .collect(); + if on.is_empty() { + return format!("{c} (interstitial)"); + } + let mut text = on + .iter() + .map(|a| format!("{c}{a}")) + .collect::>() + .join(" and "); + if !tel.is_empty() { + text.push_str(", to the "); + text.push_str( + &tel.iter() + .map(|a| format!("{a} telomere")) + .collect::>() + .join(" and "), + ); + } + text +} + +/// The reason MoChA's UK Biobank filter drops a call, or `None`. +// The negated comparisons are on purpose: a NaN fails every comparison, as +// in the Python, so `!(x > 10.0)` holds for a NaN and `x <= 10.0` would not. +#[allow(clippy::neg_cmp_op_on_partial_ord)] +fn mocha_filter(r: &og_tsv::Row) -> Option { + let t = r.get("type").unwrap_or(""); + if t.starts_with("CNP") { + return Some(format!( + "germline copy-number polymorphism (MoChA type {t})" + )); + } + let n = |k| num(r.get(k)); + let (bdev, n50, conc) = (n("bdev"), n("n50_hets"), n("lod_baf_conc")); + if !(bdev < 0.1 || n50 < 2e5 || (!conc.is_nan() && conc > 10.0)) { + return Some( + "likely constitutional (BAF deviation 0.1 or more with sparse heterozygous sites)" + .into(), + ); + } + if n("bdev_se").is_nan() && !(n("lod_baf_phase") > 10.0) { + return Some( + "weak: no BAF deviation estimate and a BAF-and-phase LOD of 10 or less".into(), + ); + } + let (rel, length) = (n("rel_cov"), n("length")); + if !(rel < 2.1 + || bdev < 0.05 + || (length > 5e5 && bdev < 0.1 && rel < 2.5) + || (length > 5e6 && bdev < 0.15)) + { + return Some("likely germline duplication (relative coverage 2.1 or more)".into()); + } + None +} + +/// The report: call rows (by [`CALL_COLUMNS`]) and the summary rows. +pub type Report = (Vec>, Vec<(&'static str, String)>); + +/// The report of `sample` from MoChA's rows, the sites and the phasing. +pub fn report( + sample: &str, + calls: &[og_tsv::Row], + stats: &[og_tsv::Row], + sites: &[Sites], + phasing: &str, +) -> Result { + let st = stats + .iter() + .find(|r| r.get("sample_id") == Some(sample)) + .or(if stats.len() == 1 { + stats.first() + } else { + None + }) + .ok_or_else(|| Error(format!("MoChA's stats table has no row for {sample}")))?; + let auto: Vec<&Sites> = sites.iter().filter(|s| autosome(&s.chrom)).collect(); + let span: i128 = auto.iter().map(|s| (s.last - s.first + 1).max(0)).sum(); + let hets: i128 = auto.iter().map(|s| s.hets).sum(); + let phased: i128 = auto.iter().map(|s| s.phased).sum(); + let depth = num(st.get("cov_median")); + let h10 = if span != 0 { + hets as f64 * FLOOR_MB * 1e6 / span as f64 + } else { + f64::NAN + }; + let fr = floor_read(depth, h10); + let floor = if fr.is_nan() { + f64::NAN + } else if fr > FLOOR_MIN { + fr + } else { + FLOOR_MIN + }; + let (call_rate, baf_auto) = (num(st.get("call_rate")), num(st.get("baf_auto"))); + let why = if auto.is_empty() || hets == 0 { + Some("no heterozygous site on chr1 to chr22 reached MoChA".to_owned()) + } else if phased == 0 { + Some("no heterozygous site is phased, so MoChA cannot follow a haplotype".into()) + } else if floor.is_nan() { + Some("MoChA gave no median depth, so no floor can be set".into()) + } else if call_rate.is_nan() || baf_auto.is_nan() { + Some("MoChA gave no call rate or phased BAF autocorrelation, so its sample filter cannot be applied".into()) + } else if call_rate < CALL_RATE_MIN { + Some(format!( + "call rate {} is under {} (MoChA's sample filter)", + crate::fixed(call_rate, 4), + pycompat::fmt::repr(CALL_RATE_MIN) + )) + } else if baf_auto > BAF_AUTO_MAX { + Some(format!( + "phased BAF autocorrelation {} is over {} (MoChA's sample filter): noisy data", + crate::fixed(baf_auto, 4), + pycompat::fmt::repr(BAF_AUTO_MAX) + )) + } else { + None + }; + let mut rows = Vec::new(); + for r in calls { + if stats.len() > 1 && r.get("sample_id").is_some_and(|s| s != sample) { + continue; + } + for k in NEED_CALLS { + r.text(k)?; + } + let chrom = r.get("chrom").unwrap_or(""); + let cf = num(r.get("cf")); + let reason = if !autosome(chrom) { + Some("not an autosome (mosaic loss of Y and X: step 33)".to_owned()) + } else if let Some(f) = mocha_filter(r) { + Some(f) + } else if cf.is_nan() { + Some("cell fraction not estimated".into()) + } else if !floor.is_nan() && cf < floor { + Some(format!( + "cell fraction {} under the floor {}", + crate::fixed(cf, 3), + crate::fixed(floor, 3) + )) + } else if why.is_some() { + Some("sample not assessed".into()) + } else { + None + }; + let prefixed = |p: &str| { + r.dict() + .iter() + .find(|(k, _)| k.starts_with(p)) + .map_or(".".to_owned(), |(_, v)| v.unwrap_or("None").to_owned()) + }; + let raw = |k: &str| r.get(k).unwrap_or("None").to_owned(); + rows.push(vec![ + chrom.to_owned(), + prefixed("beg_"), + prefixed("end_"), + fmt(num(r.get("length")) / 1e6, 2), + arms(chrom, r.get("p_arm"), r.get("q_arm")), + Kind::read(r.get("type").unwrap_or("")).word().to_owned(), + fmt(cf, 3), + fmt(num(r.get("bdev")), 4), + fmt(num(r.get("rel_cov")), 4), + raw("n_hets"), + raw("lod_baf_phase"), + raw("lod_lrr_baf"), + if reason.is_some() { "no" } else { "yes" }.into(), + reason.unwrap_or_else(|| ".".into()), + ]); + } + let reported = rows.iter().filter(|r| r[12] == "yes").count(); + let summary = vec![ + ( + "status", + if why.is_some() { + "not assessed" + } else { + "assessed" + } + .to_owned(), + ), + ("message", why.unwrap_or_else(|| ".".into())), + ("depth", fmt(depth, 1)), + ("het_sites", hets.to_string()), + ("phased_het_sites", phased.to_string()), + ("het_per_10mb", fmt(h10, 0)), + ("analysed_mb", fmt(span as f64 / 1e6, 1)), + ( + "chromosomes", + if auto.is_empty() { + ".".into() + } else { + auto.iter() + .map(|s| s.chrom.as_str()) + .collect::>() + .join(",") + }, + ), + ("call_rate", fmt(call_rate, 4)), + ("baf_auto", fmt(baf_auto, 4)), + ("floor_read", fmt(fr, 3)), + ("floor_min", fmt(FLOOR_MIN, 2)), + ("floor", fmt(floor, 3)), + ("calls_mocha", rows.len().to_string()), + ("calls_reported", reported.to_string()), + ("phasing", phasing.to_owned()), + ]; + Ok((rows, summary)) +} + +const SPEC: Spec = Spec { + prog: "og mocha report", + usage: "usage: og mocha report [-h] --sample SAMPLE --calls CALLS --stats STATS --sites SITES --phasing {beagle,input} --out-calls OUT_CALLS --out-summary OUT_SUMMARY\n", + options: &[ + "--sample", + "--calls", + "--stats", + "--sites", + "--phasing", + "--out-calls", + "--out-summary", + ], + positionals: &[], + required: &[ + "--sample", + "--calls", + "--stats", + "--sites", + "--phasing", + "--out-calls", + "--out-summary", + ], +}; + +/// `og mocha VERB ARGS`: the exit code. +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if verb != Some("report") { + return crate::unknown_verb("mocha", "report", verb, err); + } + crate::run(&SPEC, rest, out, err, |a| { + let phasing = crate::opt(a, "--phasing"); + if phasing != "beagle" && phasing != "input" { + return Err(og_filters::args::ArgError::Usage(format!( + "argument --phasing: invalid choice: {} (choose from 'beagle', 'input')", + og_filters::py_repr(phasing) + ))); + } + Ok(run( + crate::opt(a, "--sample"), + crate::opt(a, "--calls"), + crate::opt(a, "--stats"), + crate::opt(a, "--sites"), + phasing, + crate::opt(a, "--out-calls"), + crate::opt(a, "--out-summary"), + )) + }) +} + +fn run( + sample: &str, + calls: &str, + stats: &str, + sites: &str, + phasing: &str, + out_calls: &str, + out_summary: &str, +) -> Outcome { + let bad = |e: Error| (e, 2); + let c = read_table(calls, &NEED_CALLS).map_err(bad)?; + let s = read_table(stats, &NEED_STATS).map_err(bad)?; + let si = read_sites(sites).map_err(bad)?; + let (rows, summary) = report(sample, &c, &s, &si, phasing).map_err(bad)?; + let mut body = CALL_COLUMNS.join("\t") + "\n"; + for r in &rows { + body.push_str(&r.join("\t")); + body.push('\n'); + } + crate::write_atomic(out_calls, body.as_bytes()).map_err(|e| (e, 1))?; + let mut sum = String::from("metric\tvalue\n"); + for (k, v) in &summary { + sum.push_str(&format!("{k}\t{v}\n")); + } + crate::write_atomic(out_summary, sum.as_bytes()).map_err(|e| (e, 1))?; + let get = |k: &str| { + summary + .iter() + .find(|(n, _)| *n == k) + .map_or("", |(_, v)| v.as_str()) + }; + let msg = get("message"); + Ok(format!( + "{sample}: {}; {} of {} MoChA calls reported (floor {}, depth {}x){}\n", + get("status"), + get("calls_reported"), + get("calls_mocha"), + get("floor"), + get("depth"), + if msg == "." { + String::new() + } else { + format!(": {msg}") + } + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arms_and_autosomes() { + assert_eq!( + arms("chr20", Some("N"), Some("T")), + "20q, to the q telomere" + ); + assert_eq!(arms("7", Some("C"), Some("Y")), "7p and 7q"); + assert_eq!(arms("chr1", Some("N"), Some("N")), "1 (interstitial)"); + assert!(autosome("chr22") && autosome("1") && !autosome("chr23") && !autosome("chrX")); + assert!(!autosome("chr01") && !autosome("chr0")); + } + + #[test] + fn floor() { + // 30x, 120k hets per 100 Mb: 6 / sqrt(30 * 12000) = 0.01, under 0.05. + assert!((floor_read(30.0, 12_000.0) - 0.01).abs() < 1e-12); + assert!(floor_read(f64::NAN, 1.0).is_nan()); + assert!(floor_read(0.0, 1.0).is_nan()); + } + + fn table(d: &crate::testutil::Dir, name: &str, text: &str, need: &[&str]) -> Vec { + read_table(&d.put(name, text), need).unwrap() + } + + const CALL_HEAD: &str = "sample_id\tchrom\tbeg_GRCh38\tend_GRCh38\tlength\tp_arm\tq_arm\tn_hets\tn50_hets\tbdev\tbdev_se\trel_cov\tlod_baf_phase\tlod_lrr_baf\tlod_baf_conc\ttype\tcf\n"; + const LOSS: &str = "S\tchr20\t30100000\t42300000\t12200000\tN\tY\t4210\t350000\t0.031\t0.004\t1.86\t48.2\t35.1\tnan\tLoss\t0.120\n"; + + fn run_with(call_rate: &str, calls: &str) -> Report { + let d = crate::testutil::Dir::new("mocha"); + let stats = format!( + "sample_id\tcall_rate\tcov_median\tbaf_auto\tn_hets\nS\t{call_rate}\t31\t0.0124\t2100000\n" + ); + let c = table(&d, "c.tsv", &format!("{CALL_HEAD}{calls}"), &NEED_CALLS); + let s = table(&d, "s.tsv", &stats, &NEED_STATS); + let sites: Vec = (1..=22) + .map(|i| Sites { + chrom: format!("chr{i}"), + first: 10001, + last: 100_010_001, + hets: 120_000, + phased: 94_000, + }) + .collect(); + report("S", &c, &s, &sites, "beagle").unwrap() + } + + #[test] + fn a_loss_is_reported() { + let (rows, summary) = run_with("0.9981", LOSS); + assert_eq!(rows[0][4], "20q"); + assert_eq!(rows[0][12], "yes", "{rows:?}"); + assert_eq!(summary[0].1, "assessed"); + assert_eq!(summary[12], ("floor", "0.050".to_owned())); + } + + /// Red first: a call rate of exactly 0.97 meets MoChA's sample filter. + #[test] + fn red_first_a_call_rate_of_0_97_is_assessed() { + let (rows, summary) = run_with("0.97", LOSS); + assert_eq!(summary[0].1, "assessed"); + assert_eq!(rows[0][12], "yes"); + let (rows, summary) = run_with("0.9699", LOSS); + assert_eq!(summary[0].1, "not assessed"); + assert_eq!(rows[0][13], "sample not assessed"); + } + + #[test] + fn a_short_row_is_an_error() { + let d = crate::testutil::Dir::new("mocha-short"); + let stats = table( + &d, + "s.tsv", + "sample_id\tcall_rate\tcov_median\tbaf_auto\tn_hets\nS\t1\t31\t0\t1\n", + &NEED_STATS, + ); + let c = table( + &d, + "c.tsv", + &format!("{CALL_HEAD}S\tchr20\t1\n"), + &NEED_CALLS, + ); + let e = report("S", &c, &stats, &[], "beagle").unwrap_err(); + assert!( + e.0.ends_with("c.tsv:2: the row ends before column length"), + "{e}" + ); + } +} diff --git a/rust/crates/og-reports/src/viral.rs b/rust/crates/og-reports/src/viral.rs new file mode 100644 index 000000000..1bcc276a9 --- /dev/null +++ b/rust/crates/og-reports/src/viral.rs @@ -0,0 +1,600 @@ +//! `og viral report` (step 47): the viral DNA in the reads that do not map +//! to the human genome. +//! +//! ```text +//! og viral report --sample S --viral-fasta viral_genomes.fa +//! --viral-coverage S_viral_coverage.tsv --viral-depth S_viral_depth.tsv +//! --human-coverage S_human_coverage.tsv --out S_viral.tsv +//! ``` +//! +//! The unmapped reads are realigned to the viral genomes; `samtools +//! coverage` of that BAM gives reads per virus, `samtools depth -aa` the +//! depth at every viral base, and `samtools coverage` of the sample's BAM +//! the human depth (chr1 to chr22, over the bases with at least one read). +//! One row per viral genome, in the FASTA's order. +//! +//! The call: inherited integrated HHV-6 when the mode of the per-base depth +//! over HHV-6A or HHV-6B is at least 0.30 of the human depth (Liu et al. +//! 2020, PLoS Genet, doi:10.1371/journal.pgen.1008915: one integrated copy +//! per cell gives about 0.5, viraemia 10 to 1000 times less; when both +//! species pass, the higher one is called and the other row reads as +//! reads shared by the two); `lab control (phiX)` for Illumina's spike-in; +//! `viral DNA found` (10 reads or more over 10% of the genome or more); +//! `a few reads`; `not found`. None of these is a diagnosis of infection. +//! +//! The depth file (about 1.6 million lines over the 24 genomes) is read +//! once into a histogram per genome: memory is bounded by the number of +//! distinct depths, not the genome length. +//! +//! What changes on purpose: a number in a samtools table that is not one +//! stops with the file and line (`ERROR:`, exit 1) instead of a traceback, +//! and problems start `ERROR:`, not `viral_report.py:`. + +use crate::Error; +use indexmap::IndexMap; +use og_filters::args::Spec; +use og_io::Utf8; +use std::collections::{BTreeMap, HashMap}; +use std::io::Write; + +const MIN_READS: i128 = 10; +const FOUND_BREADTH: f64 = 0.10; +const ICI_RELATIVE_DEPTH: f64 = 0.30; +pub const HEADER: [&str; 12] = [ + "virus", + "accession", + "length", + "reads", + "breadth", + "mean_depth", + "mode_depth", + "human_depth", + "relative_depth", + "copies_per_cell", + "call", + "note", +]; + +/// The call of one viral genome. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ViralCall { + Ici, + LabControl, + Found, + FewReads, + NotFound, +} + +impl ViralCall { + fn text(self) -> &'static str { + match self { + ViralCall::Ici => "inherited integrated HHV-6 likely", + ViralCall::LabControl => "lab control (phiX)", + ViralCall::Found => "viral DNA found", + ViralCall::FewReads => "a few reads", + ViralCall::NotFound => "not found", + } + } +} + +fn hhv6(base: &str) -> Option<&'static str> { + match base { + "NC_001664" => Some("HHV-6A"), + "NC_000898" => Some("HHV-6B"), + _ => None, + } +} + +const PHIX: &str = "NC_001422"; +/// HCV: an RNA virus with no DNA stage. +const RNA_VIRUS: &str = "NC_004102"; + +fn base(acc: &str) -> &str { + acc.split('.').next().unwrap_or(acc) +} + +/// `[(accession, title)]` of the FASTA, in file order. +fn read_fasta_titles(path: &str) -> Result, Error> { + let mut out = Vec::new(); + let file = og_io::open(path).map_err(|e| Error::io(&e, path))?; + let mut lines = og_io::Lines::new(file, Utf8::Replace); + let mut line = String::new(); + while lines + .read_line(&mut line) + .map_err(|e| Error::io(&e, path))? + { + if let Some(rest) = line.strip_prefix('>') { + let t = crate::strip(rest); + let (acc, title) = t.split_once(' ').unwrap_or((t, "")); + let title = crate::strip(title); + out.push(( + acc.to_owned(), + if title.is_empty() { acc } else { title }.to_owned(), + )); + } + } + if out.is_empty() { + return Err(Error(format!("{path} has no sequence"))); + } + Ok(out) +} + +/// One `samtools coverage` row. +#[derive(Debug, Clone, Copy)] +struct Cov { + reads: i128, + length: i128, + covbases: i128, + meandepth: f64, +} + +/// `samtools coverage` rows by contig, in file order. +fn read_coverage(path: &str) -> Result, Error> { + let mut rows = IndexMap::new(); + let mut header: Option> = None; + for (i, line) in crate::read_lines(path, Utf8::Replace)?.iter().enumerate() { + let cols: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(line) + .split('\t') + .collect(); + if line.starts_with('#') { + header = Some( + cols.iter() + .map(|c| c.trim_start_matches('#').to_owned()) + .collect(), + ); + continue; + } + let Some(h) = header.as_ref().filter(|h| h.len() == cols.len()) else { + return Err(Error(format!("{path} is not samtools coverage output"))); + }; + let r: HashMap<&str, &str> = h + .iter() + .map(String::as_str) + .zip(cols.iter().copied()) + .collect(); + let cell = |k: &str| { + r.get(k).copied().ok_or_else(|| { + Error(format!( + "{path} is not samtools coverage output (no {k} column)" + )) + }) + }; + let bad = |k: &str, v: &str| { + Error(format!( + "{path} line {}: {k} {} is not a number", + i + 1, + og_filters::py_repr(v) + )) + }; + let int = |k: &str| -> Result { + let v = cell(k)?; + crate::int(v).ok_or_else(|| bad(k, v)) + }; + let mean = cell("meandepth")?; + let cov = Cov { + reads: int("numreads")?, + length: int("endpos")? - int("startpos")? + 1, + covbases: int("covbases")?, + meandepth: crate::float(mean).ok_or_else(|| bad("meandepth", mean))?, + }; + rows.insert(cell("rname")?.to_owned(), cov); + } + Ok(rows) +} + +/// chr1 to chr22, with or without the prefix. +fn autosome(name: &str) -> bool { + let n = name.strip_prefix("chr").unwrap_or(name); + let b = n.as_bytes(); + match b { + [d] => (b'1'..=b'9').contains(d), + [b'1', d] => d.is_ascii_digit(), + [b'2', d] => (b'0'..=b'2').contains(d), + _ => false, + } +} + +/// Mean depth over chr1 to chr22's bases with at least one read. +fn human_depth(cov: &IndexMap) -> Option { + let (mut total, mut covered) = (0.0f64, 0.0f64); + for (name, r) in cov { + if autosome(name) && r.covbases > 0 { + total += r.meandepth * r.length as f64; + covered += r.covbases as f64; + } + } + (covered != 0.0).then(|| total / covered) +} + +/// Depths seen over one genome: small depths in a vector, the rest in a map. +#[derive(Default)] +struct Hist { + small: Vec, + big: BTreeMap, + positions: u64, +} + +const SMALL: usize = 4096; + +impl Hist { + fn add(&mut self, d: i128) { + self.positions += 1; + match usize::try_from(d) { + Ok(i) if i < SMALL => { + if self.small.len() <= i { + self.small.resize(i + 1, 0); + } + self.small[i] += 1; + } + _ => *self.big.entry(d).or_default() += 1, + } + } + + /// (depth, count) of every depth seen, smallest first. + fn counts(&self) -> Vec<(i128, u64)> { + let mut v: Vec<(i128, u64)> = self.big.range(..0).map(|(d, c)| (*d, *c)).collect(); + v.extend( + self.small + .iter() + .enumerate() + .filter(|(_, c)| **c > 0) + .map(|(d, c)| (d as i128, *c)), + ); + v.extend(self.big.range(0..).map(|(d, c)| (*d, *c))); + v + } +} + +/// (breadth, mean, mode) per genome of `lengths`, from `samtools depth -aa`. +fn read_depth( + path: &str, + lengths: &IndexMap, +) -> Result, Error> { + let file = og_io::open(path).map_err(|e| Error::io(&e, path))?; + let mut lines = og_io::Lines::new(file, Utf8::Replace); + let mut hist: HashMap = HashMap::new(); + let mut line = String::new(); + let mut n = 0u64; + let mut last: Option = None; + while lines + .read_line(&mut line) + .map_err(|e| Error::io(&e, path))? + { + n += 1; + let mut cols = line.split('\t'); + let (Some(name), Some(_), Some(d), None) = + (cols.next(), cols.next(), cols.next(), cols.next()) + else { + return Err(Error(format!( + "{path} line {n} is not samtools depth output (contig, position, depth)" + ))); + }; + let d = crate::int(d).ok_or_else(|| { + Error(format!( + "{path} line {n}: depth {} is not a whole number", + og_filters::py_repr(d.trim_end_matches('\n')) + )) + })?; + // The file runs contig by contig: look the contig up only when it changes. + if last.as_deref() != Some(name) { + last = Some(name.to_owned()); + hist.entry(name.to_owned()).or_default(); + } + if let Some(h) = hist.get_mut(name) { + h.add(d); + } + } + let mut out = HashMap::new(); + for (name, &length) in lengths { + let seen = hist.get(name).map_or(0, |h| h.positions); + if seen != 0 && i128::from(seen) != length { + return Err(Error(format!( + "{path} has {seen} positions for {name}, its length is {length} (run samtools depth -aa)" + ))); + } + if length == 0 { + return Err(Error(format!("{name} has length 0 in the viral coverage"))); + } + let counts = match hist.get(name) { + Some(h) if h.positions > 0 => h.counts(), + _ => vec![(0, u64::try_from(length).unwrap_or(0))], + }; + let zero = counts.iter().find(|(d, _)| *d == 0).map_or(0, |(_, c)| *c); + let breadth = 1.0 - zero as f64 / length as f64; + let sum: i128 = counts.iter().map(|(d, c)| d * i128::from(*c)).sum(); + let mean = sum as f64 / length as f64; + // The most common depth; a tie goes to the lower one. + let mode = counts + .iter() + .fold(None::<(i128, u64)>, |best, &(d, c)| match best { + Some((_, bc)) if bc >= c => best, + _ => Some((d, c)), + }) + .map_or(0, |(d, _)| d); + out.insert(name.clone(), (breadth, mean, mode)); + } + Ok(out) +} + +/// A word character for Python's `\b` on a str. +fn word(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +/// The title without its `, complete genome` tail: +/// `re.sub(r",? (complete (virion )?genome|genome|DNA)\b.*$", "", title)`. +fn short_title(title: &str) -> &str { + const ALTS: [&str; 4] = ["complete virion genome", "complete genome", "genome", "DNA"]; + let at = |rest: &str| { + ALTS.iter().any(|a| { + rest.strip_prefix(a) + .is_some_and(|after| !after.chars().next().is_some_and(word)) + }) + }; + for (i, _) in title.char_indices() { + let rest = &title[i..]; + let tail = rest.strip_prefix(", ").or_else(|| rest.strip_prefix(' ')); + if tail.is_some_and(at) { + return &title[..i]; + } + } + title +} + +fn short_name(acc: &str, title: &str) -> String { + if let Some(n) = hhv6(base(acc)) { + return n.into(); + } + let t = crate::strip(short_title(title)); + if t.is_empty() { acc } else { t }.to_owned() +} + +/// The call and its note for one viral genome (before the HHV-6 pick). +fn call_row(acc: &str, reads: i128, breadth: f64, relative: Option) -> (ViralCall, String) { + let b = base(acc); + if b == PHIX { + if reads != 0 { + return ( + ViralCall::LabControl, + "Illumina adds phiX to the run as a control: these reads were added in the lab." + .into(), + ); + } + return ( + ViralCall::NotFound, + "Illumina's phiX control: none of its reads reached this sample's data.".into(), + ); + } + if let (Some(_), Some(rel)) = (hhv6(b), relative) + && rel >= ICI_RELATIVE_DEPTH + { + return ( + ViralCall::Ici, + format!( + "Depth over the genome at {} of the human depth (Liu et al. 2020 call integration at {} or more; one copy per cell gives about 0.5). Inherited integrated HHV-6 is in every cell and is passed on to about half of the children; it can make a blood test read as an active HHV-6 infection.", + crate::fixed(rel, 2), + crate::fixed(ICI_RELATIVE_DEPTH, 2) + ), + ); + } + if reads < MIN_READS { + return (ViralCall::NotFound, String::new()); + } + let note = if hhv6(b).is_some() { + if relative.is_some() { + "Below the integration threshold of Liu et al. 2020: not the one-copy-per-cell pattern of inherited HHV-6." + } else { + "No human depth (no read on chr1 to chr22): integration is not assessed." + } + } else if b == RNA_VIRUS { + "An RNA virus with no DNA stage: DNA reads point to a contamination or an integration (Moustafa et al. 2017)." + } else { + "" + }; + let call = if breadth >= FOUND_BREADTH { + ViralCall::Found + } else { + ViralCall::FewReads + }; + (call, note.into()) +} + +/// One row: the cells of [`HEADER`] and the call. +struct Row { + virus: String, + cells: Vec, + call: ViralCall, + note: String, + relative: f64, +} + +/// The files of `og viral report`. +pub struct Files<'a> { + pub viral_fasta: &'a str, + pub viral_coverage: &'a str, + pub viral_depth: &'a str, + pub human_coverage: &'a str, + pub out: &'a str, +} + +/// `og viral report`: writes the table; the stdout line. +pub fn report(sample: &str, f: &Files<'_>) -> Result { + let titles = read_fasta_titles(f.viral_fasta)?; + let vcov = read_coverage(f.viral_coverage)?; + let lengths: IndexMap = titles + .iter() + .filter_map(|(a, _)| vcov.get(a).map(|c| (a.clone(), c.length))) + .collect(); + let vdepth = read_depth(f.viral_depth, &lengths)?; + let hdepth = human_depth(&read_coverage(f.human_coverage)?).filter(|h| *h != 0.0); + let mut rows = Vec::new(); + for (acc, title) in &titles { + let Some(c) = vcov.get(acc) else { + return Err(Error(format!( + "the viral coverage has no row for {acc}: was the BAM aligned to this FASTA?" + ))); + }; + let (breadth, mean, mode) = vdepth[acc]; + let relative = hdepth.map(|h| mode as f64 / h); + let copies = hdepth.map(|h| 2.0 * mean / h); + let (call, note) = call_row(acc, c.reads, breadth, relative); + let virus = short_name(acc, title); + rows.push(Row { + cells: vec![ + virus.clone(), + acc.clone(), + c.length.to_string(), + c.reads.to_string(), + crate::fixed(breadth, 4), + crate::fixed(mean, 3), + mode.to_string(), + hdepth.map_or(".".into(), |h| crate::fixed(h, 2)), + relative.map_or(".".into(), |r| crate::fixed(r, 3)), + copies.map_or(".".into(), |x| crate::fixed(x, 4)), + ], + virus, + call, + note, + relative: relative.unwrap_or(0.0), + }); + } + // Both HHV-6 species pass: the higher relative depth is the integrated + // genome; the other's depth is reads shared by the two. + let ici: Vec = (0..rows.len()) + .filter(|&i| rows[i].call == ViralCall::Ici) + .collect(); + if ici.len() > 1 { + let keep = ici.iter().copied().fold(ici[0], |k, i| { + if rows[i].relative > rows[k].relative { + i + } else { + k + } + }); + let name = rows[keep].virus.clone(); + for &i in &ici { + if i != keep { + rows[i].call = ViralCall::Found; + rows[i].note = format!( + "Reads shared with {name}, which has the higher depth and is the integrated species called." + ); + } + } + } + let mut w = pycompat::csv::Writer::new(Vec::new(), '\t'); + let io = |e: std::io::Error| Error::io(&e, f.out); + w.write_row(HEADER).map_err(io)?; + for r in &rows { + let mut cells = r.cells.clone(); + cells.push(r.call.text().into()); + cells.push(r.note.clone()); + w.write_row(cells).map_err(io)?; + } + crate::write_atomic(f.out, &w.into_inner())?; + let found: Vec = rows + .iter() + .filter(|r| r.call != ViralCall::NotFound) + .map(|r| format!("{} ({})", r.virus, r.call.text())) + .collect(); + Ok(format!( + "{sample}: {} viral genomes; {}\n", + rows.len(), + if found.is_empty() { + "no viral DNA found".to_owned() + } else { + found.join("; ") + } + )) +} + +const SPEC: Spec = Spec { + prog: "og viral report", + usage: "usage: og viral report [-h] --sample SAMPLE --viral-fasta VIRAL_FASTA --viral-coverage VIRAL_COVERAGE --viral-depth VIRAL_DEPTH --human-coverage HUMAN_COVERAGE --out OUT\n", + options: &[ + "--sample", + "--viral-fasta", + "--viral-coverage", + "--viral-depth", + "--human-coverage", + "--out", + ], + positionals: &[], + required: &[ + "--sample", + "--viral-fasta", + "--viral-coverage", + "--viral-depth", + "--human-coverage", + "--out", + ], +}; + +/// `og viral VERB ARGS`: the exit code. +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if verb != Some("report") { + return crate::unknown_verb("viral", "report", verb, err); + } + crate::run(&SPEC, rest, out, err, |a| { + let files = Files { + viral_fasta: crate::opt(a, "--viral-fasta"), + viral_coverage: crate::opt(a, "--viral-coverage"), + viral_depth: crate::opt(a, "--viral-depth"), + human_coverage: crate::opt(a, "--human-coverage"), + out: crate::opt(a, "--out"), + }; + Ok(report(crate::opt(a, "--sample"), &files).map_err(|e| (e, 1))) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn titles_and_names() { + assert_eq!( + short_title("Human gammaherpesvirus 4, complete genome"), + "Human gammaherpesvirus 4" + ); + assert_eq!( + short_title("Hepatitis B virus genome sequence"), + "Hepatitis B virus" + ); + assert_eq!( + short_title("Torque teno virus DNA, isolate x"), + "Torque teno virus" + ); + assert_eq!(short_title("Some genomes"), "Some genomes"); + assert_eq!(short_name("NC_001664.4", "x"), "HHV-6A"); + assert_eq!(short_name("NC_1", ", complete genome"), "NC_1"); + assert!(autosome("chr22") && autosome("7") && !autosome("chr23") && !autosome("chr0")); + } + + #[test] + fn mode_tie_goes_to_the_lower_depth() { + let d = crate::testutil::Dir::new("viral"); + let p = d.put("d.tsv", "V\t1\t3\nV\t2\t3\nV\t3\t1\nV\t4\t1\n"); + let lengths: IndexMap = [("V".to_owned(), 4)].into_iter().collect(); + let got = read_depth(&p, &lengths).unwrap(); + assert_eq!(got["V"], (1.0, 2.0, 1)); + } + + /// Red first: a relative depth of exactly 0.30 is integrated HHV-6. + #[test] + fn red_first_the_ici_threshold_is_inclusive() { + assert_eq!( + call_row("NC_000898.1", 0, 0.0, Some(0.30)).0, + ViralCall::Ici + ); + assert_eq!( + call_row("NC_000898.1", 50, 0.5, Some(0.2999)).0, + ViralCall::Found + ); + assert_eq!( + call_row("NC_001422.1", 3, 0.0, None).0, + ViralCall::LabControl + ); + } +} diff --git a/rust/crates/og-reports/src/vntyper.rs b/rust/crates/og-reports/src/vntyper.rs new file mode 100644 index 000000000..9228dc486 --- /dev/null +++ b/rust/crates/og-reports/src/vntyper.rs @@ -0,0 +1,483 @@ +//! `og vntyper report` (step 52): VNtyper's MUC1 VNTR result in plain words. +//! +//! ADTKD-MUC1 is caused by frameshifts inside the MUC1 coding VNTR, a repeat +//! of 60 bp units, most often one extra C in a run of seven (dupC; Kirby et +//! al., Nat Genet 2013, PMID 23396133). Short reads cannot be placed inside +//! the repeat, so standard callers do not see it. VNtyper (Saei et al., +//! iScience 2023, PMID 37456840) genotypes it with k-mers (Kestrel); this +//! reads the run folder VNtyper writes and says positive, possible, negative +//! or not assessed, with the share of known positives VNtyper found at the +//! depth this sample has over the VNTR. +//! +//! ```text +//! og vntyper report --run-dir DIR --out TABLE [--exit-code N --log FILE] +//! ``` +//! +//! DIR is VNtyper's `--output-dir` (`kestrel/kestrel_result.tsv` and +//! `coverage/coverage_summary.tsv`). `--exit-code` is VNtyper's own exit +//! status; a non-zero one, a missing file, or a Kestrel file with no result +//! row gives "not assessed". `--log`, VNtyper's console output, tells a BAM +//! with no read over MUC1 (VNtyper 2.0.47 stops on it with "FASTQ layout +//! 'empty'") from other errors. TABLE gets `metricvalue` rows. +//! +//! What changes on purpose: a negative mean depth in VNtyper's coverage +//! table is an input error (exit 2); the Python stopped on an assertion. + +use crate::{Error, Outcome}; +use og_filters::args::Spec; +use og_tsv::{KvTable, Options, Utf8}; +use std::io::Write; +use std::path::Path; + +/// The rows of `FIELDS`, in order. +pub const FIELDS: [&str; 13] = [ + "status", + "confidence", + "variant", + "flag", + "alt_depth", + "region_depth", + "depth_score", + "vntr_mean_depth", + "vntr_uncovered_pct", + "coverage_qc", + "sensitivity", + "message", + "advice", +]; + +/// What VNtyper 2.0.47 says about sensitivity at depth (docs/user-guide/ +/// configuration.md, "Coverage thresholds"): confirmed positive GRCh38 +/// exomes downsampled and re-run, 42 samples at nine fractions each. Rows: +/// lowest mean depth over the VNTR, detected runs, runs. +const SENSITIVITY: [(f64, u32, u32); 3] = [(100.0, 160, 161), (50.0, 95, 108), (0.0, 48, 109)]; +const SENSITIVITY_SOURCE: &str = "VNtyper 2.0.47's downsampling of 42 confirmed positive GRCh38 exomes (repeated runs, not independent tests)"; + +const ADVICE_CALL: &str = "A research call, not a diagnosis. VNtyper's authors ask for confirmation by another method: SNaPshot for dupC, long-read sequencing for other variants. ADTKD-MUC1 is dominant: discuss it with a nephrologist or a clinical geneticist."; +const ADVICE_NONE: &str = "A negative here lowers the chance of the frameshifts VNtyper looks for; it does not rule ADTKD-MUC1 out. A family history of kidney failure with gout or a bland urine sediment still deserves a clinical test (SNaPshot for dupC)."; + +const NO_READS: &str = "FASTQ layout 'empty'"; +const REGION: &str = "chr1:155,184,000-155,194,000"; +const KESTREL: &str = "kestrel/kestrel_result.tsv"; +const COVERAGE: &str = "coverage/coverage_summary.tsv"; + +/// Kestrel's confidence of its best call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Confidence { + HighPrecisionStar, + HighPrecision, + LowPrecision, + /// Anything else, `Negative` when Kestrel left it empty: no call. + Other(String), +} + +impl Confidence { + fn read(s: &str) -> Confidence { + match s { + "High_Precision*" => Confidence::HighPrecisionStar, + "High_Precision" => Confidence::HighPrecision, + "Low_Precision" => Confidence::LowPrecision, + "" => Confidence::Other("Negative".into()), + other => Confidence::Other(other.into()), + } + } + + fn text(&self) -> &str { + match self { + Confidence::HighPrecisionStar => "High_Precision*", + Confidence::HighPrecision => "High_Precision", + Confidence::LowPrecision => "Low_Precision", + Confidence::Other(s) => s, + } + } +} + +/// Why a caveat makes a call "possible", not "positive". +#[derive(Debug, Clone, PartialEq)] +enum Caveat { + Flagged(String), + LowPrecision, + Uncovered(f64), + NoUncoveredMeasure, +} + +impl Caveat { + fn text(&self) -> String { + match self { + Caveat::Flagged(f) => format!("flagged: {f}"), + Caveat::LowPrecision => "low precision".into(), + Caveat::Uncovered(u) => format!("{}% of the VNTR has no read", crate::fixed(*u, 0)), + Caveat::NoUncoveredMeasure => { + "VNtyper measured no share of the VNTR without reads".into() + } + } + } +} + +/// The table, one value per field of [`FIELDS`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reading(Vec<(&'static str, String)>); + +impl Reading { + fn new() -> Reading { + Reading(FIELDS.iter().map(|f| (*f, ".".to_owned())).collect()) + } + + fn set(&mut self, key: &str, value: impl Into) { + if let Some(slot) = self.0.iter_mut().find(|(k, _)| *k == key) { + slot.1 = value.into(); + } + } + + /// The value of `key`. + pub fn get(&self, key: &str) -> &str { + self.0 + .iter() + .find(|(k, _)| *k == key) + .map_or(".", |(_, v)| v) + } + + fn not_assessed(mut self, message: String) -> Reading { + self.set("status", "not assessed"); + self.set("message", message); + self.set("advice", ADVICE_NONE); + self + } +} + +/// A number of VNtyper's tables: `None` for a missing cell, text that is no +/// number, or NaN. +fn number(v: Option<&str>) -> Option { + v.and_then(crate::float).filter(|x| !x.is_nan()) +} + +/// The sentence on VNtyper's sensitivity at the sample's mean depth. +fn sensitivity_line(mean: Option) -> Result { + let Some(mean) = mean else { + return Ok("Sensitivity not known: VNtyper measured no depth over the VNTR.".into()); + }; + let Some(&(low, hit, runs)) = SENSITIVITY.iter().find(|(low, ..)| mean >= *low) else { + return Err(Error(format!( + "VNtyper's coverage table gives a negative mean depth over the VNTR ({})", + pycompat::fmt::repr(mean) + ))); + }; + let band = if low == 100.0 { + format!("at {}x or more", crate::fixed(low, 0)) + } else if low != 0.0 { + format!("between {}x and 100x", crate::fixed(low, 0)) + } else { + "below 50x".to_owned() + }; + Ok(format!( + "Mean depth over the MUC1 VNTR: {}x. At that depth ({band}) VNtyper found the known variant in {hit} of {runs} runs ({}%), in {SENSITIVITY_SOURCE}.", + crate::fixed(mean, 0), + crate::fixed(100.0 * f64::from(hit) / f64::from(runs), 0) + )) +} + +/// The data rows of a VNtyper TSV: its `#` banner lines and blank lines +/// skipped, the first line left the header. +fn rows(path: &str) -> Result, Error> { + let opts = Options::default() + .utf8(Utf8::Strict) + .skip(|l| l.starts_with('#') || crate::strip(l).is_empty()); + Ok(crate::read_table(path, opts)?.1) +} + +/// The reading of VNtyper's run folder `run_dir`. +pub fn read(run_dir: &str, exit_code: i64, log_text: &str) -> Result { + let out = Reading::new(); + if exit_code != 0 { + let why = if log_text.contains(NO_READS) { + format!( + "The BAM has no read in the MUC1 region ({REGION}), so VNtyper had nothing to read." + ) + } else { + format!("VNtyper stopped with exit code {exit_code}: see its log.") + }; + return Ok(out.not_assessed(why)); + } + let dir = Path::new(run_dir); + let missing: Vec<&str> = [KESTREL, COVERAGE] + .into_iter() + .filter(|f| !dir.join(f).is_file()) + .collect(); + if !missing.is_empty() { + return Ok(out.not_assessed(format!("VNtyper wrote no {}.", missing.join(" and no ")))); + } + let path = |f: &str| dir.join(f).display().to_string(); + let cov = rows(&path(COVERAGE))?; + let cov = cov.first(); + let cell = |r: Option<&og_tsv::Row>, k: &str| r.and_then(|r| r.get(k)).map(str::to_owned); + let mean = number(cell(cov, "mean").as_deref()); + let uncovered = number(cell(cov, "percent_uncovered").as_deref()); + let qc = cell(cov, "coverage_qc") + .map(|s| crate::strip(&s).to_owned()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "NOT_EVALUATED".into()); + let k = rows(&path(KESTREL))?; + let Some(best) = k.first() else { + return Ok( + out.not_assessed("VNtyper wrote kestrel/kestrel_result.tsv with no result row.".into()) + ); + }; + let get = |c: &str| best.get(c).unwrap_or(""); + let conf = Confidence::read(crate::strip(get("Confidence"))); + let flag = crate::strip(get("Flag")).to_owned(); + let flagged = !flag.is_empty() && flag != "Not flagged"; + let covered = qc == "PASS" || qc == "REDUCED"; + let mut out = out; + out.set("confidence", conf.text()); + out.set("coverage_qc", qc.clone()); + out.set("sensitivity", sensitivity_line(mean)?); + out.set( + "vntr_mean_depth", + mean.map_or(".".into(), |m| crate::fixed(m, 2)), + ); + out.set( + "vntr_uncovered_pct", + uncovered.map_or(".".into(), |u| crate::fixed(u, 2)), + ); + if !matches!(conf, Confidence::Other(_)) { + let name = crate::strip(get("Nomenclature")); + let variant = if name.is_empty() { + let v = crate::strip(best.get("Variant").filter(|v| !v.is_empty()).unwrap_or(".")); + if v.is_empty() { "." } else { v } + } else { + name + }; + let or_dot = |c: &str| { + let v = get(c); + if v.is_empty() { + ".".to_owned() + } else { + v.to_owned() + } + }; + out.set("variant", variant); + out.set("flag", if flag.is_empty() { "." } else { &flag }); + out.set("alt_depth", or_dot("Estimated_Depth_AlternateVariant")); + out.set( + "region_depth", + or_dot("Estimated_Depth_Variant_ActiveRegion"), + ); + out.set("depth_score", or_dot("Depth_Score")); + out.set("advice", ADVICE_CALL); + let what = format!( + "a MUC1 VNTR frameshift ({variant}), Kestrel confidence {}", + conf.text() + ); + let high = matches!( + conf, + Confidence::HighPrecision | Confidence::HighPrecisionStar + ); + if high && !flagged && uncovered.is_some_and(|u| u <= 50.0) { + out.set("status", "positive"); + out.set("message", format!("VNtyper calls {what}.")); + } else { + let mut why = Vec::new(); + if flagged { + why.push(Caveat::Flagged(flag.clone())); + } + if conf == Confidence::LowPrecision { + why.push(Caveat::LowPrecision); + } + match uncovered { + Some(u) if u > 50.0 => why.push(Caveat::Uncovered(u)), + None => why.push(Caveat::NoUncoveredMeasure), + Some(_) => {} + } + let why: Vec = why.iter().map(Caveat::text).collect(); + out.set("status", "possible"); + out.set( + "message", + format!("VNtyper calls {what}, with caveats ({}).", why.join("; ")), + ); + } + return Ok(out); + } + if covered { + out.set("status", "negative"); + out.set("message", "VNtyper calls no MUC1 VNTR frameshift."); + out.set("advice", ADVICE_NONE); + return Ok(out); + } + let depth = mean.map_or("no measured depth".into(), |m| { + format!("{}x", crate::fixed(m, 0)) + }); + Ok(out.not_assessed(format!( + "VNtyper calls no frameshift, but its coverage check is {qc} ({depth} over the VNTR, 50x is its floor on GRCh38), so a frameshift cannot be ruled out." + ))) +} + +const SPEC: Spec = Spec { + prog: "og vntyper report", + usage: "usage: og vntyper report [-h] --run-dir RUN_DIR --out OUT [--exit-code EXIT_CODE] [--log LOG]\n", + options: &["--run-dir", "--out", "--exit-code", "--log"], + positionals: &[], + required: &["--run-dir", "--out"], +}; + +/// `og vntyper VERB ARGS`: the exit code (2 for unreadable results or a +/// refused command line). +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if verb != Some("report") { + return crate::unknown_verb("vntyper", "report", verb, err); + } + crate::run(&SPEC, rest, out, err, |a| { + let exit_code = crate::int_opt(a, "--exit-code", 0)?; + let run_dir = crate::opt(a, "--run-dir"); + Ok(report( + run_dir, + exit_code, + a.get("--log"), + crate::opt(a, "--out"), + )) + }) +} + +fn report(run_dir: &str, exit_code: i64, log: Option<&str>, out: &str) -> Outcome { + let unreadable = |e: Error| { + ( + Error(format!("cannot read VNtyper's results in {run_dir}: {e}")), + 2, + ) + }; + let log_text = match log.filter(|l| Path::new(l).is_file()) { + Some(l) => crate::read_lines(l, Utf8::Replace) + .map_err(unreadable)? + .concat(), + None => String::new(), + }; + let t = read(run_dir, exit_code, &log_text).map_err(unreadable)?; + let mut kv = KvTable::metrics(); + for k in FIELDS { + kv.push(k, t.get(k)); + } + let mut body = Vec::new(); + kv.write_to(&mut body) + .map_err(|e| (Error::io(&e, out), 2))?; + crate::write_atomic(out, &body).map_err(|e| (e, 1))?; + Ok(String::new()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + const KHEAD: &str = "Motif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\tNomenclature\n"; + + fn run(kestrel: Option<&str>, cov: Option<&str>, exit: i64, log: &str) -> Reading { + let d = Dir::new("vntyper"); + if let Some(k) = kestrel { + d.put("kestrel/kestrel_result.tsv", k); + } + if let Some(c) = cov { + d.put("coverage/coverage_summary.tsv", c); + } + read(&d.root().display().to_string(), exit, log).unwrap() + } + + fn cov(mean: &str, unc: &str, qc: &str) -> String { + format!("mean\tpercent_uncovered\tcoverage_qc\n{mean}\t{unc}\t{qc}\n") + } + + fn call(conf: &str, flag: &str) -> String { + format!("## VNtyper\n{KHEAD}X\tdupC\t60\tG\tGC\tm\t12\t300\t0.04\t{conf}\t{flag}\tdupC\n") + } + + #[test] + fn statuses() { + let r = run( + Some(&call("High_Precision", "Not flagged")), + Some(&cov("212.4", "0.0", "PASS")), + 0, + "", + ); + assert_eq!(r.get("status"), "positive"); + assert_eq!(r.get("vntr_mean_depth"), "212.40"); + let r = run( + Some(&call("Low_Precision", "")), + Some(&cov("212.4", "0.0", "PASS")), + 0, + "", + ); + assert_eq!(r.get("status"), "possible"); + assert!(r.get("message").contains("(low precision)")); + let r = run( + Some(&call("High_Precision*", "False_Positive_4bp")), + Some(&cov("80", "60", "PASS")), + 0, + "", + ); + assert!( + r.get("message").ends_with( + "with caveats (flagged: False_Positive_4bp; 60% of the VNTR has no read)." + ) + ); + let r = run(Some(KHEAD), Some(&cov("80", "0", "PASS")), 0, ""); + assert_eq!(r.get("status"), "not assessed"); + let r = run( + Some(&format!( + "{KHEAD}x\tNone\t\t\t\t\t\t\t\tNegative\tNot flagged\t\n" + )), + Some(&cov("30", "0", "FAIL")), + 0, + "", + ); + assert_eq!(r.get("status"), "not assessed"); + assert!(r.get("message").contains("is FAIL (30x over")); + let r = run(None, None, 1, "x FASTQ layout 'empty' y"); + assert!(r.get("message").starts_with("The BAM has no read")); + let r = run(None, Some(&cov("1", "1", "PASS")), 0, ""); + assert_eq!( + r.get("message"), + "VNtyper wrote no kestrel/kestrel_result.tsv." + ); + } + + /// Red first: a call with exactly half the VNTR uncovered is positive. + #[test] + fn red_first_half_uncovered_is_still_positive() { + let r = run( + Some(&call("High_Precision", "Not flagged")), + Some(&cov("212.4", "50", "PASS")), + 0, + "", + ); + assert_eq!(r.get("status"), "positive"); + } + + #[test] + fn sensitivity_bands_and_ties() { + assert!( + sensitivity_line(Some(100.0)) + .unwrap() + .contains("(at 100x or more)") + ); + assert!( + sensitivity_line(Some(50.0)) + .unwrap() + .contains("(between 50x and 100x)") + ); + // Half-even on the exact value, as Python's format. + assert!( + sensitivity_line(Some(48.5)) + .unwrap() + .starts_with("Mean depth over the MUC1 VNTR: 48x.") + ); + assert!( + sensitivity_line(Some(49.5)) + .unwrap() + .starts_with("Mean depth over the MUC1 VNTR: 50x.") + ); + assert!( + sensitivity_line(Some(10.0)) + .unwrap() + .contains("in 48 of 109 runs (44%)") + ); + assert!(sensitivity_line(Some(-1.0)).is_err()); + } +} diff --git a/rust/crates/og-reports/src/ystr.rs b/rust/crates/og-reports/src/ystr.rs new file mode 100644 index 000000000..1f47390ed --- /dev/null +++ b/rust/crates/og-reports/src/ystr.rs @@ -0,0 +1,704 @@ +//! `og y str-bed` and `og y str-report` (step 45): Y-STR repeat counts from +//! HipSTR's calls, in FamilyTreeDNA's marker order. +//! +//! ```text +//! og y str-bed --panel assets/ystr_panel.tsv --out regions.bed +//! og y str-report --panel assets/ystr_panel.tsv --vcf _y_str.vcf.gz \ +//! --tsv _y_str.tsv --csv _y_str.csv [--log _y_str.log] [--read-length 150] +//! ``` +//! +//! `str-bed` writes HipSTR's region file: every region of the panel's +//! single and parts markers, and one region per span marker, sorted by +//! position. `str-report` writes one row per panel marker, in the panel's +//! order; with HipSTR's log, a region with no record carries the log's +//! reason. +//! +//! How a count is made: HipSTR writes per region GB, the base pairs the +//! call differs from the reference. A region's count is the panel's +//! reference copies (halves rounded up) plus GB / motif length, by floor +//! division, so GB -2 on 11 copies of a 4 bp motif is `10.2` (ten repeats +//! and two bases). Parts are summed; a span marker is one region; the +//! panel's offset is added last. Confidence: high (HipSTR's haploid filters +//! pass and 10 reads or more), low (the note says which filter failed), no +//! call, not callable. +//! +//! What changes on purpose: a motif length under 1, reference copies that +//! are negative or not finite, and an order that is not a whole number are +//! panel errors (the Python crashed later or divided by zero); an +//! unreadable VCF or log is an `ERROR:` line (exit 1), not a traceback. + +use crate::Error; +use og_filters::args::Spec; +use og_io::Utf8; +use std::collections::HashMap; +use std::io::Write; + +const MIN_READS: i128 = 10; +const MIN_Q: f64 = 0.9; +const MAX_STUTTER: f64 = 0.15; +const MAX_FLANK: f64 = 0.15; +const FLANK: i128 = 10; + +pub const TSV_COLUMNS: [&str; 10] = [ + "order", + "panel", + "marker", + "count", + "confidence", + "reads", + "stutter_reads", + "flank_indel_reads", + "q", + "note", +]; +const NEED: [&str; 8] = [ + "order", "panel", "marker", "kind", "regions", "motif", "offset", "note", +]; + +/// One HipSTR region. +#[derive(Debug, Clone, PartialEq)] +pub struct Region { + chrom: String, + start: i128, + end: i128, + period: i128, + copies: f64, + name: String, +} + +/// What a panel marker is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Single, + Parts, + Span, + Multicopy, + Absent, +} + +/// One panel row. +#[derive(Debug, Clone)] +pub struct Marker { + order: String, + panel: String, + marker: String, + kind: Kind, + /// The regions HipSTR genotypes: a span marker has one. + regions: Vec, + offset: Option, + note: String, +} + +/// The reference copies of a region, halves up. +fn halves_up(copies: f64) -> i128 { + (copies + 0.5).trunc() as i128 +} + +/// The panel's rows, checked. +pub fn read_panel(path: &str) -> Result, Error> { + let lines: Vec = crate::read_lines(path, Utf8::Strict)? + .into_iter() + .filter(|l| !crate::strip(l).is_empty() && !l.starts_with('#')) + .map(|l| l.strip_suffix('\n').unwrap_or(&l).to_owned()) + .collect(); + let Some(head) = lines.first() else { + return Err(Error(format!("{path}: no header"))); + }; + if head.split('\t').collect::>() != NEED { + return Err(Error(format!( + "{path}: the header is not {}", + NEED.join(" ") + ))); + } + let mut rows = Vec::new(); + for (i, line) in lines[1..].iter().enumerate() { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() != NEED.len() { + return Err(Error(format!( + "{path}: row {} has {} fields, not {}", + i + 2, + c.len(), + NEED.len() + ))); + } + let marker = c[2]; + let kind = match c[3] { + "single" => Kind::Single, + "parts" => Kind::Parts, + "span" => Kind::Span, + "multicopy" => Kind::Multicopy, + "absent" => Kind::Absent, + k => return Err(Error(format!("{path}: {marker}: unknown kind {k}"))), + }; + let mut regs = Vec::new(); + if c[4] != "." { + for part in c[4].split(';') { + let bad = || Error(format!("{path}: {marker}: unreadable region {part}")); + let f: Vec<&str> = part.splitn(4, '/').collect(); + let [where_, period, copies, name] = f[..] else { + return Err(bad()); + }; + let w: Vec<&str> = where_.split(':').collect(); + let [chrom, span] = w[..] else { + return Err(bad()); + }; + let s: Vec<&str> = span.split('-').collect(); + let [start, end] = s[..] else { + return Err(bad()); + }; + let (Some(start), Some(end), Some(period), Some(copies)) = ( + crate::int(start), + crate::int(end), + crate::int(period), + crate::float(copies), + ) else { + return Err(bad()); + }; + if period < 1 || !copies.is_finite() || copies < 0.0 { + return Err(bad()); + } + regs.push(Region { + chrom: chrom.into(), + start, + end, + period, + copies, + name: name.into(), + }); + } + } + let has = matches!(kind, Kind::Single | Kind::Parts | Kind::Span); + if has == regs.is_empty() + || (kind == Kind::Single && regs.len() != 1) + || (matches!(kind, Kind::Parts | Kind::Span) && regs.len() < 2) + { + return Err(Error(format!( + "{path}: {marker}: kind {} with {} regions", + c[3], + regs.len() + ))); + } + if regs.iter().any(|r| r.period != regs[0].period) { + return Err(Error(format!( + "{path}: {marker}: parts with different motif lengths" + ))); + } + let offset = if c[6] == "." { + None + } else { + Some(crate::int(c[6]).ok_or_else(|| { + Error(format!( + "{path}: {marker}: offset {} is not a whole number", + c[6] + )) + })?) + }; + if kind == Kind::Span { + if regs.iter().any(|r| r.chrom != regs[0].chrom) + || regs.windows(2).any(|w| w[1].start <= w[0].end) + { + return Err(Error(format!( + "{path}: {marker}: span parts not in order on one chromosome" + ))); + } + // One region over the whole locus, named after the marker; its + // copies are the parts' (each rounded, halves up). + let copies = regs.iter().map(|r| halves_up(r.copies)).sum::() as f64; + regs = vec![Region { + chrom: regs[0].chrom.clone(), + start: regs[0].start, + end: regs[regs.len() - 1].end, + period: regs[0].period, + copies, + name: marker.into(), + }]; + } + rows.push(Marker { + order: c[0].into(), + panel: c[1].into(), + marker: marker.into(), + kind, + regions: regs, + offset, + note: c[7].into(), + }); + } + for (i, r) in rows.iter().enumerate() { + if crate::int(&r.order) != Some(i as i128 + 1) { + return Err(Error(format!( + "{path}: the order column is not 1 to {}", + rows.len() + ))); + } + } + Ok(rows) +} + +/// HipSTR's region file. +pub fn bed(panel: &[Marker]) -> String { + let mut seen: Vec<&Region> = Vec::new(); + let mut at: HashMap<&str, usize> = HashMap::new(); + for r in panel.iter().flat_map(|m| &m.regions) { + match at.get(r.name.as_str()) { + Some(&i) => seen[i] = r, + None => { + at.insert(&r.name, seen.len()); + seen.push(r); + } + } + } + seen.sort_by(|a, b| (&a.chrom, a.start, a.end).cmp(&(&b.chrom, b.start, b.end))); + seen.iter() + .map(|r| { + format!( + "{}\t{}\t{}\t{}\t{}\t{}\n", + r.chrom, + r.start, + r.end, + r.period, + pycompat::fmt::general(r.copies, 6), + r.name + ) + }) + .collect() +} + +/// HipSTR's records by ID: the FORMAT values of the one sample. +fn read_calls(path: &str) -> Result>, Error> { + let mut calls = HashMap::new(); + let mut samples: Option = None; + for line in crate::lines(path, Utf8::Replace)? { + let line = line?; + if line.starts_with("##") { + continue; + } + let c: Vec<&str> = line + .strip_suffix('\n') + .unwrap_or(&line) + .split('\t') + .collect(); + if line.starts_with('#') { + let n = c.len().saturating_sub(9); + if n != 1 { + return Err(Error(format!("{path}: {n} samples; step 45 reads one"))); + } + samples = Some(n); + continue; + } + if samples.is_none() || c.len() < 10 { + return Err(Error(format!( + "{path}: a record before the header, or one without a sample column" + ))); + } + let fmt = c[8] + .split(':') + .zip(c[9].split(':')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(); + calls.insert(c[2].to_owned(), fmt); + } + if samples.is_none() { + return Err(Error(format!("{path}: no #CHROM header line"))); + } + Ok(calls) +} + +const LOG_REASONS: [&str; 5] = [ + "Skipping", + "Aborting", + "Stutter model training failed", + "Haplotype construction failed", + "Retraining stutter model training failed", +]; + +/// HipSTR's reason per region: (chrom, 1-based start, end) to the first +/// reason line after its "Processing region" line. +fn read_log(path: &str) -> Result, Error> { + let mut why = HashMap::new(); + let mut cur: Option<(String, i128, i128)> = None; + for line in crate::read_lines(path, Utf8::Replace)? { + let line = crate::strip(&line); + if line.starts_with("Processing region ") { + let c = pycompat::text::split(line, None, None); + cur = match ( + c.get(2), + c.get(3).and_then(|s| crate::int(s)), + c.get(4).and_then(|s| crate::int(s)), + ) { + (Some(ch), Some(s), Some(e)) => Some(((*ch).to_owned(), s + 1, e)), + _ => None, + }; + } else if let Some(k) = &cur + && !why.contains_key(k) + && LOG_REASONS.iter().any(|r| line.starts_with(r)) + { + why.insert(k.clone(), line.to_owned()); + } + } + Ok(why) +} + +/// A called region: bp difference, reads, stutter, flank indels, Q, and the +/// filters it fails. +struct PartCall { + bp: i128, + dp: i128, + st: i128, + fl: i128, + q: Option, + problems: Vec, +} + +fn part_call(fmt: &HashMap) -> Result { + let get = |k: &str| fmt.get(k).map(String::as_str); + let (gt, gb) = (get("GT").unwrap_or("."), get("GB").unwrap_or(".")); + let none = |s: &str| matches!(s, "." | "./." | ".|."); + if none(gt) || none(gb) { + let why = get("FILTER").filter(|w| !w.is_empty() && *w != "." && *w != "PASS"); + return Err(format!( + "HipSTR made no genotype{}", + why.map_or(String::new(), |w| format!(" ({w})")) + )); + } + let mut lens: Vec> = gb.replace('|', "/").split('/').map(crate::int).collect(); + lens.sort(); + lens.dedup(); + if lens.contains(&None) { + return Err(format!("unreadable GB {gb}")); + } + if lens.len() > 1 { + return Err(format!("two lengths on the Y chromosome (GB {gb})")); + } + let bp = lens[0].unwrap_or(0); + let n = |k: &str| get(k).and_then(crate::int).unwrap_or(0); + let (dp, st, fl) = (n("DP"), n("DSTUTTER"), n("DFLANKINDEL")); + let q = get("Q").and_then(crate::float); + let mut problems = Vec::new(); + if q.is_none_or(|q| q < MIN_Q) { + problems.push(format!( + "posterior {} below {}", + get("Q").unwrap_or("."), + pycompat::fmt::repr(MIN_Q) + )); + } + if dp != 0 && st as f64 / dp as f64 > MAX_STUTTER { + problems.push(format!("{st} of {dp} reads with stutter")); + } + if dp != 0 && fl as f64 / dp as f64 > MAX_FLANK { + problems.push(format!("{fl} of {dp} reads with a flank indel")); + } + if dp < MIN_READS { + problems.push(format!("{dp} reads, fewer than {MIN_READS}")); + } + Ok(PartCall { + bp, + dp, + st, + fl, + q, + problems, + }) +} + +/// Python's `divmod` on ints: the quotient rounded down. +fn divmod(a: i128, b: i128) -> (i128, i128) { + let (mut q, mut r) = (a / b, a % b); + if r != 0 && ((r < 0) != (b < 0)) { + q -= 1; + r += b; + } + (q, r) +} + +/// The count: reference copies plus the bp difference in repeats, the rest +/// after a dot. +fn count_text(ref_copies: i128, bp: i128, period: i128, offset: Option) -> String { + let (whole, rest) = divmod(bp, period); + let n = ref_copies + whole + offset.unwrap_or(0); + if rest != 0 { + format!("{n}.{rest}") + } else { + n.to_string() + } +} + +/// `min` as Python's builtin: the first item no later one is smaller than. +fn py_min(xs: &[f64]) -> f64 { + xs[1..] + .iter() + .fold(xs[0], |m, &x| if x < m { x } else { m }) +} + +/// One marker's row, by [`TSV_COLUMNS`]. +pub fn marker_row( + m: &Marker, + calls: &HashMap>, + read_length: i64, + log: Option<&HashMap<(String, i128, i128), String>>, +) -> Vec { + let mut row: Vec = vec![ + m.order.clone(), + m.panel.clone(), + m.marker.clone(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + ]; + let notes: Vec = if m.note == "." { + vec![] + } else { + vec![m.note.clone()] + }; + let finish = |mut row: Vec, conf: &str, note: String| { + row[4] = conf.into(); + row[9] = note; + row + }; + match m.kind { + Kind::Multicopy => { + return finish( + row, + "not callable", + "not callable from short reads: a multi-copy marker in duplicated Y sequence" + .into(), + ); + } + Kind::Absent => { + return finish( + row, + "not callable", + "not callable: no region for it in the HipSTR marker set".into(), + ); + } + _ => {} + } + let longest = m + .regions + .iter() + .map(|r| r.end - r.start + 1) + .max() + .unwrap_or(0); + if longest + 2 * FLANK > i128::from(read_length) { + return finish( + row, + "not callable", + format!( + "not callable from short reads: a {longest} bp repeat is longer than {read_length} bp reads can span with {FLANK} bases on each side" + ), + ); + } + let period = m.regions[0].period; + let (mut bp_sum, mut ref_sum) = (0i128, 0i128); + let (mut dps, mut sts, mut fls, mut qs) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let mut problems = Vec::new(); + let many = m.regions.len() > 1; + for r in &m.regions { + let Some(fmt) = calls.get(&r.name) else { + let why = match log { + None => format!("HipSTR wrote no record for {}; see HipSTR's log", r.name), + Some(l) => match l.get(&(r.chrom.clone(), r.start, r.end)) { + Some(reason) => { + format!("HipSTR wrote no record for {}; its log: {reason}", r.name) + } + None => format!( + "HipSTR wrote no record for {}, and its log gives no reason", + r.name + ), + }, + }; + let mut n = notes.clone(); + n.push(why); + return finish(row, "no call", n.join("; ")); + }; + let got = match part_call(fmt) { + Ok(g) => g, + Err(why) => { + let mut n = notes.clone(); + n.push(format!("{}: {why}", r.name)); + return finish(row, "no call", n.join("; ")); + } + }; + bp_sum += got.bp; + ref_sum += halves_up(r.copies); + dps.push(got.dp); + sts.push(got.st); + fls.push(got.fl); + qs.push(got.q); + problems.extend( + got.problems + .into_iter() + .map(|x| if many { format!("{}: {x}", r.name) } else { x }), + ); + } + row[3] = count_text(ref_sum, bp_sum, period, m.offset); + row[4] = if problems.is_empty() { "high" } else { "low" }.into(); + row[5] = dps.iter().min().map_or(String::new(), ToString::to_string); + row[6] = sts.iter().max().map_or(String::new(), ToString::to_string); + row[7] = fls.iter().max().map_or(String::new(), ToString::to_string); + row[8] = match qs.iter().copied().collect::>>() { + Some(q) => pycompat::fmt::general(py_min(&q), 6), + None => ".".into(), + }; + let mut n = notes; + n.extend(problems); + row[9] = if n.is_empty() { + ".".into() + } else { + n.join("; ") + }; + row +} + +/// The files of `og y str-report`. +pub struct Files<'a> { + pub panel: &'a str, + pub vcf: &'a str, + pub tsv: &'a str, + pub csv: &'a str, + pub log: Option<&'a str>, +} + +/// `og y str-report`: writes both tables; the stdout line. +pub fn report(f: &Files<'_>, read_length: i64) -> Result { + let panel = read_panel(f.panel)?; + let calls = read_calls(f.vcf)?; + let log = match f.log { + Some(l) => Some(read_log(l)?), + None => None, + }; + let rows: Vec> = panel + .iter() + .map(|m| marker_row(m, &calls, read_length, log.as_ref())) + .collect(); + let mut tsv = TSV_COLUMNS.join("\t") + "\n"; + for r in &rows { + tsv.push_str( + &r.iter() + .map(|c| c.replace('\t', " ")) + .collect::>() + .join("\t"), + ); + tsv.push('\n'); + } + let mut w = pycompat::csv::Writer::new(Vec::new(), ','); + let io = |e: std::io::Error| Error::io(&e, f.csv); + w.write_row(["marker", "value", "confidence"]).map_err(io)?; + for r in &rows { + w.write_row([r[2].as_str(), &r[3], &r[4]]).map_err(io)?; + } + crate::write_atomic(f.tsv, tsv.as_bytes())?; + crate::write_atomic(f.csv, &w.into_inner())?; + let tally = |k: &str| rows.iter().filter(|r| r[4] == k).count(); + Ok(format!( + "Y-STR markers: {} high, {} low, {} no call, {} not callable\n", + tally("high"), + tally("low"), + tally("no call"), + tally("not callable") + )) +} + +const BED: Spec = Spec { + prog: "og y str-bed", + usage: "usage: og y str-bed [-h] --panel PANEL --out OUT\n", + options: &["--panel", "--out"], + positionals: &[], + required: &["--panel", "--out"], +}; + +const REPORT: Spec = Spec { + prog: "og y str-report", + usage: "usage: og y str-report [-h] --panel PANEL --vcf VCF --tsv TSV --csv CSV [--log LOG] [--read-length READ_LENGTH]\n", + options: &[ + "--panel", + "--vcf", + "--tsv", + "--csv", + "--log", + "--read-length", + ], + positionals: &[], + required: &["--panel", "--vcf", "--tsv", "--csv"], +}; + +/// `og y VERB ARGS`: the exit code (1 for a problem, 2 for a refused +/// command line). +pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + match verb { + Some("str-bed") => crate::run(&BED, rest, out, err, |a| { + Ok((|| { + let panel = read_panel(crate::opt(a, "--panel"))?; + crate::write_atomic(crate::opt(a, "--out"), bed(&panel).as_bytes())?; + Ok(String::new()) + })() + .map_err(|e: Error| (e, 1))) + }), + Some("str-report") => crate::run(&REPORT, rest, out, err, |a| { + let read_length = crate::int_opt(a, "--read-length", 150)?; + let files = Files { + panel: crate::opt(a, "--panel"), + vcf: crate::opt(a, "--vcf"), + tsv: crate::opt(a, "--tsv"), + csv: crate::opt(a, "--csv"), + log: a.get("--log"), + }; + Ok(report(&files, read_length).map_err(|e| (e, 1))) + }), + other => crate::unknown_verb("y", "str-bed|str-report", other, err), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn floor_division_of_a_negative_difference() { + assert_eq!(count_text(11, -2, 4, None), "10.2"); + assert_eq!(count_text(11, 0, 4, None), "11"); + assert_eq!(count_text(11, 6, 4, Some(-1)), "11.2"); + assert_eq!(divmod(-7, 4), (-2, 1)); + assert_eq!(divmod(7, -2), (-4, -1)); + } + + /// Red first: GB -4 on a 4 bp motif is one repeat fewer, not `11.-4`. + #[test] + fn red_first_truncating_division_would_be_wrong() { + assert_eq!(count_text(11, -4, 4, None), "10"); + assert_eq!(count_text(11, -1, 4, None), "10.3"); + } + + #[test] + fn part_calls() { + let f = |s: &str| -> HashMap { + s.split(';') + .map(|kv| { + let (k, v) = kv.split_once('=').unwrap(); + (k.into(), v.into()) + }) + .collect() + }; + let p = part_call(&f("GT=1;GB=-4;Q=0.95;DP=30;DSTUTTER=6;DFLANKINDEL=0")).unwrap(); + assert_eq!(p.problems, ["6 of 30 reads with stutter"]); + let p = part_call(&f("GT=1;GB=0;Q=0.5;DP=5;DSTUTTER=0;DFLANKINDEL=0")).unwrap(); + assert_eq!( + p.problems, + ["posterior 0.5 below 0.9", "5 reads, fewer than 10"] + ); + assert_eq!( + part_call(&f("GT=.;GB=.;FILTER=LowQ")).err().unwrap(), + "HipSTR made no genotype (LowQ)" + ); + assert_eq!( + part_call(&f("GT=0/1;GB=0/4")).err().unwrap(), + "two lengths on the Y chromosome (GB 0/4)" + ); + assert_eq!(part_call(&f("GT=0;GB=x")).err().unwrap(), "unreadable GB x"); + assert!(py_min(&[f64::NAN, 0.5]).is_nan()); + assert_eq!(py_min(&[0.5, f64::NAN]), 0.5); + } +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml index f05413c7e..3025aa0db 100644 --- a/rust/og/Cargo.toml +++ b/rust/og/Cargo.toml @@ -17,6 +17,7 @@ og-vcf.workspace = true og-steps.workspace = true og-filters.workspace = true og-chip.workspace = true +og-reports.workspace = true og-runlog.workspace = true [features] diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index acc0e4476..b100626ac 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -18,7 +18,7 @@ fn tree_hash() -> &'static str { &TREE_MARK["og-tree-hash:".len()..TREE_MARK.len() - 1] } -const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og dev parity-compare KIND ORACLE NEW\n og version\n og --tree-hash\n"; +const USAGE: &str = "usage: og [args]\n og map [args]\n og run state|tasks|eta|headroom|delta [args]\n og setup mask-fasta SRC BED DEST\n og vcf csq-rank [options]\n og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE\n og sv svlen-fix|inv-filter [args]\n og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args]\n og str motif [args]\n og chip prepare|report|check-table [args]\n og vntyper report [args]\n og locityper report [args]\n og mocha report [args]\n og mei report [args]\n og mei annotsv-input [args]\n og viral report [args]\n og y str-bed [args]\n og y str-report [args]\n og exomiser prepare [args]\n og exomiser report [args]\n og dev parity-compare KIND ORACLE NEW\n og version\n og --tree-hash\n"; /// argparse's way to refuse a command line: usage and the error on stderr, exit 2. fn usage_error(msg: &str) -> ExitCode { @@ -119,6 +119,20 @@ fn main() -> ExitCode { )), None => usage_error("the following arguments are required: chip tool"), }, + "vntyper" | "locityper" | "mocha" | "mei" | "viral" | "y" | "exomiser" => { + let verb = rest.first().map(String::as_str); + let args = rest.get(1..).unwrap_or_default(); + let (out, err) = (&mut std::io::stdout().lock(), &mut std::io::stderr().lock()); + ExitCode::from(match tool.as_str() { + "vntyper" => og_reports::vntyper::cli(verb, args, out, err), + "locityper" => og_reports::locityper::cli(verb, args, out, err), + "mocha" => og_reports::mocha::cli(verb, args, out, err), + "mei" => og_reports::mei::cli(verb, args, out, err), + "viral" => og_reports::viral::cli(verb, args, out, err), + "y" => og_reports::ystr::cli(verb, args, out, err), + _ => og_reports::exomiser::cli(verb, args, out, err), + }) + } "str" => match rest.first().map(String::as_str) { #[cfg(feature = "bam")] Some("motif") => ExitCode::from(og_bam::motif::cli( From 81be2658b9c005a97e32c9a6307861f7b44a0709 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:16 +0100 Subject: [PATCH 22/53] test(parity): the seven opt-in step reports against the Python they replace 123 cases from the old unit tests and their traps, with the oracle at 9885cc5 in PYTHON_IMAGE. Goldens name each deliberate fix. Generators write the VNtyper and Locityper run folders and the viral inputs; the harness now copies a generator's folders whole. Bead: pgp-bs9u.17 --- scripts/ci/oracle-diff.sh | 2 +- tests/parity/cases.tsv | 123 +++++++++++++++ tests/parity/gen/locityper-runs.sh | 54 +++++++ tests/parity/gen/viral-inputs.sh | 52 +++++++ tests/parity/gen/vntyper-runs.sh | 38 +++++ .../exomiser/prepare-bad-assembly/stderr | 2 + .../golden/exomiser/report-top-default/stdout | 5 + .../golden/locityper-report/cut/out.tsv | 2 + .../golden/locityper-report/int-genotype/exit | 1 + .../locityper-report/int-genotype/out.tsv | 2 + .../locityper-report/int-genotype/stderr | 0 .../locityper-report/int-genotype/stdout | 1 + .../golden/locityper-report/list-json/exit | 1 + .../golden/locityper-report/list-json/out.tsv | 2 + .../golden/locityper-report/list-json/stderr | 0 .../golden/locityper-report/list-json/stdout | 1 + .../golden/locityper-report/notjson/out.tsv | 2 + .../locityper-report/unicode-bed/stderr | 1 + .../mei-annotsv-input/no-chrom-line/stderr | 1 + .../golden/mei-report/annotsv-bad/stderr | 1 + .../golden/mei-report/bad-catalog/stderr | 1 + .../parity/golden/mei-report/bad-depth/stderr | 1 + .../golden/mei-report/no-chrom-line/stderr | 1 + .../golden/mei-report/not-megane/stderr | 1 + .../golden/mei-report/two-windows/s.tsv | 4 + .../golden/mocha-report/bad-phasing/stderr | 2 + .../golden/mocha-report/short-call/exit | 1 + .../golden/mocha-report/short-call/stderr | 1 + .../golden/mocha-report/two-samples/s.tsv | 17 +++ .../golden/mocha-report/two-samples/stdout | 1 + .../golden/viral-report/bad-coverage/stderr | 1 + .../golden/viral-report/cut-depth/stderr | 1 + .../golden/vntyper-report/negative-mean/exit | 1 + .../vntyper-report/negative-mean/stderr | 1 + .../golden/vntyper-report/unreadable/stderr | 1 + .../golden/y-str-report/panel-copies/exit | 1 + .../y-str-report/panel-copies/o.csv.absent | 0 .../y-str-report/panel-copies/o.tsv.absent | 0 .../golden/y-str-report/panel-copies/stderr | 1 + .../golden/y-str-report/panel-copies/stdout | 0 .../golden/y-str-report/panel-period/stderr | 1 + tests/parity/inputs/exomiser/S1_exomiser.ped | 3 + tests/parity/inputs/exomiser/empty.tsv | 0 tests/parity/inputs/exomiser/fam.vcf.gz | Bin 0 -> 126 bytes tests/parity/inputs/exomiser/family.tsv | 2 + tests/parity/inputs/exomiser/header-only.tsv | 1 + tests/parity/inputs/exomiser/hp.obo | 24 +++ tests/parity/inputs/exomiser/no-acmg.tsv | 7 + tests/parity/inputs/exomiser/one.vcf.gz | Bin 0 -> 119 bytes tests/parity/inputs/exomiser/pedigree.tsv | 5 + tests/parity/inputs/exomiser/short.tsv | 2 + tests/parity/inputs/exomiser/stranger.vcf.gz | Bin 0 -> 126 bytes tests/parity/inputs/exomiser/terms.tsv | 2 + tests/parity/inputs/exomiser/variants.tsv | 7 + tests/parity/inputs/mei/annotsv-bad.tsv | 2 + tests/parity/inputs/mei/annotsv-empty.tsv | 0 tests/parity/inputs/mei/annotsv.tsv | 5 + tests/parity/inputs/mei/catalog.tsv | 9 ++ tests/parity/inputs/mei/classes.vcf | 20 +++ tests/parity/inputs/mei/depth-bad.tsv | 1 + tests/parity/inputs/mei/depth-catalog.tsv | 2 + tests/parity/inputs/mei/depth-empty.tsv | 0 tests/parity/inputs/mei/depth-thin.tsv | 1 + tests/parity/inputs/mei/depth-two-windows.tsv | 2 + tests/parity/inputs/mei/depth.tsv | 2 + tests/parity/inputs/mei/empty.vcf | 8 + tests/parity/inputs/mei/fktn.vcf | 10 ++ tests/parity/inputs/mei/nochrom.vcf | 1 + tests/parity/inputs/mei/none.vcf | 9 ++ tests/parity/inputs/mei/not-megane.vcf | 9 ++ tests/parity/inputs/mei/outside.vcf | 9 ++ tests/parity/inputs/mei/sites-bad.tsv | 2 + tests/parity/inputs/mei/sites-two-windows.tsv | 5 + tests/parity/inputs/mei/sites.tsv | 4 + tests/parity/inputs/mocha/calls-crlf.tsv | 2 + tests/parity/inputs/mocha/calls-empty.tsv | 1 + tests/parity/inputs/mocha/calls-filter.tsv | 9 ++ tests/parity/inputs/mocha/calls-low.tsv | 2 + tests/parity/inputs/mocha/calls-one.tsv | 2 + tests/parity/inputs/mocha/calls-short.tsv | 2 + tests/parity/inputs/mocha/calls-two.tsv | 4 + tests/parity/inputs/mocha/sites-300.tsv | 1 + tests/parity/inputs/mocha/sites-many.tsv | 23 +++ tests/parity/inputs/mocha/sites-short.tsv | 1 + tests/parity/inputs/mocha/sites-unphased.tsv | 1 + tests/parity/inputs/mocha/sites.tsv | 1 + tests/parity/inputs/mocha/stats-5x.tsv | 2 + tests/parity/inputs/mocha/stats-autocorr.tsv | 2 + tests/parity/inputs/mocha/stats-low-rate.tsv | 2 + tests/parity/inputs/mocha/stats-nan.tsv | 2 + tests/parity/inputs/mocha/stats-nocov.tsv | 2 + tests/parity/inputs/mocha/stats-other.tsv | 2 + tests/parity/inputs/mocha/stats-two.tsv | 3 + tests/parity/inputs/mocha/stats.tsv | 2 + tests/parity/inputs/y-str/calls.vcf | 24 +++ tests/parity/inputs/y-str/calls.vcf.gz | Bin 0 -> 455 bytes tests/parity/inputs/y-str/nohead.vcf | 1 + tests/parity/inputs/y-str/panel-copies.tsv | 142 ++++++++++++++++++ tests/parity/inputs/y-str/panel-kind.tsv | 142 ++++++++++++++++++ tests/parity/inputs/y-str/panel-order.tsv | 142 ++++++++++++++++++ tests/parity/inputs/y-str/panel-period.tsv | 142 ++++++++++++++++++ tests/parity/inputs/y-str/panel.tsv | 142 ++++++++++++++++++ tests/parity/inputs/y-str/two.vcf.gz | Bin 0 -> 201 bytes tests/parity/oracle.ref | 9 ++ 104 files changed, 1292 insertions(+), 1 deletion(-) create mode 100644 tests/parity/gen/locityper-runs.sh create mode 100644 tests/parity/gen/viral-inputs.sh create mode 100644 tests/parity/gen/vntyper-runs.sh create mode 100644 tests/parity/golden/exomiser/prepare-bad-assembly/stderr create mode 100644 tests/parity/golden/exomiser/report-top-default/stdout create mode 100644 tests/parity/golden/locityper-report/cut/out.tsv create mode 100644 tests/parity/golden/locityper-report/int-genotype/exit create mode 100644 tests/parity/golden/locityper-report/int-genotype/out.tsv create mode 100644 tests/parity/golden/locityper-report/int-genotype/stderr create mode 100644 tests/parity/golden/locityper-report/int-genotype/stdout create mode 100644 tests/parity/golden/locityper-report/list-json/exit create mode 100644 tests/parity/golden/locityper-report/list-json/out.tsv create mode 100644 tests/parity/golden/locityper-report/list-json/stderr create mode 100644 tests/parity/golden/locityper-report/list-json/stdout create mode 100644 tests/parity/golden/locityper-report/notjson/out.tsv create mode 100644 tests/parity/golden/locityper-report/unicode-bed/stderr create mode 100644 tests/parity/golden/mei-annotsv-input/no-chrom-line/stderr create mode 100644 tests/parity/golden/mei-report/annotsv-bad/stderr create mode 100644 tests/parity/golden/mei-report/bad-catalog/stderr create mode 100644 tests/parity/golden/mei-report/bad-depth/stderr create mode 100644 tests/parity/golden/mei-report/no-chrom-line/stderr create mode 100644 tests/parity/golden/mei-report/not-megane/stderr create mode 100644 tests/parity/golden/mei-report/two-windows/s.tsv create mode 100644 tests/parity/golden/mocha-report/bad-phasing/stderr create mode 100644 tests/parity/golden/mocha-report/short-call/exit create mode 100644 tests/parity/golden/mocha-report/short-call/stderr create mode 100644 tests/parity/golden/mocha-report/two-samples/s.tsv create mode 100644 tests/parity/golden/mocha-report/two-samples/stdout create mode 100644 tests/parity/golden/viral-report/bad-coverage/stderr create mode 100644 tests/parity/golden/viral-report/cut-depth/stderr create mode 100644 tests/parity/golden/vntyper-report/negative-mean/exit create mode 100644 tests/parity/golden/vntyper-report/negative-mean/stderr create mode 100644 tests/parity/golden/vntyper-report/unreadable/stderr create mode 100644 tests/parity/golden/y-str-report/panel-copies/exit create mode 100644 tests/parity/golden/y-str-report/panel-copies/o.csv.absent create mode 100644 tests/parity/golden/y-str-report/panel-copies/o.tsv.absent create mode 100644 tests/parity/golden/y-str-report/panel-copies/stderr create mode 100644 tests/parity/golden/y-str-report/panel-copies/stdout create mode 100644 tests/parity/golden/y-str-report/panel-period/stderr create mode 100644 tests/parity/inputs/exomiser/S1_exomiser.ped create mode 100644 tests/parity/inputs/exomiser/empty.tsv create mode 100644 tests/parity/inputs/exomiser/fam.vcf.gz create mode 100644 tests/parity/inputs/exomiser/family.tsv create mode 100644 tests/parity/inputs/exomiser/header-only.tsv create mode 100644 tests/parity/inputs/exomiser/hp.obo create mode 100644 tests/parity/inputs/exomiser/no-acmg.tsv create mode 100644 tests/parity/inputs/exomiser/one.vcf.gz create mode 100644 tests/parity/inputs/exomiser/pedigree.tsv create mode 100644 tests/parity/inputs/exomiser/short.tsv create mode 100644 tests/parity/inputs/exomiser/stranger.vcf.gz create mode 100644 tests/parity/inputs/exomiser/terms.tsv create mode 100644 tests/parity/inputs/exomiser/variants.tsv create mode 100644 tests/parity/inputs/mei/annotsv-bad.tsv create mode 100644 tests/parity/inputs/mei/annotsv-empty.tsv create mode 100644 tests/parity/inputs/mei/annotsv.tsv create mode 100644 tests/parity/inputs/mei/catalog.tsv create mode 100644 tests/parity/inputs/mei/classes.vcf create mode 100644 tests/parity/inputs/mei/depth-bad.tsv create mode 100644 tests/parity/inputs/mei/depth-catalog.tsv create mode 100644 tests/parity/inputs/mei/depth-empty.tsv create mode 100644 tests/parity/inputs/mei/depth-thin.tsv create mode 100644 tests/parity/inputs/mei/depth-two-windows.tsv create mode 100644 tests/parity/inputs/mei/depth.tsv create mode 100644 tests/parity/inputs/mei/empty.vcf create mode 100644 tests/parity/inputs/mei/fktn.vcf create mode 100644 tests/parity/inputs/mei/nochrom.vcf create mode 100644 tests/parity/inputs/mei/none.vcf create mode 100644 tests/parity/inputs/mei/not-megane.vcf create mode 100644 tests/parity/inputs/mei/outside.vcf create mode 100644 tests/parity/inputs/mei/sites-bad.tsv create mode 100644 tests/parity/inputs/mei/sites-two-windows.tsv create mode 100644 tests/parity/inputs/mei/sites.tsv create mode 100644 tests/parity/inputs/mocha/calls-crlf.tsv create mode 100644 tests/parity/inputs/mocha/calls-empty.tsv create mode 100644 tests/parity/inputs/mocha/calls-filter.tsv create mode 100644 tests/parity/inputs/mocha/calls-low.tsv create mode 100644 tests/parity/inputs/mocha/calls-one.tsv create mode 100644 tests/parity/inputs/mocha/calls-short.tsv create mode 100644 tests/parity/inputs/mocha/calls-two.tsv create mode 100644 tests/parity/inputs/mocha/sites-300.tsv create mode 100644 tests/parity/inputs/mocha/sites-many.tsv create mode 100644 tests/parity/inputs/mocha/sites-short.tsv create mode 100644 tests/parity/inputs/mocha/sites-unphased.tsv create mode 100644 tests/parity/inputs/mocha/sites.tsv create mode 100644 tests/parity/inputs/mocha/stats-5x.tsv create mode 100644 tests/parity/inputs/mocha/stats-autocorr.tsv create mode 100644 tests/parity/inputs/mocha/stats-low-rate.tsv create mode 100644 tests/parity/inputs/mocha/stats-nan.tsv create mode 100644 tests/parity/inputs/mocha/stats-nocov.tsv create mode 100644 tests/parity/inputs/mocha/stats-other.tsv create mode 100644 tests/parity/inputs/mocha/stats-two.tsv create mode 100644 tests/parity/inputs/mocha/stats.tsv create mode 100644 tests/parity/inputs/y-str/calls.vcf create mode 100644 tests/parity/inputs/y-str/calls.vcf.gz create mode 100644 tests/parity/inputs/y-str/nohead.vcf create mode 100644 tests/parity/inputs/y-str/panel-copies.tsv create mode 100644 tests/parity/inputs/y-str/panel-kind.tsv create mode 100644 tests/parity/inputs/y-str/panel-order.tsv create mode 100644 tests/parity/inputs/y-str/panel-period.tsv create mode 100644 tests/parity/inputs/y-str/panel.tsv create mode 100644 tests/parity/inputs/y-str/two.vcf.gz diff --git a/scripts/ci/oracle-diff.sh b/scripts/ci/oracle-diff.sh index e64437059..260df68d0 100755 --- a/scripts/ci/oracle-diff.sh +++ b/scripts/ci/oracle-diff.sh @@ -110,7 +110,7 @@ stage() { mkdir -p "$cache" (cd "$cache" && bash "${PARITY}/gen/${item#gen:}") || die "generator ${item#gen:} failed" fi - cp -p "$cache"/* "$dir"/ ;; + cp -pR "$cache"/. "$dir"/ ;; *) [ -f "${PARITY}/inputs/${item}" ] || die "input ${item} is not in tests/parity/inputs" cp -p "${PARITY}/inputs/${item}" "$dir"/ ;; diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index d71c7f6ae..48feac7c2 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -118,3 +118,126 @@ chip check-bad-table chip/bad-table.tsv check-table bad-table.tsv - bytes PYTHON chip check-bad-header chip/bad-header.tsv check-table bad-header.tsv - bytes PYTHON_IMAGE chip check-short-row chip/short-row.tsv check-table short-row.tsv - bytes PYTHON_IMAGE chip check-no-genes chip/no-genes.tsv check-table no-genes.tsv - bytes PYTHON_IMAGE +vntyper-report pos gen:vntyper-runs.sh --run-dir pos --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report low gen:vntyper-runs.sh --run-dir low --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report flag gen:vntyper-runs.sh --run-dir flag --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report holes gen:vntyper-runs.sh --run-dir holes --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report nounc gen:vntyper-runs.sh --run-dir nounc --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report neg gen:vntyper-runs.sh --run-dir neg --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report red gen:vntyper-runs.sh --run-dir red --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report thin gen:vntyper-runs.sh --run-dir thin --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report na gen:vntyper-runs.sh --run-dir na --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report hdr gen:vntyper-runs.sh --run-dir hdr --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report nok gen:vntyper-runs.sh --run-dir nok --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report crlf gen:vntyper-runs.sh --run-dir crlf --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report exit-code gen:vntyper-runs.sh --run-dir pos --exit-code 1 --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report no-read-log gen:vntyper-runs.sh --run-dir empty --exit-code 1 --log empty.log --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report missing-log gen:vntyper-runs.sh --run-dir empty --exit-code 1 --log none.log --out out.tsv out.tsv bytes PYTHON_IMAGE +vntyper-report negative-mean gen:vntyper-runs.sh --run-dir negmean --out out.tsv - bytes PYTHON_IMAGE +vntyper-report unreadable gen:vntyper-runs.sh --run-dir bad --out out.tsv - bytes PYTHON_IMAGE +locityper-report clean gen:locityper-runs.sh --gt-dir clean --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report far gen:locityper-runs.sh --gt-dir far --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report unexplained gen:locityper-runs.sh --gt-dir unexplained --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report share gen:locityper-runs.sh --gt-dir share --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report warned gen:locityper-runs.sh --gt-dir warned --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nodist gen:locityper-runs.sh --gt-dir nodist --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nocounts gen:locityper-runs.sh --gt-dir nocounts --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nulltotal gen:locityper-runs.sh --gt-dir nulltotal --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nores gen:locityper-runs.sh --gt-dir nores --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report haploid gen:locityper-runs.sh --gt-dir haploid --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report notjson gen:locityper-runs.sh --gt-dir notjson --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report cut gen:locityper-runs.sh --gt-dir cut --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report plain gen:locityper-runs.sh --gt-dir plain --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report boolcounts gen:locityper-runs.sh --gt-dir boolcounts --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report strquality gen:locityper-runs.sh --gt-dir strquality --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nanquality gen:locityper-runs.sh --gt-dir nanquality --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report nosuccess gen:locityper-runs.sh --gt-dir nosuccess --loci loci.bed --exit-code 101 --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report three-loci gen:locityper-runs.sh --gt-dir clean --loci three.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report list-json gen:locityper-runs.sh --gt-dir list --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report int-genotype gen:locityper-runs.sh --gt-dir intgenotype --loci loci.bed --out out.tsv out.tsv bytes PYTHON_IMAGE +locityper-report bad-bed gen:locityper-runs.sh --gt-dir clean --loci bad.bed --out out.tsv - bytes PYTHON_IMAGE +locityper-report short-bed gen:locityper-runs.sh --gt-dir clean --loci short.bed --out out.tsv - bytes PYTHON_IMAGE +locityper-report unicode-bed gen:locityper-runs.sh --gt-dir clean --loci unicode.bed --out out.tsv - bytes PYTHON_IMAGE +locityper-report missing-bed gen:locityper-runs.sh --gt-dir clean --loci none.bed --out out.tsv - bytes PYTHON_IMAGE +mocha-report one mocha/stats.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report under-floor mocha/stats.tsv,mocha/calls-low.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-low.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report 5x mocha/stats-5x.tsv,mocha/calls-one.tsv,mocha/sites-300.tsv --sample S --stats stats-5x.tsv --calls calls-one.tsv --sites sites-300.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report filters mocha/stats.tsv,mocha/calls-filter.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-filter.tsv --sites sites.tsv --phasing beagle --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report low-call-rate mocha/stats-low-rate.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats-low-rate.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report nan-call-rate mocha/stats-nan.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats-nan.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report autocorrelation mocha/stats-autocorr.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats-autocorr.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report unphased mocha/stats.tsv,mocha/calls-one.tsv,mocha/sites-unphased.tsv --sample S --stats stats.tsv --calls calls-one.tsv --sites sites-unphased.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report no-calls mocha/stats.tsv,mocha/calls-empty.tsv,mocha/sites-many.tsv --sample S --stats stats.tsv --calls calls-empty.tsv --sites sites-many.tsv --phasing beagle --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report two-samples mocha/stats-two.tsv,mocha/calls-two.tsv,mocha/sites.tsv --sample S --stats stats-two.tsv --calls calls-two.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report crlf mocha/stats.tsv,mocha/calls-crlf.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-crlf.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv o.tsv,s.tsv bytes PYTHON_IMAGE +mocha-report no-cov-median mocha/stats-nocov.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats-nocov.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mocha-report sample-not-in-stats mocha/stats-other.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats-other.tsv --calls calls-one.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mocha-report short-sites mocha/stats.tsv,mocha/calls-one.tsv,mocha/sites-short.tsv --sample S --stats stats.tsv --calls calls-one.tsv --sites sites-short.tsv --phasing input --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mocha-report short-call mocha/stats.tsv,mocha/calls-short.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-short.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mocha-report missing-calls mocha/stats.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls none.tsv --sites sites.tsv --phasing input --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mocha-report bad-phasing mocha/stats.tsv,mocha/calls-one.tsv,mocha/sites.tsv --sample S --stats stats.tsv --calls calls-one.tsv --sites sites.tsv --phasing shapeit --out-calls o.tsv --out-summary s.tsv - bytes PYTHON_IMAGE +mei-report fktn-seen mei/fktn.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report fktn-not-seen mei/none.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf none.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report outside-window mei/outside.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf outside.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report thin mei/fktn.vcf,mei/sites.tsv,mei/depth-thin.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth-thin.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report thin-no-call mei/none.vcf,mei/sites.tsv,mei/depth-thin.tsv --sample S --vcf none.vcf --sites sites.tsv --depth depth-thin.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report empty-depth mei/fktn.vcf,mei/sites.tsv,mei/depth-empty.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth-empty.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report classes mei/classes.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf classes.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report no-record mei/empty.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf empty.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report catalog mei/fktn.vcf,mei/catalog.tsv,mei/depth-catalog.tsv --sample S --vcf fktn.vcf --sites catalog.tsv --depth depth-catalog.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report two-windows mei/fktn.vcf,mei/sites-two-windows.tsv,mei/depth-two-windows.tsv --sample S --vcf fktn.vcf --sites sites-two-windows.tsv --depth depth-two-windows.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report annotsv mei/fktn.vcf,mei/sites.tsv,mei/depth.tsv,mei/annotsv.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv --annotsv annotsv.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report annotsv-empty mei/fktn.vcf,mei/sites.tsv,mei/depth.tsv,mei/annotsv-empty.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv --annotsv annotsv-empty.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report annotsv-bad mei/fktn.vcf,mei/sites.tsv,mei/depth.tsv,mei/annotsv-bad.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv --annotsv annotsv-bad.tsv - bytes PYTHON_IMAGE +mei-report no-chrom-line mei/nochrom.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf nochrom.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE +mei-report not-megane mei/not-megane.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf not-megane.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE +mei-report bad-depth mei/fktn.vcf,mei/sites.tsv,mei/depth-bad.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth-bad.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE +mei-report bad-catalog mei/fktn.vcf,mei/sites-bad.tsv,mei/depth.tsv --sample S --vcf fktn.vcf --sites sites-bad.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE +mei-annotsv-input calls mei/classes.vcf --sample S --vcf classes.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE +mei-annotsv-input annotsv-ids mei/fktn.vcf --sample S --vcf fktn.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE +mei-annotsv-input empty mei/empty.vcf --sample S --vcf empty.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE +mei-annotsv-input no-chrom-line mei/nochrom.vcf --sample S --vcf nochrom.vcf --out in.vcf - bytes PYTHON_IMAGE +viral-report ici gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage ici.vcov.tsv --viral-depth ici.vdepth.tsv --human-coverage ici.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report relative-30 gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage relative-30.vcov.tsv --viral-depth relative-30.vdepth.tsv --human-coverage relative-30.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report pileup gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage pileup.vcov.tsv --viral-depth pileup.vdepth.tsv --human-coverage pileup.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report low gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage low.vcov.tsv --viral-depth low.vdepth.tsv --human-coverage low.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report few gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage few.vcov.tsv --viral-depth few.vdepth.tsv --human-coverage few.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report phix-and-both-hhv6 gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage phix.vcov.tsv --viral-depth phix.vdepth.tsv --human-coverage phix.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report no-human-depth gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage no-human.vcov.tsv --viral-depth no-human.vdepth.tsv --human-coverage no-human.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report human-mix gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage human-mix.vcov.tsv --viral-depth human-mix.vdepth.tsv --human-coverage human-mix.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report genome-left-out gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage ici.vcov.tsv --viral-depth gone.vdepth.tsv --human-coverage ici.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report crlf-fasta gen:viral-inputs.sh --sample S --viral-fasta crlf.fa --viral-coverage ici.vcov.tsv --viral-depth ici.vdepth.tsv --human-coverage ici.hcov.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE +viral-report cut-depth gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage ici.vcov.tsv --viral-depth cut.vdepth.tsv --human-coverage ici.hcov.tsv --out out.tsv - bytes PYTHON_IMAGE +viral-report bad-coverage gen:viral-inputs.sh --sample S --viral-fasta v.fa --viral-coverage ici.vcov.tsv --viral-depth ici.vdepth.tsv --human-coverage bad.hcov.tsv --out out.tsv - bytes PYTHON_IMAGE +y-str-bed panel y-str/panel.tsv --panel panel.tsv --out regions.bed regions.bed bytes PYTHON_IMAGE +y-str-bed panel-order y-str/panel-order.tsv --panel panel-order.tsv --out regions.bed - bytes PYTHON_IMAGE +y-str-report calls y-str/panel.tsv,y-str/calls.vcf.gz --panel panel.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv o.tsv,o.csv bytes PYTHON_IMAGE +y-str-report plain-vcf y-str/panel.tsv,y-str/calls.vcf --panel panel.tsv --vcf calls.vcf --tsv o.tsv --csv o.csv o.tsv,o.csv bytes PYTHON_IMAGE +y-str-report read-length-250 y-str/panel.tsv,y-str/calls.vcf.gz --panel panel.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv --read-length 250 o.tsv,o.csv bytes PYTHON_IMAGE +y-str-report log y-str/panel.tsv,y-str/calls.vcf.gz,y-str/hipstr.log --panel panel.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv --log hipstr.log o.tsv,o.csv bytes PYTHON_IMAGE +y-str-report empty-log y-str/panel.tsv,y-str/calls.vcf.gz,y-str/empty.log --panel panel.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv --log empty.log o.tsv,o.csv bytes PYTHON_IMAGE +y-str-report two-samples y-str/panel.tsv,y-str/two.vcf.gz --panel panel.tsv --vcf two.vcf.gz --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +y-str-report no-header y-str/panel.tsv,y-str/nohead.vcf --panel panel.tsv --vcf nohead.vcf --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +y-str-report panel-order y-str/panel-order.tsv,y-str/calls.vcf.gz --panel panel-order.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +y-str-report panel-period y-str/panel-period.tsv,y-str/calls.vcf.gz --panel panel-period.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +y-str-report panel-copies y-str/panel-copies.tsv,y-str/calls.vcf.gz --panel panel-copies.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +y-str-report panel-kind y-str/panel-kind.tsv,y-str/calls.vcf.gz --panel panel-kind.tsv --vcf calls.vcf.gz --tsv o.tsv --csv o.csv - bytes PYTHON_IMAGE +exomiser prepare-terms exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156,HP:0001363,HP:0009803,HP:0011304 --obo hp.obo a.yml,terms.tsv bytes PYTHON_IMAGE +exomiser prepare-no-obo exomiser/one.vcf.gz prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 a.yml,terms.tsv bytes PYTHON_IMAGE +exomiser prepare-hg19 exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --obo hp.obo --assembly hg19 a.yml,terms.tsv bytes PYTHON_IMAGE +exomiser prepare-family exomiser/fam.vcf.gz,exomiser/hp.obo,exomiser/pedigree.tsv prepare --sample S1 --vcf fam.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --obo hp.obo --pedigree pedigree.tsv --ped fam.ped --unaffected DAD --affected SIB a.yml,terms.tsv,fam.ped bytes PYTHON_IMAGE +exomiser prepare-malformed exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:12345 --obo hp.obo - bytes PYTHON_IMAGE +exomiser prepare-unknown exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0009999 --obo hp.obo - bytes PYTHON_IMAGE +exomiser prepare-obsolete exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001250 --obo hp.obo - bytes PYTHON_IMAGE +exomiser prepare-two-samples exomiser/fam.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf fam.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --obo hp.obo - bytes PYTHON_IMAGE +exomiser prepare-affected-alone exomiser/one.vcf.gz,exomiser/hp.obo prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --obo hp.obo --affected S2 - bytes PYTHON_IMAGE +exomiser prepare-stranger exomiser/stranger.vcf.gz,exomiser/hp.obo,exomiser/pedigree.tsv prepare --sample S1 --vcf stranger.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --obo hp.obo --pedigree pedigree.tsv --ped fam.ped - bytes PYTHON_IMAGE +exomiser prepare-bad-assembly exomiser/one.vcf.gz prepare --sample S1 --vcf one.vcf.gz --assembly hg38 --out-dir results --analysis a.yml --terms terms.tsv --hpo HP:0001156 --assembly grch99 - bytes PYTHON_IMAGE +exomiser report-top-3 exomiser/variants.tsv,exomiser/terms.tsv report --sample S1 --variants variants.tsv --terms terms.tsv --out top.tsv --text words.txt --top 3 top.tsv,words.txt bytes PYTHON_IMAGE +exomiser report-top-default exomiser/variants.tsv,exomiser/terms.tsv report --sample S1 --variants variants.tsv --terms terms.tsv --out top.tsv --text words.txt top.tsv,words.txt bytes PYTHON_IMAGE +exomiser report-top-1 exomiser/variants.tsv,exomiser/terms.tsv report --sample S1 --variants variants.tsv --terms terms.tsv --out top.tsv --text words.txt --top 1 top.tsv,words.txt bytes PYTHON_IMAGE +exomiser report-header-only exomiser/header-only.tsv,exomiser/terms.tsv report --sample S1 --variants header-only.tsv --terms terms.tsv --out top.tsv --text words.txt top.tsv,words.txt bytes PYTHON_IMAGE +exomiser report-family exomiser/family.tsv,exomiser/terms.tsv,exomiser/S1_exomiser.ped report --sample S1 --variants family.tsv --terms terms.tsv --out top.tsv --text words.txt top.tsv,words.txt bytes PYTHON_IMAGE +exomiser report-missing-column exomiser/no-acmg.tsv,exomiser/terms.tsv report --sample S1 --variants no-acmg.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE +exomiser report-short-line exomiser/short.tsv,exomiser/terms.tsv report --sample S1 --variants short.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE +exomiser report-empty-file exomiser/empty.tsv,exomiser/terms.tsv report --sample S1 --variants empty.tsv --terms terms.tsv --out top.tsv --text words.txt - bytes PYTHON_IMAGE diff --git a/tests/parity/gen/locityper-runs.sh b/tests/parity/gen/locityper-runs.sh new file mode 100644 index 000000000..4801a29cd --- /dev/null +++ b/tests/parity/gen/locityper-runs.sh @@ -0,0 +1,54 @@ +# The Locityper output folders of the locityper-report cases: c/loci/CYP2D6 +# with res.json.gz and success as Locityper writes them, and the loci BED. +set -euo pipefail +python3 - <<'PY' +import gzip, json, os +def res(genotype='HG00733.1,NA19240.2', quality=27.8, total=900, unexpl=12, wd=0.2, warnings=None): + r = {'total_reads': total, 'quality': quality, 'unexpl_reads': unexpl, 'genotype': genotype, + 'options': 120, 'dist_type': 'edit'} + if wd is not None: + r['weight_dist'] = wd + if warnings is not None: + r['warnings'] = warnings + return r +def put(name, raw=None, success=True, cut=False, plain=False): + d = f'{name}/loci/CYP2D6' + os.makedirs(d) + if raw is not None: + body = raw if plain else gzip.compress(raw, mtime=0) + with open(f'{d}/res.json.gz', 'wb') as f: + f.write(body[:len(body) // 2] if cut else body) + if success: + open(f'{d}/success', 'w').close() +j = lambda r: json.dumps(r).encode() +put('clean', j(res())) +put('far', j(res(wd=31.5))) +put('unexplained', j(res(total=3000, unexpl=1200))) +put('share', j(res(total=10000, unexpl=1500))) +put('warned', j(res(warnings=['UnexplainedReads']))) +put('nodist', j(res(wd=None))) +put('nocounts', j({k: v for k, v in res().items() if k not in ('total_reads', 'unexpl_reads')})) +put('nulltotal', j(res(total=None))) +put('nosuccess', j(res()), success=False) +put('nores') +put('haploid', j(res(genotype='HG00733.1'))) +put('notjson', b'{not json') +put('cut', j(res()), cut=True) +put('plain', j(res()), plain=True) +put('list', b'[1, 2]') +put('intgenotype', j(res(genotype=5))) +put('boolcounts', j(res(total=True, unexpl=False))) +put('strquality', j(res(quality='high', wd='x'))) +put('nanquality', b'{"genotype": "A.1,B.2", "quality": NaN, "weight_dist": Infinity, "total_reads": 10, "unexpl_reads": 0}') +with open('loci.bed', 'w') as f: + f.write('# a comment line\nchr22\t42122284\t42152960\tCYP2D6\n') +with open('three.bed', 'w') as f: + f.write('# a comment line\n\nchr6\t32652603\t32669077\tHLA-DQB1\nchr22\t42122284\t42152960\tCYP2D6\tx\n' + 'chr1\t161506761\t161681047\tFCGR\n') +with open('bad.bed', 'w') as f: + f.write('chr22\tx\t42152960\tCYP2D6\n') +with open('short.bed', 'w') as f: + f.write('chr22\t42122284\t42152960\n') +with open('unicode.bed', 'w') as f: + f.write('chr22\t\u00b2\t42152960\tCYP2D6\n') +PY diff --git a/tests/parity/gen/viral-inputs.sh b/tests/parity/gen/viral-inputs.sh new file mode 100644 index 000000000..38ae9cd2f --- /dev/null +++ b/tests/parity/gen/viral-inputs.sh @@ -0,0 +1,52 @@ +# The inputs of the viral-report cases: a viral FASTA of five small genomes and, +# per case, samtools coverage of the viral and the human reads and samtools +# depth -aa of the viral ones, with chosen depths. +set -euo pipefail +python3 - <<'PY' +COV_HEAD = "#rname\tstartpos\tendpos\tnumreads\tcovbases\tcoverage\tmeandepth\tmeanbaseq\tmeanmapq\n" +GENOMES = [("NC_001664.4", "Human betaherpesvirus 6A, variant A DNA, complete virion genome, isolate U1102", 2000), + ("NC_000898.1", "Human herpesvirus 6B, complete genome", 2000), + ("NC_007605.1", "Human gammaherpesvirus 4, complete genome", 1500), + ("NC_004102.1", "Hepatitis C virus genotype 1, complete genome", 900), + ("NC_001422.1", "Escherichia phage phiX174, complete genome", 500)] +A, B, EBV, HCV, PHIX = (g[0] for g in GENOMES) + + +def put(name, text): + with open(name, "w") as f: + f.write(text) + + +def viral(tag, depths, reads, human=None): + cov, dep = [COV_HEAD], [] + for acc, _, length in GENOMES: + fn = depths.get(acc, lambda p: 0) + d = [fn(p) for p in range(1, length + 1)] + c = sum(1 for x in d if x) + cov.append(f"{acc}\t1\t{length}\t{reads.get(acc, 0)}\t{c}\t{100 * c / length:.4g}\t{sum(d) / length:.6g}\t30\t60\n") + dep += [f"{acc}\t{p}\t{x}\n" for p, x in enumerate(d, 1)] + if human is None: + human = [("chr1", 100000, 100000, 30.0), ("chr22", 50000, 50000, 30.0)] + h = [COV_HEAD] + [f"{r}\t1\t{ln}\t1000\t{cb}\t{100 * cb / ln:.4g}\t{m:.6g}\t30\t60\n" for r, ln, cb, m in human] + put(f"{tag}.vcov.tsv", "".join(cov)) + put(f"{tag}.vdepth.tsv", "".join(dep)) + put(f"{tag}.hcov.tsv", "".join(h)) + + +flat = lambda x: (lambda p: x) # noqa: E731 +put("v.fa", "".join(f">{acc} {title}\n" + "A" * length + "\n\n" for acc, title, length in GENOMES)) +put("crlf.fa", "".join(f">{acc} {title}\r\n" + "A" * length + "\r\n" for acc, title, length in GENOMES)) +viral("ici", {B: flat(15)}, {B: 400}) +viral("relative-30", {B: flat(9)}, {B: 400}) +viral("pileup", {B: lambda p: 100 if p < 50 else 0}, {B: 30}) +viral("low", {EBV: lambda p: 1 if p % 2 else 0, HCV: flat(2)}, {EBV: 40, HCV: 25}) +viral("few", {EBV: lambda p: 1 if p <= 100 else 0}, {EBV: 12}) +viral("phix", {PHIX: flat(31), B: flat(15), A: flat(17)}, {PHIX: 900, B: 400, A: 420}) +viral("no-human", {B: flat(15)}, {B: 400}, human=[("chrX", 1000, 1000, 30.0)]) +viral("human-mix", {}, {}, human=[("chr1", 1000, 500, 10.0), ("22", 1000, 1000, 40.0), ("chrX", 1000, 1000, 99.0), + ("chrUn_KI270302v1", 1000, 1000, 99.0), ("chrEBV", 1000, 1000, 99.0)]) +lines = open("ici.vdepth.tsv").readlines() +put("cut.vdepth.tsv", "".join(lines[:-10])) +put("gone.vdepth.tsv", "".join(x for x in lines if not x.startswith(HCV + "\t"))) +put("bad.hcov.tsv", "chr1\t1\t2\n") +PY diff --git a/tests/parity/gen/vntyper-runs.sh b/tests/parity/gen/vntyper-runs.sh new file mode 100644 index 000000000..06de2df37 --- /dev/null +++ b/tests/parity/gen/vntyper-runs.sh @@ -0,0 +1,38 @@ +# The VNtyper run folders of the vntyper-report cases, one folder per case +# (kestrel/kestrel_result.tsv after its ## banner, coverage/coverage_summary.tsv). +set -euo pipefail +python3 - <<'PY' +import os +runs = { + 'pos': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tHigh_Precision*\tNot flagged\t3\t59dupC\tCCCCACC\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'low': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tLow_Precision\tNot flagged\t3\t59dupC\tCCCCACC\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'flag': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tHigh_Precision*\tFalse_Positive_4bp_Insertion\t3\t59dupC\tCCCCACC\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'holes': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tHigh_Precision*\tNot flagged\t3\t59dupC\tCCCCACC\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t62.00\tFAIL\n'), + 'nounc': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tHigh_Precision*\tNot flagged\t3\t59dupC\tCCCCACC\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\tNA\tPASS\n'), + 'neg': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\nNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNegative\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n147.94\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'red': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\nNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNegative\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n70.00\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tREDUCED\n'), + 'thin': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\nNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNegative\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n20.00\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tFAIL\n'), + 'na': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\nNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNegative\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\nNA\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tNOT_EVALUATED\n'), + 'empty': (None, 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'hdr': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n147.94\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'negmean': ('##VNtyper Kestrel result\n##source=test\nMotif\tVariant\tPOS\tREF\tALT\tMotif_sequence\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\nNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNone\tNegative\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n-5.00\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), + 'crlf': ('##VNtyper Kestrel result\n##source=test\nMotifs\tVariant\tPOS\tREF\tALT\tEstimated_Depth_AlternateVariant\tEstimated_Depth_Variant_ActiveRegion\tDepth_Score\tConfidence\tFlag\thaplo_count\tNomenclature\tMotif_sequence\r\n2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\tHigh_Precision*\tNot flagged\t3\t59dupC\tCCCCACC\r\n', 'mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n'), +} +for name, (kestrel, cov) in runs.items(): + os.makedirs(f'{name}/kestrel') + os.makedirs(f'{name}/coverage') + if kestrel is not None: + with open(f'{name}/kestrel/kestrel_result.tsv', 'w', newline='') as f: + f.write(kestrel) + with open(f'{name}/coverage/coverage_summary.tsv', 'w') as f: + f.write(cov) +os.makedirs('nok') +os.makedirs('bad/kestrel') +os.makedirs('bad/coverage') +with open('bad/kestrel/kestrel_result.tsv', 'wb') as f: + f.write(b'\xff\xfe\x00bad') +with open('bad/coverage/coverage_summary.tsv', 'w') as f: + f.write('mean\tmedian\tstdev\tmin\tmax\tregion_length\tuncovered_bases\tpercent_uncovered\tcoverage_qc\n1075.32\t1000.00\t300.00\t95\t1942\t4500\t0\t0.00\tPASS\n') +with open('empty.log', 'w') as f: + f.write("ValueError: FASTQ layout 'empty' cannot be consumed without dropping reads.\n") +PY diff --git a/tests/parity/golden/exomiser/prepare-bad-assembly/stderr b/tests/parity/golden/exomiser/prepare-bad-assembly/stderr new file mode 100644 index 000000000..1d9b280dc --- /dev/null +++ b/tests/parity/golden/exomiser/prepare-bad-assembly/stderr @@ -0,0 +1,2 @@ +usage: og exomiser prepare [-h] --hpo HPO [--obo OBO] --sample SAMPLE --vcf VCF [--assembly {hg38,hg19}] --out-dir OUT_DIR --analysis ANALYSIS --terms TERMS [--pedigree PEDIGREE] [--ped PED] [--affected AFFECTED] [--unaffected UNAFFECTED] +og exomiser prepare: error: argument --assembly: invalid choice: 'grch99' (choose from 'hg38', 'hg19') diff --git a/tests/parity/golden/exomiser/report-top-default/stdout b/tests/parity/golden/exomiser/report-top-default/stdout new file mode 100644 index 000000000..3b84494e7 --- /dev/null +++ b/tests/parity/golden/exomiser/report-top-default/stdout @@ -0,0 +1,5 @@ +Symptoms searched: Brachydactyly (HP:0001156). +1. GENEA (autosomal dominant): Exomiser score 0.998 (symptom match 0.800, variant 0.950). chr10:123256215 T>G heterozygous, missense_variant GENEA:ENST1:c.1688A>C:p.(Glu563Ala), ACMG likely pathogenic (PM2,PP3_Strong,PP4), linked disease Synthetic syndrome one (OMIM:100001), ClinVar likely pathogenic (1 star), highest population frequency not seen. +2. GENEB (autosomal recessive): Exomiser score 0.900 (symptom match 0.800, variant 0.950). chr6:1000 G>A heterozygous, splice_donor_variant GENEB:ENST2:c.10+1G>A:p.?, ACMG uncertain significance (PVS1_Strong), highest population frequency GNOMAD_E_SAS 0.0032%; chr6:2000 C>T heterozygous, stop_gained GENEB:ENST2:c.300C>T:p.(Gln100*), ACMG pathogenic (PVS1,PM2), highest population frequency not seen. +3. GENEC (X-linked recessive): Exomiser score 0.900 (symptom match 0.800, variant 0.950). chrX:5000 A>G hemizygous, missense_variant GENEC:ENST3:c.5A>G:p.(Asp2Gly), ACMG uncertain significance, highest population frequency not seen. +4. GENED (autosomal dominant): Exomiser score 0.900 (symptom match 0.800, variant 0.950). chr1:7000 A>T heterozygous, missense_variant GENED:ENST4:c.7A>T:p.(Asn3Ile), ACMG vus, highest population frequency not seen. diff --git a/tests/parity/golden/locityper-report/cut/out.tsv b/tests/parity/golden/locityper-report/cut/out.tsv new file mode 100644 index 000000000..171cbf97f --- /dev/null +++ b/tests/parity/golden/locityper-report/cut/out.tsv @@ -0,0 +1,2 @@ +locus region status genotype haplotype_1 haplotype_2 quality total_reads unexplained_reads weighted_distance warnings filter reason +CYP2D6 chr22:42122285-42152960 not called . . . . . . . . . res.json.gz cannot be read: incomplete deflate stream diff --git a/tests/parity/golden/locityper-report/int-genotype/exit b/tests/parity/golden/locityper-report/int-genotype/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/locityper-report/int-genotype/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/locityper-report/int-genotype/out.tsv b/tests/parity/golden/locityper-report/int-genotype/out.tsv new file mode 100644 index 000000000..ce185feea --- /dev/null +++ b/tests/parity/golden/locityper-report/int-genotype/out.tsv @@ -0,0 +1,2 @@ +locus region status genotype haplotype_1 haplotype_2 quality total_reads unexplained_reads weighted_distance warnings filter reason +CYP2D6 chr22:42122285-42152960 not called . . . . . . . . . Locityper gave a genotype that is not text diff --git a/tests/parity/golden/locityper-report/int-genotype/stderr b/tests/parity/golden/locityper-report/int-genotype/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/locityper-report/int-genotype/stdout b/tests/parity/golden/locityper-report/int-genotype/stdout new file mode 100644 index 000000000..5badc2578 --- /dev/null +++ b/tests/parity/golden/locityper-report/int-genotype/stdout @@ -0,0 +1 @@ +0 of 1 loci called, 0 pass the authors' filter diff --git a/tests/parity/golden/locityper-report/list-json/exit b/tests/parity/golden/locityper-report/list-json/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/locityper-report/list-json/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/locityper-report/list-json/out.tsv b/tests/parity/golden/locityper-report/list-json/out.tsv new file mode 100644 index 000000000..d3364a88e --- /dev/null +++ b/tests/parity/golden/locityper-report/list-json/out.tsv @@ -0,0 +1,2 @@ +locus region status genotype haplotype_1 haplotype_2 quality total_reads unexplained_reads weighted_distance warnings filter reason +CYP2D6 chr22:42122285-42152960 not called . . . . . . . . . res.json.gz is not a JSON object diff --git a/tests/parity/golden/locityper-report/list-json/stderr b/tests/parity/golden/locityper-report/list-json/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/locityper-report/list-json/stdout b/tests/parity/golden/locityper-report/list-json/stdout new file mode 100644 index 000000000..5badc2578 --- /dev/null +++ b/tests/parity/golden/locityper-report/list-json/stdout @@ -0,0 +1 @@ +0 of 1 loci called, 0 pass the authors' filter diff --git a/tests/parity/golden/locityper-report/notjson/out.tsv b/tests/parity/golden/locityper-report/notjson/out.tsv new file mode 100644 index 000000000..f09dd6afc --- /dev/null +++ b/tests/parity/golden/locityper-report/notjson/out.tsv @@ -0,0 +1,2 @@ +locus region status genotype haplotype_1 haplotype_2 quality total_reads unexplained_reads weighted_distance warnings filter reason +CYP2D6 chr22:42122285-42152960 not called . . . . . . . . . res.json.gz cannot be read: Expecting property name enclosed in double quotes: byte 1 diff --git a/tests/parity/golden/locityper-report/unicode-bed/stderr b/tests/parity/golden/locityper-report/unicode-bed/stderr new file mode 100644 index 000000000..988b1aefe --- /dev/null +++ b/tests/parity/golden/locityper-report/unicode-bed/stderr @@ -0,0 +1 @@ +ERROR: unicode.bed:1: want chrom, start, end, name (tab separated) diff --git a/tests/parity/golden/mei-annotsv-input/no-chrom-line/stderr b/tests/parity/golden/mei-annotsv-input/no-chrom-line/stderr new file mode 100644 index 000000000..8c342a22e --- /dev/null +++ b/tests/parity/golden/mei-annotsv-input/no-chrom-line/stderr @@ -0,0 +1 @@ +ERROR: nochrom.vcf: a record before the #CHROM line (line 1): not a VCF diff --git a/tests/parity/golden/mei-report/annotsv-bad/stderr b/tests/parity/golden/mei-report/annotsv-bad/stderr new file mode 100644 index 000000000..77099099f --- /dev/null +++ b/tests/parity/golden/mei-report/annotsv-bad/stderr @@ -0,0 +1 @@ +ERROR: annotsv-bad.tsv: not AnnotSV's table (a column of ID, Annotation_mode, Gene_name, ACMG_class is missing) diff --git a/tests/parity/golden/mei-report/bad-catalog/stderr b/tests/parity/golden/mei-report/bad-catalog/stderr new file mode 100644 index 000000000..60b3499b1 --- /dev/null +++ b/tests/parity/golden/mei-report/bad-catalog/stderr @@ -0,0 +1 @@ +ERROR: sites-bad.tsv: no rows, or a column of gene, chrom, start, end, site, element, inheritance, condition, clinvar, pmid is missing diff --git a/tests/parity/golden/mei-report/bad-depth/stderr b/tests/parity/golden/mei-report/bad-depth/stderr new file mode 100644 index 000000000..d70bfde5a --- /dev/null +++ b/tests/parity/golden/mei-report/bad-depth/stderr @@ -0,0 +1 @@ +ERROR: depth-bad.tsv line 1: not samtools bedcov -c output of a 4-column BED diff --git a/tests/parity/golden/mei-report/no-chrom-line/stderr b/tests/parity/golden/mei-report/no-chrom-line/stderr new file mode 100644 index 000000000..8c342a22e --- /dev/null +++ b/tests/parity/golden/mei-report/no-chrom-line/stderr @@ -0,0 +1 @@ +ERROR: nochrom.vcf: a record before the #CHROM line (line 1): not a VCF diff --git a/tests/parity/golden/mei-report/not-megane/stderr b/tests/parity/golden/mei-report/not-megane/stderr new file mode 100644 index 000000000..ca2f7beec --- /dev/null +++ b/tests/parity/golden/mei-report/not-megane/stderr @@ -0,0 +1 @@ +ERROR: not-megane.vcf line 9: no SVTYPE or MEPRED in INFO: not MEGAnE's VCF diff --git a/tests/parity/golden/mei-report/two-windows/s.tsv b/tests/parity/golden/mei-report/two-windows/s.tsv new file mode 100644 index 000000000..72c11247c --- /dev/null +++ b/tests/parity/golden/mei-report/two-windows/s.tsv @@ -0,0 +1,4 @@ +gene chrom start end site element inheritance condition clinvar pmid status mean_depth call +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy 3199 9690476 seen 30.0 chr9:105639660 SVA_F 1 copy PASS +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism 9862 29474918 not callable . . +FKTN chr9 105700000 105701000 105700500 SVA autosomal recessive a second window . . not callable 5.0 . diff --git a/tests/parity/golden/mocha-report/bad-phasing/stderr b/tests/parity/golden/mocha-report/bad-phasing/stderr new file mode 100644 index 000000000..0c6cd525e --- /dev/null +++ b/tests/parity/golden/mocha-report/bad-phasing/stderr @@ -0,0 +1,2 @@ +usage: og mocha report [-h] --sample SAMPLE --calls CALLS --stats STATS --sites SITES --phasing {beagle,input} --out-calls OUT_CALLS --out-summary OUT_SUMMARY +og mocha report: error: argument --phasing: invalid choice: 'shapeit' (choose from 'beagle', 'input') diff --git a/tests/parity/golden/mocha-report/short-call/exit b/tests/parity/golden/mocha-report/short-call/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/mocha-report/short-call/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/mocha-report/short-call/stderr b/tests/parity/golden/mocha-report/short-call/stderr new file mode 100644 index 000000000..23a5ea284 --- /dev/null +++ b/tests/parity/golden/mocha-report/short-call/stderr @@ -0,0 +1 @@ +ERROR: calls-short.tsv:2: the row ends before column length diff --git a/tests/parity/golden/mocha-report/two-samples/s.tsv b/tests/parity/golden/mocha-report/two-samples/s.tsv new file mode 100644 index 000000000..3ee6276a9 --- /dev/null +++ b/tests/parity/golden/mocha-report/two-samples/s.tsv @@ -0,0 +1,17 @@ +metric value +status assessed +message . +depth 30.0 +het_sites 18000 +phased_het_sites 18000 +het_per_10mb 3000 +analysed_mb 60.0 +chromosomes chr20 +call_rate 0.9990 +baf_auto 0.0010 +floor_read 0.020 +floor_min 0.05 +floor 0.050 +calls_mocha 1 +calls_reported 1 +phasing input diff --git a/tests/parity/golden/mocha-report/two-samples/stdout b/tests/parity/golden/mocha-report/two-samples/stdout new file mode 100644 index 000000000..b41cf86b2 --- /dev/null +++ b/tests/parity/golden/mocha-report/two-samples/stdout @@ -0,0 +1 @@ +S: assessed; 1 of 1 MoChA calls reported (floor 0.050, depth 30.0x) diff --git a/tests/parity/golden/viral-report/bad-coverage/stderr b/tests/parity/golden/viral-report/bad-coverage/stderr new file mode 100644 index 000000000..d567e64b2 --- /dev/null +++ b/tests/parity/golden/viral-report/bad-coverage/stderr @@ -0,0 +1 @@ +ERROR: bad.hcov.tsv is not samtools coverage output diff --git a/tests/parity/golden/viral-report/cut-depth/stderr b/tests/parity/golden/viral-report/cut-depth/stderr new file mode 100644 index 000000000..6c478f7ad --- /dev/null +++ b/tests/parity/golden/viral-report/cut-depth/stderr @@ -0,0 +1 @@ +ERROR: cut.vdepth.tsv has 490 positions for NC_001422.1, its length is 500 (run samtools depth -aa) diff --git a/tests/parity/golden/vntyper-report/negative-mean/exit b/tests/parity/golden/vntyper-report/negative-mean/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/vntyper-report/negative-mean/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/vntyper-report/negative-mean/stderr b/tests/parity/golden/vntyper-report/negative-mean/stderr new file mode 100644 index 000000000..51cece7e7 --- /dev/null +++ b/tests/parity/golden/vntyper-report/negative-mean/stderr @@ -0,0 +1 @@ +ERROR: cannot read VNtyper's results in negmean: VNtyper's coverage table gives a negative mean depth over the VNTR (-5.0) diff --git a/tests/parity/golden/vntyper-report/unreadable/stderr b/tests/parity/golden/vntyper-report/unreadable/stderr new file mode 100644 index 000000000..2081d1ff1 --- /dev/null +++ b/tests/parity/golden/vntyper-report/unreadable/stderr @@ -0,0 +1 @@ +ERROR: cannot read VNtyper's results in bad: bad/kestrel/kestrel_result.tsv:1: invalid utf-8 sequence of 1 bytes from index 0 diff --git a/tests/parity/golden/y-str-report/panel-copies/exit b/tests/parity/golden/y-str-report/panel-copies/exit new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/tests/parity/golden/y-str-report/panel-copies/exit @@ -0,0 +1 @@ +1 diff --git a/tests/parity/golden/y-str-report/panel-copies/o.csv.absent b/tests/parity/golden/y-str-report/panel-copies/o.csv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/y-str-report/panel-copies/o.tsv.absent b/tests/parity/golden/y-str-report/panel-copies/o.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/y-str-report/panel-copies/stderr b/tests/parity/golden/y-str-report/panel-copies/stderr new file mode 100644 index 000000000..244607a03 --- /dev/null +++ b/tests/parity/golden/y-str-report/panel-copies/stderr @@ -0,0 +1 @@ +ERROR: panel-copies.tsv: DYS393: unreadable region chrY:3263111-3263158/4/-3/DYS393 diff --git a/tests/parity/golden/y-str-report/panel-copies/stdout b/tests/parity/golden/y-str-report/panel-copies/stdout new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/y-str-report/panel-period/stderr b/tests/parity/golden/y-str-report/panel-period/stderr new file mode 100644 index 000000000..01fbd3370 --- /dev/null +++ b/tests/parity/golden/y-str-report/panel-period/stderr @@ -0,0 +1 @@ +ERROR: panel-period.tsv: DYS393: unreadable region chrY:3263111-3263158/0/12/DYS393 diff --git a/tests/parity/inputs/exomiser/S1_exomiser.ped b/tests/parity/inputs/exomiser/S1_exomiser.ped new file mode 100644 index 000000000..aa821f856 --- /dev/null +++ b/tests/parity/inputs/exomiser/S1_exomiser.ped @@ -0,0 +1,3 @@ +family M 0 0 2 1 +family F 0 0 1 1 +family S1 F M 1 2 diff --git a/tests/parity/inputs/exomiser/empty.tsv b/tests/parity/inputs/exomiser/empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/exomiser/fam.vcf.gz b/tests/parity/inputs/exomiser/fam.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..d3bae0e7ee6ca85ddec011d440280c2e527a5fbb GIT binary patch literal 126 zcmV-^0D=D>iwFP!00002|24<44uUWg1YYYky8k`L}68WvkFbDgmzPB7k6?Oms0NwmEVE_OC literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/exomiser/family.tsv b/tests/parity/inputs/exomiser/family.tsv new file mode 100644 index 000000000..25da1d490 --- /dev/null +++ b/tests/parity/inputs/exomiser/family.tsv @@ -0,0 +1,2 @@ +#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_CLASSIFICATION EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIATION_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH +1 chr10-123256215-T-G_AD GENEA 1 AD 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr10 123256215 123256215 T G 0 50.0000 PASS 0/0:0/1:0/1 missense_variant GENEA:ENST1:c.1688A>C:p.(Glu563Ala) PATHOGENIC NOT_PROVIDED 0 diff --git a/tests/parity/inputs/exomiser/header-only.tsv b/tests/parity/inputs/exomiser/header-only.tsv new file mode 100644 index 000000000..3ffee9530 --- /dev/null +++ b/tests/parity/inputs/exomiser/header-only.tsv @@ -0,0 +1 @@ +#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_CLASSIFICATION EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIATION_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH diff --git a/tests/parity/inputs/exomiser/hp.obo b/tests/parity/inputs/exomiser/hp.obo new file mode 100644 index 000000000..06befb56f --- /dev/null +++ b/tests/parity/inputs/exomiser/hp.obo @@ -0,0 +1,24 @@ +format-version: 1.2 + +[Term] +id: HP:0001156 +name: Brachydactyly +alt_id: HP:0009803 + +[Term] +id: HP:0001363 +name: Craniosynostosis + +[Term] +id: HP:0011304 +name: Broad thumb + +[Term] +id: HP:0001250 +name: obsolete Seizure +is_obsolete: true +replaced_by: HP:0012345 + +[Typedef] +id: part_of +name: part of diff --git a/tests/parity/inputs/exomiser/no-acmg.tsv b/tests/parity/inputs/exomiser/no-acmg.tsv new file mode 100644 index 000000000..b7489a7df --- /dev/null +++ b/tests/parity/inputs/exomiser/no-acmg.tsv @@ -0,0 +1,7 @@ +#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIATION_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH +1 chr10-123256215-T-G_AD GENEA 1 AD 0.0010 0.9981 0.8000 0.9500 0.9500 1 0 chr10 123256215 123256215 T G 0 50.0000 PASS 0/1 missense_variant GENEA:ENST1:c.1688A>C:p.(Glu563Ala) PM2,PP3_Strong,PP4 OMIM:100001 Synthetic syndrome one LIKELY_PATHOGENIC 1 +2 chr6-1000-G-A_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr6 1000 1000 G A 0 50.0000 PASS 0/1 splice_donor_variant GENEB:ENST2:c.10+1G>A:p.? PVS1_Strong NOT_PROVIDED 0 GNOMAD_E_SAS 0.0032 +2 chr6-2000-C-T_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr6 2000 2000 C T 0 50.0000 PASS 1|0 stop_gained GENEB:ENST2:c.300C>T:p.(Gln100*) PVS1,PM2 NOT_PROVIDED 0 +2 chr6-3000-A-C_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 0 0 chr6 3000 3000 A C 0 50.0000 PASS 0/1 missense_variant GENEB:ENST2:c.400A>C:p.(Lys134Thr) NOT_PROVIDED 0 +3 chrX-5000-A-G_XR GENEC 1 XR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chrX 5000 5000 A G 0 50.0000 PASS 1 missense_variant GENEC:ENST3:c.5A>G:p.(Asp2Gly) NOT_PROVIDED 0 +4 chr1-7000-A-T_AD GENED 1 AD 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr1 7000 7000 A T 0 50.0000 PASS 0/1 missense_variant GENED:ENST4:c.7A>T:p.(Asn3Ile) NOT_PROVIDED 0 diff --git a/tests/parity/inputs/exomiser/one.vcf.gz b/tests/parity/inputs/exomiser/one.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..a5afcc190ce4fd95de67ad62b0be97f3bf7ed7e7 GIT binary patch literal 119 zcmV--0Eqt|iwFP!00002{|(2n4uUWg1H&0w?LIAUid^cL!%J Z3bPbA006j0Im7?} literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/exomiser/pedigree.tsv b/tests/parity/inputs/exomiser/pedigree.tsv new file mode 100644 index 000000000..70f12ff7e --- /dev/null +++ b/tests/parity/inputs/exomiser/pedigree.tsv @@ -0,0 +1,5 @@ +sample sex father mother partner +S1 female DAD MUM . +SIB male DAD MUM . +DAD male . . MUM +MUM female . . DAD diff --git a/tests/parity/inputs/exomiser/short.tsv b/tests/parity/inputs/exomiser/short.tsv new file mode 100644 index 000000000..800cd0f8c --- /dev/null +++ b/tests/parity/inputs/exomiser/short.tsv @@ -0,0 +1,2 @@ +#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_CLASSIFICATION EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIATION_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH +1 x GENEA diff --git a/tests/parity/inputs/exomiser/stranger.vcf.gz b/tests/parity/inputs/exomiser/stranger.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..3ce2fd318987ac82014721f7fa20de6f227138e8 GIT binary patch literal 126 zcmV-^0D=D>iwFP!00002{}sov3c@fD2H^S6YhVnwXnlR`X1nT8bD gO!vjZJBJ2@gq_1kBEKbt+xS281wwADHgf;~03OXb&Hw-a literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/exomiser/terms.tsv b/tests/parity/inputs/exomiser/terms.tsv new file mode 100644 index 000000000..399c823ea --- /dev/null +++ b/tests/parity/inputs/exomiser/terms.tsv @@ -0,0 +1,2 @@ +hpo_id label given_as +HP:0001156 Brachydactyly HP:0001156 diff --git a/tests/parity/inputs/exomiser/variants.tsv b/tests/parity/inputs/exomiser/variants.tsv new file mode 100644 index 000000000..c707ebb11 --- /dev/null +++ b/tests/parity/inputs/exomiser/variants.tsv @@ -0,0 +1,7 @@ +#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_CLASSIFICATION EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIATION_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH +1 chr10-123256215-T-G_AD GENEA 1 AD 0.0010 0.9981 0.8000 0.9500 0.9500 1 0 chr10 123256215 123256215 T G 0 50.0000 PASS 0/1 missense_variant GENEA:ENST1:c.1688A>C:p.(Glu563Ala) LIKELY_PATHOGENIC PM2,PP3_Strong,PP4 OMIM:100001 Synthetic syndrome one LIKELY_PATHOGENIC 1 +2 chr6-1000-G-A_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr6 1000 1000 G A 0 50.0000 PASS 0/1 splice_donor_variant GENEB:ENST2:c.10+1G>A:p.? UNCERTAIN_SIGNIFICANCE PVS1_Strong NOT_PROVIDED 0 GNOMAD_E_SAS 0.0032 +2 chr6-2000-C-T_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr6 2000 2000 C T 0 50.0000 PASS 1|0 stop_gained GENEB:ENST2:c.300C>T:p.(Gln100*) PATHOGENIC PVS1,PM2 NOT_PROVIDED 0 +2 chr6-3000-A-C_AR GENEB 1 AR 0.0010 0.9000 0.8000 0.9500 0.9500 0 0 chr6 3000 3000 A C 0 50.0000 PASS 0/1 missense_variant GENEB:ENST2:c.400A>C:p.(Lys134Thr) NOT_AVAILABLE NOT_PROVIDED 0 +3 chrX-5000-A-G_XR GENEC 1 XR 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chrX 5000 5000 A G 0 50.0000 PASS 1 missense_variant GENEC:ENST3:c.5A>G:p.(Asp2Gly) UNCERTAIN_SIGNIFICANCE NOT_PROVIDED 0 +4 chr1-7000-A-T_AD GENED 1 AD 0.0010 0.9000 0.8000 0.9500 0.9500 1 0 chr1 7000 7000 A T 0 50.0000 PASS 0/1 missense_variant GENED:ENST4:c.7A>T:p.(Asn3Ile) VUS NOT_PROVIDED 0 diff --git a/tests/parity/inputs/mei/annotsv-bad.tsv b/tests/parity/inputs/mei/annotsv-bad.tsv new file mode 100644 index 000000000..c3586b871 --- /dev/null +++ b/tests/parity/inputs/mei/annotsv-bad.tsv @@ -0,0 +1,2 @@ +SV_chrom SV_start +20 1 diff --git a/tests/parity/inputs/mei/annotsv-empty.tsv b/tests/parity/inputs/mei/annotsv-empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/mei/annotsv.tsv b/tests/parity/inputs/mei/annotsv.tsv new file mode 100644 index 000000000..c70640183 --- /dev/null +++ b/tests/parity/inputs/mei/annotsv.tsv @@ -0,0 +1,5 @@ +AnnotSV_ID SV_chrom SV_start SV_end SV_type ID Annotation_mode Gene_name Location B_ins_AFmax ACMG_class +20_5040477_5040478_INS_1 20 5040477 5040478 INS chr20_5040478_Alu full FIXG1;FIXG2 0.0312 NA +20_5040477_5040478_INS_1 20 5040477 5040478 INS chr20_5040478_Alu split FIXG1 intron2-intron2 full=NA +20_5040477_5040478_INS_1 20 5040477 5040478 INS chr20_5040478_Alu split "FIXG2 exon1-exon1 full=NA +9_105639659_105639660_INS_1 9 105639659 105639660 INS chr9_105639660_SVA full NA diff --git a/tests/parity/inputs/mei/catalog.tsv b/tests/parity/inputs/mei/catalog.tsv new file mode 100644 index 000000000..06bdb9bf7 --- /dev/null +++ b/tests/parity/inputs/mei/catalog.tsv @@ -0,0 +1,9 @@ +# Known disease-causing mobile element insertions (step 50, og mei report). One row per insertion. +# GRCh38 coordinates, 1-based. site: the base after which the element is inserted, from the ClinVar record +# (GRCh38 location, or its HGVS mapped with Ensembl variant_recoder, or its RefSeqGene sequence found on +# chrX); start and end: site -/+ 500 bp, where a call of the same element counts as this insertion. +# Checked 2026-10-27 against ClinVar, PubMed and Ensembl. +gene chrom start end site element inheritance condition clinvar pmid +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy (3' UTR insertion) 3199 9690476 +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism (intron 32 insertion) 9862 29474918 +BBS1 chr11 66526182 66527182 66526682 SVA autosomal recessive Bardet-Biedl syndrome 1 (exon 13 insertion, c.1214_1215insSVA) 981241 33169370 diff --git a/tests/parity/inputs/mei/classes.vcf b/tests/parity/inputs/mei/classes.vcf new file mode 100644 index 000000000..7c8556a2f --- /dev/null +++ b/tests/parity/inputs/mei/classes.vcf @@ -0,0 +1,20 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr20 5040478 . N . LC SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=5040486;MEI=AluYb8,0/298,+/+|AluYb9,0/298,+/+;CN_conf=L CN 1 +chr2 1000 . N . PASS SVTYPE=LINE/L1;MEPRED=PASS;HOMLEN=9;MEI_rpos=1008;MEI_lbp=L1HS,5581,+;MEI_rbp=pA;CN_conf=H CN 2 +chr2 2000 . N . PASS SVTYPE=LINE/L1;MEPRED=PASS;HOMLEN=9;MEI_rpos=2008;MEI_lbp=NA,NA,+;MEI_rbp=pA;CN_conf=H CN 1 +chrX 3000 . N . PASS SVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=3008;MEI=SVA_E,1/900,+/+;CN_conf=H CN 1 +chr20 6665193 . N . PASS SVTYPE=DNA/hAT-Charlie;MEPRED=PASS;HOMLEN=9;MEI_rpos=6665201;MEI=Charlie5,2546/2611,+/+;CN_conf=H CN 1 +chr20 16134625 . N . PASS SVTYPE=LTR/ERV1;MEPRED=PASS;HOMLEN=9;MEI_rpos=16134633;MEI=MER41A,91/60,-/-;CN_conf=H CN 1 +chr20 28576054 . N . PASS SVTYPE=LINE/L1;MEPRED=FAILED;HOMLEN=9;MEI_rpos=28576062;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 +chr20 30000000 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=30000008;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 0 +chr20 100 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=108;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 +chr20 100 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=108;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 +chrUn_KI270302v1 50 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=58;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 +chr9 105639660 . N . PASS SVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=105639668;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 2 diff --git a/tests/parity/inputs/mei/depth-bad.tsv b/tests/parity/inputs/mei/depth-bad.tsv new file mode 100644 index 000000000..908f60e24 --- /dev/null +++ b/tests/parity/inputs/mei/depth-bad.tsv @@ -0,0 +1 @@ +chr9 105639155 105640156 diff --git a/tests/parity/inputs/mei/depth-catalog.tsv b/tests/parity/inputs/mei/depth-catalog.tsv new file mode 100644 index 000000000..4b65a4b9c --- /dev/null +++ b/tests/parity/inputs/mei/depth-catalog.tsv @@ -0,0 +1,2 @@ +chr9 105639155 105640156 FKTN 31000 200 +chrX 71439992 71440993 TAF1 2000 20 diff --git a/tests/parity/inputs/mei/depth-empty.tsv b/tests/parity/inputs/mei/depth-empty.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/inputs/mei/depth-thin.tsv b/tests/parity/inputs/mei/depth-thin.tsv new file mode 100644 index 000000000..adf544f5e --- /dev/null +++ b/tests/parity/inputs/mei/depth-thin.tsv @@ -0,0 +1 @@ +chr9 105639155 105640156 FKTN 9009 60 diff --git a/tests/parity/inputs/mei/depth-two-windows.tsv b/tests/parity/inputs/mei/depth-two-windows.tsv new file mode 100644 index 000000000..79cf28aa0 --- /dev/null +++ b/tests/parity/inputs/mei/depth-two-windows.tsv @@ -0,0 +1,2 @@ +chr9 105639155 105640156 FKTN 30030 200 +chr9 105699999 105701000 FKTN 5005 30 diff --git a/tests/parity/inputs/mei/depth.tsv b/tests/parity/inputs/mei/depth.tsv new file mode 100644 index 000000000..5be256e14 --- /dev/null +++ b/tests/parity/inputs/mei/depth.tsv @@ -0,0 +1,2 @@ +chr9 105639155 105640156 FKTN 30030 200 +chrX 71439992 71440993 TAF1 30030 200 diff --git a/tests/parity/inputs/mei/empty.vcf b/tests/parity/inputs/mei/empty.vcf new file mode 100644 index 000000000..62f72ba32 --- /dev/null +++ b/tests/parity/inputs/mei/empty.vcf @@ -0,0 +1,8 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S diff --git a/tests/parity/inputs/mei/fktn.vcf b/tests/parity/inputs/mei/fktn.vcf new file mode 100644 index 000000000..804f74cfa --- /dev/null +++ b/tests/parity/inputs/mei/fktn.vcf @@ -0,0 +1,10 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr20 5040478 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=5040486;MEI=AluYa5,0/288,+/+;CN_conf=H CN 1 +chr9 105639660 . N . PASS SVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=105639668;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 diff --git a/tests/parity/inputs/mei/nochrom.vcf b/tests/parity/inputs/mei/nochrom.vcf new file mode 100644 index 000000000..739534f3b --- /dev/null +++ b/tests/parity/inputs/mei/nochrom.vcf @@ -0,0 +1 @@ +chr9 105639660 . N . PASS SVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=105639668;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 diff --git a/tests/parity/inputs/mei/none.vcf b/tests/parity/inputs/mei/none.vcf new file mode 100644 index 000000000..2f147982b --- /dev/null +++ b/tests/parity/inputs/mei/none.vcf @@ -0,0 +1,9 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr20 5040478 . N . PASS SVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=5040486;MEI=AluYa5,0/288,+/+;CN_conf=H CN 1 diff --git a/tests/parity/inputs/mei/not-megane.vcf b/tests/parity/inputs/mei/not-megane.vcf new file mode 100644 index 000000000..cd955418b --- /dev/null +++ b/tests/parity/inputs/mei/not-megane.vcf @@ -0,0 +1,9 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr9 1 . N . PASS END=5 CN 1 diff --git a/tests/parity/inputs/mei/outside.vcf b/tests/parity/inputs/mei/outside.vcf new file mode 100644 index 000000000..3060e1cc4 --- /dev/null +++ b/tests/parity/inputs/mei/outside.vcf @@ -0,0 +1,9 @@ +##fileformat=VCFv4.1 +##source=MEI search version "v1.0.1 2022/04/12" +##contig= +##ALT= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr9 105641200 . N . PASS SVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=105641208;MEI=SVA_F,1/1500,+/+;CN_conf=H CN 1 diff --git a/tests/parity/inputs/mei/sites-bad.tsv b/tests/parity/inputs/mei/sites-bad.tsv new file mode 100644 index 000000000..6540a8fcc --- /dev/null +++ b/tests/parity/inputs/mei/sites-bad.tsv @@ -0,0 +1,2 @@ +gene chrom +FKTN chr9 diff --git a/tests/parity/inputs/mei/sites-two-windows.tsv b/tests/parity/inputs/mei/sites-two-windows.tsv new file mode 100644 index 000000000..946d63410 --- /dev/null +++ b/tests/parity/inputs/mei/sites-two-windows.tsv @@ -0,0 +1,5 @@ +# a test catalog +gene chrom start end site element inheritance condition clinvar pmid +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy 3199 9690476 +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism 9862 29474918 +FKTN chr9 105700000 105701000 105700500 SVA autosomal recessive a second window . . diff --git a/tests/parity/inputs/mei/sites.tsv b/tests/parity/inputs/mei/sites.tsv new file mode 100644 index 000000000..26d3f7061 --- /dev/null +++ b/tests/parity/inputs/mei/sites.tsv @@ -0,0 +1,4 @@ +# a test catalog +gene chrom start end site element inheritance condition clinvar pmid +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy 3199 9690476 +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism 9862 29474918 diff --git a/tests/parity/inputs/mocha/calls-crlf.tsv b/tests/parity/inputs/mocha/calls-crlf.tsv new file mode 100644 index 000000000..0f2f66fe0 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-crlf.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 diff --git a/tests/parity/inputs/mocha/calls-empty.tsv b/tests/parity/inputs/mocha/calls-empty.tsv new file mode 100644 index 000000000..d42987a76 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-empty.tsv @@ -0,0 +1 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf diff --git a/tests/parity/inputs/mocha/calls-filter.tsv b/tests/parity/inputs/mocha/calls-filter.tsv new file mode 100644 index 000000000..2794cfa82 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-filter.tsv @@ -0,0 +1,9 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CNP_Loss 0.15 +S U chr20 60000 20000000 19940000 T N 20000 6600 500000 0.2 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.4 +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.16 0.001 3.0 0.01 nan 900.0 0 nan nan Gain 0.6 +S U chrX 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH nan +S U chr20 30000000 60000000 30000000 N T 20000 6600 3000 0.03 0.001 2.0 0.01 nan 900.0 0 nan nan Loss 0.12 +S U 20 1000 64000000 63999000 T T 20000 6600 3000 0.075 0.001 2.4 0.01 nan 900.0 0 nan nan Gain 0.3 +S U chr1 120000000 130000000 10000000 N N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan Undetermined 0.2 diff --git a/tests/parity/inputs/mocha/calls-low.tsv b/tests/parity/inputs/mocha/calls-low.tsv new file mode 100644 index 000000000..79833af75 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-low.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.02 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.04 diff --git a/tests/parity/inputs/mocha/calls-one.tsv b/tests/parity/inputs/mocha/calls-one.tsv new file mode 100644 index 000000000..99a484cc7 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-one.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 diff --git a/tests/parity/inputs/mocha/calls-short.tsv b/tests/parity/inputs/mocha/calls-short.tsv new file mode 100644 index 000000000..963df9f6d --- /dev/null +++ b/tests/parity/inputs/mocha/calls-short.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +S U chr20 60000 diff --git a/tests/parity/inputs/mocha/calls-two.tsv b/tests/parity/inputs/mocha/calls-two.tsv new file mode 100644 index 000000000..104b16686 --- /dev/null +++ b/tests/parity/inputs/mocha/calls-two.tsv @@ -0,0 +1,4 @@ +sample_id computed_gender chrom beg_GRCh38 end_GRCh38 length p_arm q_arm n_sites n_hets n50_hets bdev bdev_se rel_cov rel_cov_se lod_lrr_baf lod_baf_phase n_flips baf_conc lod_baf_conc type cf +T U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 +S U chr20 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 +T U chr2 60000 20000000 19940000 T N 20000 6600 3000 0.075 0.001 2.0 0.01 nan 900.0 0 nan nan CN-LOH 0.15 diff --git a/tests/parity/inputs/mocha/sites-300.tsv b/tests/parity/inputs/mocha/sites-300.tsv new file mode 100644 index 000000000..a8583c0fd --- /dev/null +++ b/tests/parity/inputs/mocha/sites-300.tsv @@ -0,0 +1 @@ +chr20 1 60000000 5400 1800 1800 diff --git a/tests/parity/inputs/mocha/sites-many.tsv b/tests/parity/inputs/mocha/sites-many.tsv new file mode 100644 index 000000000..2b7cf22c9 --- /dev/null +++ b/tests/parity/inputs/mocha/sites-many.tsv @@ -0,0 +1,23 @@ +chr1 10001 100010001 120000 95000 94000 +chr2 10001 100010001 120000 95000 94000 +chr3 10001 100010001 120000 95000 94000 +chr4 10001 100010001 120000 95000 94000 +chr5 10001 100010001 120000 95000 94000 +chr6 10001 100010001 120000 95000 94000 +chr7 10001 100010001 120000 95000 94000 +chr8 10001 100010001 120000 95000 94000 +chr9 10001 100010001 120000 95000 94000 +chr10 10001 100010001 120000 95000 94000 +chr11 10001 100010001 120000 95000 94000 +chr12 10001 100010001 120000 95000 94000 +chr13 10001 100010001 120000 95000 94000 +chr14 10001 100010001 120000 95000 94000 +chr15 10001 100010001 120000 95000 94000 +chr16 10001 100010001 120000 95000 94000 +chr17 10001 100010001 120000 95000 94000 +chr18 10001 100010001 120000 95000 94000 +chr19 10001 100010001 120000 95000 94000 +chr20 10001 100010001 120000 95000 94000 +chr21 10001 100010001 120000 95000 94000 +chr22 10001 100010001 120000 95000 94000 +chrX 1 156000000 1000 10 10 diff --git a/tests/parity/inputs/mocha/sites-short.tsv b/tests/parity/inputs/mocha/sites-short.tsv new file mode 100644 index 000000000..65a9361d0 --- /dev/null +++ b/tests/parity/inputs/mocha/sites-short.tsv @@ -0,0 +1 @@ +chr20 1 100 diff --git a/tests/parity/inputs/mocha/sites-unphased.tsv b/tests/parity/inputs/mocha/sites-unphased.tsv new file mode 100644 index 000000000..2ab6c5fb2 --- /dev/null +++ b/tests/parity/inputs/mocha/sites-unphased.tsv @@ -0,0 +1 @@ +chr20 1 60000000 54000 18000 0 diff --git a/tests/parity/inputs/mocha/sites.tsv b/tests/parity/inputs/mocha/sites.tsv new file mode 100644 index 000000000..0e91b88bb --- /dev/null +++ b/tests/parity/inputs/mocha/sites.tsv @@ -0,0 +1 @@ +chr20 1 60000000 54000 18000 18000 diff --git a/tests/parity/inputs/mocha/stats-5x.tsv b/tests/parity/inputs/mocha/stats-5x.tsv new file mode 100644 index 000000000..f5dc15a2a --- /dev/null +++ b/tests/parity/inputs/mocha/stats-5x.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +S U 0.999 5 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/mocha/stats-autocorr.tsv b/tests/parity/inputs/mocha/stats-autocorr.tsv new file mode 100644 index 000000000..ae0f04c9d --- /dev/null +++ b/tests/parity/inputs/mocha/stats-autocorr.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +S U 0.999 30.0 3 0.01 0.001 0.5 0.05 100000 18000 diff --git a/tests/parity/inputs/mocha/stats-low-rate.tsv b/tests/parity/inputs/mocha/stats-low-rate.tsv new file mode 100644 index 000000000..2ac3e899d --- /dev/null +++ b/tests/parity/inputs/mocha/stats-low-rate.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +S U 0.95 30.0 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/mocha/stats-nan.tsv b/tests/parity/inputs/mocha/stats-nan.tsv new file mode 100644 index 000000000..5819c59eb --- /dev/null +++ b/tests/parity/inputs/mocha/stats-nan.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +S U nan 30.0 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/mocha/stats-nocov.tsv b/tests/parity/inputs/mocha/stats-nocov.tsv new file mode 100644 index 000000000..6a2bd2d00 --- /dev/null +++ b/tests/parity/inputs/mocha/stats-nocov.tsv @@ -0,0 +1,2 @@ +sample_id call_rate +S 0.99 diff --git a/tests/parity/inputs/mocha/stats-other.tsv b/tests/parity/inputs/mocha/stats-other.tsv new file mode 100644 index 000000000..d0a4e8a2a --- /dev/null +++ b/tests/parity/inputs/mocha/stats-other.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +T U 0.999 30.0 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/mocha/stats-two.tsv b/tests/parity/inputs/mocha/stats-two.tsv new file mode 100644 index 000000000..44ae23d23 --- /dev/null +++ b/tests/parity/inputs/mocha/stats-two.tsv @@ -0,0 +1,3 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +T U 0.999 30.0 3 0.01 0.001 0.5 0.001 100000 18000 +S U 0.999 30.0 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/mocha/stats.tsv b/tests/parity/inputs/mocha/stats.tsv new file mode 100644 index 000000000..12aca8b57 --- /dev/null +++ b/tests/parity/inputs/mocha/stats.tsv @@ -0,0 +1,2 @@ +sample_id computed_gender call_rate cov_median cov_sd cov_auto baf_corr baf_conc baf_auto n_sites n_hets +S U 0.999 30.0 3 0.01 0.001 0.5 0.001 100000 18000 diff --git a/tests/parity/inputs/y-str/calls.vcf b/tests/parity/inputs/y-str/calls.vcf new file mode 100644 index 000000000..62cf4bb06 --- /dev/null +++ b/tests/parity/inputs/y-str/calls.vcf @@ -0,0 +1,24 @@ +##fileformat=VCFv4.1 +##source=HipSTR +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S1 +chrY 3263110 DYS393 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 15163066 DYS390 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:-8:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 11982088 DYS391 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:2:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 20471986 DYS392 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:-1:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 8349972 DYS449.1 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 8350055 DYS449.2 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:-4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 0 DYS510 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 0 DYS389II A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 17022969 DYS426 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.6:0.6:20:0:0|0:20|0:.:0:0 +chrY 12635603 DYS388 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:20:0:0|0:20|0:.:5:0 +chrY 12500447 DYS389I A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:20:0:0|0:20|0:.:0:4 +chrY 7999838 DYS458 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:6:0:0|0:6|0:.:0:0 +chrY 0 DYS459a A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 9684379 DYS19/DYS394 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0|1:0|4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 7043527 DYS455 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL .:.:.:.:0:0:0|0:0|0:.:0:0 +chrY 8356114 DYS454 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:x:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 0 DYS447 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:.:0:0|0:.|0:.:0:0 +chrY 0 DYS437 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:.:.:20:0:0|0:20|0:.:0:0 +chrY 11981553 DYS552.1 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 1:4:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 19458591 DYS452 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:20:0:0|0:20|0:.:0:0 +chrY 0 DYS448 A . . PASS PERIOD=4 GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL 0:0:0.99:0.99:20:0:0|0:20|0:.:0:0 diff --git a/tests/parity/inputs/y-str/calls.vcf.gz b/tests/parity/inputs/y-str/calls.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..f6da1c7410da75edc6f0c7a73b6e4884c5c8a2a6 GIT binary patch literal 455 zcmV;&0XY62iwFP!00002|HYQOPQx%1hG+LRG824!i;txafixu|G-(o$7^?~d5(rc- zV(7!;+N~HUD1+O`%2Ax<|MH)U9Yyz#Pj~k(tLMf0_~tzS6fumV=;>?REx00kNG!$H1{T@| z=)<77Kup*MK?sath;qW}?s6DM0w4wc=8UaP%7=Rwx&WwQJe(N+F>kaZ&t`P5Mqk35dV)|0DVUZj2yd{Qv*} Dk!@Y* literal 0 HcmV?d00001 diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index ad4c6688b..e7d745eb7 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -12,3 +12,12 @@ sv-inv-filter-py3 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/manta_inv_fi pgx-refblock-fill 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_refblock_fill.py python3 pgx refblock-fill pgx-mito-merge 3150ebc2613ed786d8bd3e6b5ddef4b31a0f8376 file:bin/pgx_mito_merge.py python3 pgx mito-merge chip 6f2407f9f66cc6bee43ae13c8abae5874b53e1cf file:bin/chip_report.py python3 chip +vntyper-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/vntyper_report.py python3 vntyper report +locityper-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/locityper_report.py python3 locityper report +mocha-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/mocha_report.py python3 mocha report +mei-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/mei_report.py python3 mei report +mei-annotsv-input 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/mei_report.py sh -c 'p=$0; for a; do shift; [ "$a" = --out ] && a=--annotsv-input; set -- "$@" "$a"; done; exec python3 "$p" "$@"' mei annotsv-input +viral-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/viral_report.py python3 viral report +y-str-bed 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/ystr_report.py sh -c 'exec python3 "$0" bed "$@"' y str-bed +y-str-report 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/ystr_report.py sh -c 'exec python3 "$0" report "$@"' y str-report +exomiser 9885cc594d7b4d3295a086599cdcb2946263d899 file:bin/exomiser_report.py python3 exomiser From 6c90c9745b51496d1bf43eb7c91c60e84f2563e1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:16 +0100 Subject: [PATCH 23/53] refactor(reports): steps 45, 47, 48 and 50 to 53 call og The bash steps run /pgp-bin/og in the same images, the Nextflow modules run og from bin/, and the e2e cases call bin/og. Bead: pgp-bs9u.17 --- modules/local/exomiser/main.nf | 8 ++++---- modules/local/locityper/main.nf | 4 ++-- modules/local/mei/main.nf | 10 +++++----- modules/local/mocha/main.nf | 4 ++-- modules/local/viral/main.nf | 4 ++-- modules/local/vntyper/main.nf | 6 +++--- modules/local/y_str/main.nf | 10 +++++----- scripts/45-y-str.sh | 10 +++++----- scripts/47-viral.sh | 6 +++--- scripts/48-exomiser.sh | 8 ++++---- scripts/50-mei.sh | 8 ++++---- scripts/51-mocha.sh | 6 +++--- scripts/52-vntyper.sh | 6 +++--- scripts/53-locityper.sh | 6 +++--- scripts/ci/e2e-case-paths.tsv | 14 +++++++------- tests/e2e/ci-e2e-2-locityper.sh | 4 ++-- tests/e2e/exomiser-1.sh | 2 +- tests/e2e/mei-1.sh | 4 ++-- tests/e2e/y-str-1.sh | 4 ++-- 19 files changed, 62 insertions(+), 62 deletions(-) diff --git a/modules/local/exomiser/main.nf b/modules/local/exomiser/main.nf index 7fa7e1174..c9aaea686 100644 --- a/modules/local/exomiser/main.nf +++ b/modules/local/exomiser/main.nf @@ -5,12 +5,12 @@ Opt-in: runs when --hpo names the symptoms of one sample (--exomiser_proband, or the only sample of the samplesheet). Three processes, as in scripts/48-exomiser.sh: - EXOMISER_PREPARE bin/exomiser_report.py prepare: checks the terms against + EXOMISER_PREPARE og exomiser prepare (bin/og): checks the terms against the HPO release of the phenotype data and writes the analysis (Exomiser's exome preset) EXOMISER Exomiser on the sample's VCF, offline, with its data folder (--exomiser_data, setup.sh --exomiser-data) - EXOMISER_REPORT bin/exomiser_report.py report: the top ten ranks with + EXOMISER_REPORT og exomiser report (bin/og): the top ten ranks with their variants, and the same in words The family mode of step 48 (EXOMISER_FAMILY) is bash-only. @@ -49,7 +49,7 @@ process EXOMISER_PREPARE { for o in ${exomiser_data}/*_phenotype/hp.obo ${exomiser_data}/*_phenotype/phenix/hp.obo; do if [ -s "\$o" ]; then obo="\$o"; break; fi done - exomiser_report.py prepare --hpo '${hpo}' \${obo:+--obo "\$obo"} --sample ${meta.id} --vcf "${vcf}" \\ + og exomiser prepare --hpo '${hpo}' \${obo:+--obo "\$obo"} --sample ${meta.id} --vcf "${vcf}" \\ --assembly ${params.exomiser_assembly} --out-dir results --analysis ${meta.id}_analysis.yml \\ --terms ${meta.id}_exomiser_terms.tsv @@ -162,7 +162,7 @@ process EXOMISER_REPORT { // published with publish_dir_mode link breaks once work/ is cleaned. """ cp ${terms} ${meta.id}_exomiser_terms.tsv - exomiser_report.py report --sample ${meta.id} --variants ${variants} --terms ${terms} \\ + og exomiser report --sample ${meta.id} --variants ${variants} --terms ${terms} \\ --out ${meta.id}_exomiser_top.tsv --text ${meta.id}_exomiser.txt cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/locityper/main.nf b/modules/local/locityper/main.nf index 39b1d3847..be865d8eb 100644 --- a/modules/local/locityper/main.nf +++ b/modules/local/locityper/main.nf @@ -8,7 +8,7 @@ depth, insert size and error profile on its background region (--locityper_bg_region, or Locityper's own for GRCh38), locityper genotype picks two pangenome haplotypes at each locus of the database, - and bin/locityper_report.py, with the image's Python, writes one row per + and og locityper report (the static binary bin/og) writes one row per locus. Locityper's exit code goes to the table; a locus it did not finish reads "not called". @@ -52,7 +52,7 @@ process LOCITYPER { && locityper genotype -a ${bam} -d ${data}/db -p work/preproc -o work/gt -@ ${task.cpus}; } \\ > ${meta.id}_locityper.log 2>&1 || rc=\$? [ "\$rc" -eq 0 ] || { echo "WARNING: Locityper exited \$rc; last lines of its log:" >&2; tail -n 20 ${meta.id}_locityper.log >&2; } - locityper_report.py --gt-dir work/gt --loci ${data}/loci.bed --exit-code \$rc --out ${meta.id}_locityper.tsv + og locityper report --gt-dir work/gt --loci ${data}/loci.bed --exit-code \$rc --out ${meta.id}_locityper.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/mei/main.nf b/modules/local/mei/main.nf index 39cb9d014..892dfb509 100644 --- a/modules/local/mei/main.nf +++ b/modules/local/mei/main.nf @@ -10,11 +10,11 @@ MEI MEGAnE call_genotype_38 -only_ins on the BAM; 16 GB, the figure of the bash step (conf/base.config) MEI_REPORT in AnnotSV's image: with --annotsv_annotations, AnnotSV on - the insertions the calls table lists (bin/mei_report.py - --annotsv-input): their genes, the location in each, the + the insertions the calls table lists (og mei + annotsv-input): their genes, the location in each, the benign set's frequency of an insertion there, and AnnotSV's class, NA for every insertion (AnnotSV 3.5 scores deletions - and duplications only); then bin/mei_report.py: the calls + and duplications only); then og mei report: the calls with those columns ("." without AnnotSV's data), and the catalog as seen / not seen / not callable. 4 CPUs and 8 GB with AnnotSV's data, as step 05 gives AnnotSV, 1 and 2 GB @@ -175,7 +175,7 @@ process MEI_REPORT { exit 1 fi done - mei_report.py --sample ${p} --vcf ${vcf} --annotsv-input ${p}_mei_annotsv_input.vcf + og mei annotsv-input --sample ${p} --vcf ${vcf} --out ${p}_mei_annotsv_input.vcf AnnotSV -SVinputFile ${p}_mei_annotsv_input.vcf -outputFile ${p}_mei_annotsv.tsv -outputDir . \\ -genomeBuild GRCh38 -annotationMode both -annotationsDir ${annotations_dir} \\ -txFile ${annotations_dir}/mane_select_refseq.txt @@ -184,7 +184,7 @@ process MEI_REPORT { def annotsv_arg = annotations_dir ? "--annotsv ${p}_mei_annotsv.tsv" : '' """ ${annotsv} - mei_report.py --sample ${meta.id} --vcf ${vcf} --sites ${sites} --depth ${depth} ${annotsv_arg} \\ + og mei report --sample ${meta.id} --vcf ${vcf} --sites ${sites} --depth ${depth} ${annotsv_arg} \\ --out-calls ${meta.id}_mei.tsv --out-sites ${meta.id}_mei_sites.tsv cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/mocha/main.nf b/modules/local/mocha/main.nf index 11134b8ec..05bdf1b98 100644 --- a/modules/local/mocha/main.nf +++ b/modules/local/mocha/main.nf @@ -12,7 +12,7 @@ MOCHA_CALL the phased genotypes onto the prepared sites, MoChA (bcftools +mocha, the common germline CNVs as --cnp), and the per-chromosome site counts - MOCHA_REPORT bin/mocha_report.py: the calls above the floor + MOCHA_REPORT og mocha report (bin/og): the calls above the floor Equivalent to: scripts/51-mocha.sh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -197,7 +197,7 @@ process MOCHA_REPORT { script: """ - mocha_report.py --sample ${meta.id} --calls ${calls} --stats ${stats} --sites ${sites} --phasing beagle \\ + og mocha report --sample ${meta.id} --calls ${calls} --stats ${stats} --sites ${sites} --phasing beagle \\ --out-calls ${meta.id}_mocha.tsv --out-summary ${meta.id}_mocha_summary.tsv cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/viral/main.nf b/modules/local/viral/main.nf index ba3cf945f..d847ddbe1 100644 --- a/modules/local/viral/main.nf +++ b/modules/local/viral/main.nf @@ -8,7 +8,7 @@ with minimap2 to the viral genomes; samtools coverage of that BAM and of the sample's BAM (the human depth), and samtools depth -aa over every viral genome - VIRAL_REPORT bin/viral_report.py on those files: one row per viral + VIRAL_REPORT og viral report on those files: one row per viral genome, with inherited integrated HHV-6 called by Liu et al. 2020's depth threshold @@ -94,7 +94,7 @@ process VIRAL_REPORT { script: """ - viral_report.py --sample ${meta.id} --viral-fasta ${viral_ref} --viral-coverage ${viral_coverage} \\ + og viral report --sample ${meta.id} --viral-fasta ${viral_ref} --viral-coverage ${viral_coverage} \\ --viral-depth ${viral_depth} --human-coverage ${human_coverage} --out ${meta.id}_viral.tsv cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/vntyper/main.nf b/modules/local/vntyper/main.nf index e0793c40f..698702312 100644 --- a/modules/local/vntyper/main.nf +++ b/modules/local/vntyper/main.nf @@ -4,8 +4,8 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('vntyper' in --tools), for each sample with a BAM. One task, as scripts/52-vntyper.sh: VNtyper runs on the BAM (fast mode, GRCh38, from - /opt/vntyper, where its reference paths point), then bin/vntyper_report.py, - with the image's Python, writes the table: positive, possible, negative or + /opt/vntyper, where its reference paths point), then og vntyper report + (the static binary bin/og) writes the table: positive, possible, negative or not assessed, with the sensitivity at the sample's depth over the VNTR. VNtyper's exit code goes to the table; a run that stops reads "not assessed". The image's entrypoint takes only its own command words, so @@ -46,7 +46,7 @@ process VNTYPER { --fast-mode --reference-assembly hg38 ) > ${meta.id}_vntyper.log 2>&1 || rc=\$? [ "\$rc" -eq 0 ] || { echo "WARNING: VNtyper exited \$rc; last lines of its log:" >&2; tail -n 20 ${meta.id}_vntyper.log >&2; } mkdir -p run - vntyper_report.py --run-dir run --exit-code \$rc --log ${meta.id}_vntyper.log --out ${meta.id}_vntyper.tsv + og vntyper report --run-dir run --exit-code \$rc --log ${meta.id}_vntyper.log --out ${meta.id}_vntyper.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/y_str/main.nf b/modules/local/y_str/main.nf index 47398fce9..8b2d872d5 100644 --- a/modules/local/y_str/main.nf +++ b/modules/local/y_str/main.nf @@ -6,9 +6,9 @@ workflows/bam_analysis.nf passes the BAMs whose sex is male, the sex INDEXCOV has checked against the reads. - One task: bin/ystr_report.py writes HipSTR's region file from - assets/ystr_panel.tsv, HipSTR genotypes those chrY regions, and the - same script, with the image's Python, writes the table in + One task: og y str-bed (bin/og) writes HipSTR's region file from + assets/ystr_panel.tsv, HipSTR genotypes those chrY regions, and og y + str-report writes the table in FamilyTreeDNA's order and its CSV. Multi-copy markers and repeats longer than the reads can span are listed as not callable. @@ -39,11 +39,11 @@ process Y_STR { script: """ - ystr_report.py bed --panel ${panel} --out regions.bed + og y str-bed --panel ${panel} --out regions.bed HipSTR --bams ${bam} --fasta ${reference} --regions regions.bed --str-vcf ${meta.id}_y_str.vcf.gz \\ --bam-samps ${meta.id} --bam-libs ${meta.id} --haploid-chrs chrY --def-stutter-model --min-reads 5 \\ --max-str-len 150 --output-filters --log ${meta.id}_y_str.log - ystr_report.py report --panel ${panel} --vcf ${meta.id}_y_str.vcf.gz --log ${meta.id}_y_str.log \\ + og y str-report --panel ${panel} --vcf ${meta.id}_y_str.vcf.gz --log ${meta.id}_y_str.log \\ --tsv ${meta.id}_y_str.tsv --csv ${meta.id}_y_str.csv cat <<-END_VERSIONS > versions.yml diff --git a/scripts/45-y-str.sh b/scripts/45-y-str.sh index 6a1747b88..f3e7d479f 100755 --- a/scripts/45-y-str.sh +++ b/scripts/45-y-str.sh @@ -3,8 +3,8 @@ # Usage: ./scripts/45-y-str.sh # # Opt-in. HipSTR (HIPSTR_IMAGE) genotypes the Y-STR regions of -# assets/ystr_panel.tsv on chrY from the BAM, and bin/ystr_report.py, run with -# the image's own Python, turns its calls into repeat counts in +# assets/ystr_panel.tsv on chrY from the BAM, and og y str-report (the static +# binary bin/og), run in the same container, turns its calls into repeat counts in # FamilyTreeDNA's marker order (Y-12 to Y-111), with a confidence per marker. # Multi-copy markers (DYS385, DYS459, DYS464, YCAII, CDY, DYF395S1, DYS413) # and repeats longer than 150 bp reads can span are listed as not callable. @@ -55,15 +55,15 @@ mkdir -p "$OUTDIR" rm -f "$TSV" "${OUTDIR}/${SAMPLE}_y_str."{csv,vcf.gz,log} "${OUTDIR}/regions.bed" O=$(cpath "$OUTDIR") # One container: the region file from the panel, HipSTR, then the table. The -# flags and why: bin/ystr_report.py and docs/45-y-str.md. +# flags and why: rust/crates/og-reports/src/ystr.rs and docs/45-y-str.md. # shellcheck disable=SC2016 # $1 to $5 belong to the inner bash run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$HIPSTR_IMAGE" \ bash -euo pipefail -c ' - python3 /pgp-bin/ystr_report.py bed --panel /pgp-assets/ystr_panel.tsv --out "$1/regions.bed" + /pgp-bin/og y str-bed --panel /pgp-assets/ystr_panel.tsv --out "$1/regions.bed" HipSTR --bams "$2" --fasta "$3" --regions "$1/regions.bed" --str-vcf "$1/$4_y_str.vcf.gz" \ --bam-samps "$4" --bam-libs "$4" --haploid-chrs chrY --def-stutter-model --min-reads 5 \ --max-str-len 150 --output-filters --log "$1/$4_y_str.log" - python3 /pgp-bin/ystr_report.py report --panel /pgp-assets/ystr_panel.tsv --vcf "$1/$4_y_str.vcf.gz" \ + /pgp-bin/og y str-report --panel /pgp-assets/ystr_panel.tsv --vcf "$1/$4_y_str.vcf.gz" \ --log "$1/$4_y_str.log" --tsv "$1/$4_y_str.tsv" --csv "$1/$4_y_str.csv"' \ _ "$O" "$(cpath "$BAM")" "$REF_FASTA_C" "$SAMPLE" rm -f "${OUTDIR}/regions.bed" diff --git a/scripts/47-viral.sh b/scripts/47-viral.sh index d6099b8a6..830cf816f 100755 --- a/scripts/47-viral.sh +++ b/scripts/47-viral.sh @@ -10,7 +10,7 @@ # Illumina's phiX control), and reports per genome the reads, the breadth and # the depth against the human autosomal depth. Inherited chromosomally # integrated HHV-6 (in about 1 in 100 people) shows as the whole HHV-6 genome at -# about half the human depth. bin/viral_report.py applies the rules; it is the +# about half the human depth. og viral report applies the rules; it is the # same code as the VIRAL_REPORT Nextflow process. Not a test for an infection. # # Input: aligned/_sorted.bam with its .bai (ALIGN_DIR as for the other @@ -77,9 +77,9 @@ run_in --cpus 2 --memory 4g "$SAMTOOLS_IMAGE" bash -euo pipefail -c ' samtools coverage --ff UNMAP,SECONDARY,QCFAIL "$1_viral_reads.bam" > "$1_viral_coverage.tsv" samtools depth -aa -G UNMAP,SECONDARY,QCFAIL "$1_viral_reads.bam" > "$1_viral_depth.tsv"' _ "$O" -echo "[4/4] Calls (bin/viral_report.py)..." +echo "[4/4] Calls (og viral report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/viral_report.py --sample "$SAMPLE" --viral-fasta "$V" \ + /pgp-bin/og viral report --sample "$SAMPLE" --viral-fasta "$V" \ --viral-coverage "${O}_viral_coverage.tsv" --viral-depth "${O}_viral_depth.tsv" \ --human-coverage "${O}_human_coverage.tsv" --out "$(cpath "$TABLE")" rm -f "${P}_unmapped.fq" "${P}_viral.sam" "${P}_viral_depth.tsv" diff --git a/scripts/48-exomiser.sh b/scripts/48-exomiser.sh index 5ab4248c3..969811d00 100755 --- a/scripts/48-exomiser.sh +++ b/scripts/48-exomiser.sh @@ -7,7 +7,7 @@ # says how to pick them). Exomiser (EXOMISER_IMAGE) scores each rare coding or # splice variant of the VCF by how damaging it looks and each gene by how well # its known diseases and animal models match the terms, and ranks the genes -# under each mode of inheritance. bin/exomiser_report.py checks the terms +# under each mode of inheritance. og exomiser (bin/og) checks the terms # against the HPO release of Exomiser's data, writes the analysis (Exomiser's # exome preset) and reads the result: the top genes, their variants, scores, # mode of inheritance and Exomiser's ACMG classification, in words too. The @@ -97,9 +97,9 @@ echo "HPO ontology: ${OBO:-not found in the phenotype data; the terms are checke mkdir -p "$OUTDIR" rm -f "${P}"* "${OUTDIR}/${SAMPLE}_analysis.yml" O=$(cpath "$OUTDIR") -echo "[1/3] HPO terms and the analysis (bin/exomiser_report.py)..." +echo "[1/3] HPO terms and the analysis (og exomiser prepare)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/exomiser_report.py prepare --hpo "$HPO" ${OBO:+--obo "$(cpath "$OBO")"} --sample "$SAMPLE" \ + /pgp-bin/og exomiser prepare --hpo "$HPO" ${OBO:+--obo "$(cpath "$OBO")"} --sample "$SAMPLE" \ --vcf "$(cpath "$VCF")" --assembly "$ASSEMBLY" --out-dir "$O" --analysis "${O}/${SAMPLE}_analysis.yml" \ --terms "$(cpath "${P}_terms.tsv")" ${ARGS[@]+"${ARGS[@]}"} echo "[2/3] Exomiser..." @@ -113,7 +113,7 @@ run_in --cpus "$THREADS" --memory "${MEM}g" --entrypoint /usr/local/bin/exomiser [ -f "${P}.variants.tsv" ] || { echo "ERROR: Exomiser wrote no ${P}.variants.tsv" >&2; exit 1; } echo "[3/3] The top genes in words..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$PYTHON_IMAGE" \ - python3 /pgp-bin/exomiser_report.py report --sample "$SAMPLE" --variants "${O}/${SAMPLE}_exomiser.variants.tsv" \ + /pgp-bin/og exomiser report --sample "$SAMPLE" --variants "${O}/${SAMPLE}_exomiser.variants.tsv" \ --terms "$(cpath "${P}_terms.tsv")" --out "$(cpath "${P}_top.tsv")" --text "$(cpath "${P}.txt")" echo "" diff --git a/scripts/50-mei.sh b/scripts/50-mei.sh index 6799101a9..b13392ddb 100755 --- a/scripts/50-mei.sh +++ b/scripts/50-mei.sh @@ -5,7 +5,7 @@ # Opt-in. MEGAnE (Kojima et al. 2023, Nat Genet, PMID 37169872) finds the # copies of Alu, L1, SVA and HERV elements a genome carries and the reference # does not, from read pairs with one end in the element and from reads that -# cross its edge, and genotypes each insertion. bin/mei_report.py then lists +# cross its edge, and genotypes each insertion. og mei report then lists # the insertions it calls and reads a short catalog of insertions known to # cause disease (assets/mei_sites.tsv: the FKTN SVA of Fukuyama muscular # dystrophy, the TAF1 SVA of X-linked dystonia-parkinsonism, the BBS1 SVA of @@ -148,7 +148,7 @@ ANNOTSV_ARGS=() if $ANNOTSV; then echo "[3/4] AnnotSV on the insertions..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/mei_report.py --sample "$SAMPLE" --vcf "${O}_megane.vcf" --annotsv-input "${O}_mei_annotsv_input.vcf" + /pgp-bin/og mei annotsv-input --sample "$SAMPLE" --vcf "${O}_megane.vcf" --out "${O}_mei_annotsv_input.vcf" AC=$(cpath "$ANNOTATIONS_DIR") run_in --rw "$ANNOTATIONS_DIR" --cpus 4 --memory 8g "$ANNOTSV_IMAGE" \ AnnotSV -SVinputFile "${O}_mei_annotsv_input.vcf" -outputFile "${O}_mei_annotsv.tsv" -genomeBuild GRCh38 \ @@ -160,9 +160,9 @@ else echo "[3/4] AnnotSV: skipped (no annotation data)." fi -echo "[4/4] Calls and the catalog (bin/mei_report.py)..." +echo "[4/4] Calls and the catalog (og mei report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${SITES}:/pgp-sites.tsv:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/mei_report.py --sample "$SAMPLE" --vcf "${O}_megane.vcf" --sites /pgp-sites.tsv \ + /pgp-bin/og mei report --sample "$SAMPLE" --vcf "${O}_megane.vcf" --sites /pgp-sites.tsv \ --depth "${O}_mei_depth.tsv" ${ANNOTSV_ARGS[@]+"${ANNOTSV_ARGS[@]}"} --out-calls "${O}_mei.tsv" --out-sites "${O}_mei_sites.tsv" rm -rf "$WORK" "${P}_mei_mainchr.txt" diff --git a/scripts/51-mocha.sh b/scripts/51-mocha.sh index 1661df8a1..3063c8cbc 100755 --- a/scripts/51-mocha.sh +++ b/scripts/51-mocha.sh @@ -7,7 +7,7 @@ # (freeseek/mocha, a bcftools plugin) finds one as an allele balance that leans # the same way along a phased haplotype. The VCF's heterozygous SNVs are phased # with Beagle against the 1000 Genomes high-coverage panel (setup.sh -# --mocha-data), MoChA calls on chr1 to chr22, and bin/mocha_report.py keeps the +# --mocha-data), MoChA calls on chr1 to chr22, and og mocha report keeps the # calls MoChA's UK Biobank filter keeps whose cell fraction is at or above the # floor it prints for the sample's depth. Mosaic loss of Y and X is step 33's. # The same commands as the MOCHA_* Nextflow processes. Research result, not a @@ -142,9 +142,9 @@ run_in --cpus 2 --memory 8g "$MOCHA_IMAGE" bash -euo pipefail -c ' _ "$WC" "$PHASING" "$O" "$MHC" "$KIR" "$SITES_AWK" "$CNP" echo " $(cat "${W}/mocha_version.txt")" -echo "[4/4] Calls above the floor (bin/mocha_report.py)..." +echo "[4/4] Calls above the floor (og mocha report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/mocha_report.py --sample "$SAMPLE" --calls "${O}_mocha_calls_raw.tsv" --stats "${O}_mocha_stats.tsv" \ + /pgp-bin/og mocha report --sample "$SAMPLE" --calls "${O}_mocha_calls_raw.tsv" --stats "${O}_mocha_stats.tsv" \ --sites "${O}_mocha_sites.tsv" --phasing "$PHASING" --out-calls "${O}_mocha.tsv" --out-summary "${O}_mocha_summary.tsv" cp "${W}/mocha.log" "${P}_mocha.log" rm -rf "$W" diff --git a/scripts/52-vntyper.sh b/scripts/52-vntyper.sh index f5826ad13..99fada917 100755 --- a/scripts/52-vntyper.sh +++ b/scripts/52-vntyper.sh @@ -6,7 +6,7 @@ # inside the MUC1 coding VNTR, a repeat of 60 bp units, most often one extra C # (dupC). Short reads cannot be placed inside the repeat, so DeepVariant and # ExpansionHunter do not see it. VNtyper (VNTYPER_IMAGE, its reference data -# inside) genotypes it with k-mers; bin/vntyper_report.py, the same code as +# inside) genotypes it with k-mers; og vntyper report, the same code as # the VNTYPER Nextflow process, turns its result into positive, possible, # negative or not assessed, with the share of known positives VNtyper found # at the sample's depth over the VNTR. A research call, not a diagnosis. @@ -54,9 +54,9 @@ run_in --cpus "$THREADS" --memory 8g --entrypoint "" "$VNTYPER_IMAGE" bash -c ' _ "$(cpath "$BAM")" "$(cpath "$RUN")" "$SAMPLE" "$THREADS" "$(cpath "$LOG")" || rc=$? [ "$rc" -eq 0 ] || { echo "WARNING: VNtyper exited ${rc}; last lines of ${LOG}:" >&2; tail -n 20 "$LOG" >&2 || true; } -echo "[2/2] The reading (bin/vntyper_report.py)..." +echo "[2/2] The reading (og vntyper report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g --entrypoint "" "$VNTYPER_IMAGE" \ - python3 /pgp-bin/vntyper_report.py --run-dir "$(cpath "$RUN")" --exit-code "$rc" --log "$(cpath "$LOG")" --out "$(cpath "$TABLE")" + /pgp-bin/og vntyper report --run-dir "$(cpath "$RUN")" --exit-code "$rc" --log "$(cpath "$LOG")" --out "$(cpath "$TABLE")" [ -s "$TABLE" ] || { echo "ERROR: no table written (${TABLE})" >&2; exit 1; } echo "" diff --git a/scripts/53-locityper.sh b/scripts/53-locityper.sh index dc32f91c4..753ed2afd 100755 --- a/scripts/53-locityper.sh +++ b/scripts/53-locityper.sh @@ -8,7 +8,7 @@ # Nat Genet, PMID 41107551; MIT licence) genotypes a whole locus at once: it # picks the two haplotypes of a pangenome panel (here the HPRC v2.1 panel of # 232 people) that best explain the reads there and their depth. Its call is -# shown beside the primary caller's, never as the call. bin/locityper_report.py, +# shown beside the primary caller's, never as the call. og locityper report, # the same code as the LOCITYPER Nextflow process, writes one row per locus of # assets/locityper_loci.bed. # @@ -74,9 +74,9 @@ run_in --cpus "$THREADS" --memory 16g "$LOCITYPER_IMAGE" bash -c ' "${LOCITYPER_BG_REGION:-}" "$(cpath "$LOG")" || rc=$? [ "$rc" -eq 0 ] || { echo "WARNING: Locityper exited ${rc}; last lines of ${LOG}:" >&2; tail -n 20 "$LOG" >&2 || true; } -echo "[2/2] The table (bin/locityper_report.py)..." +echo "[2/2] The table (og locityper report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 1g "$LOCITYPER_IMAGE" \ - python3 /pgp-bin/locityper_report.py --gt-dir "$(cpath "${WORK}/gt")" --loci "$(cpath "${DATA}/loci.bed")" \ + /pgp-bin/og locityper report --gt-dir "$(cpath "${WORK}/gt")" --loci "$(cpath "${DATA}/loci.bed")" \ --exit-code "$rc" --out "$(cpath "$TABLE")" [ -s "$TABLE" ] || { echo "ERROR: no table written (${TABLE})" >&2; exit 1; } diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 2b3f5054e..2c4f40b86 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -66,7 +66,7 @@ callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv asset callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/export_consumer.py scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -ci-e2e-2-locityper assets/locityper_loci.bed bin/locityper_report.py scripts/53-locityper.sh scripts/setup.sh +ci-e2e-2-locityper assets/locityper_loci.bed bin/og scripts/53-locityper.sh scripts/setup.sh clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/clip_check.py bin/comphet_phase.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh clinical-filter-mane-and-quality-2-vep-norm scripts/13-vep-annotation.sh scripts/ci/fake-docker/lib.sh clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/comphet_phase.py bin/og modules/local/clinical_filter/* modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -86,7 +86,7 @@ couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_ couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/copy_number.py bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/exomiser_report.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/rfc1_motif.py bin/strchive_convert.py scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/cyp2d6_depth_check.py bin/family_report.py bin/family_segments.py bin/hba_copy_number.py bin/ibd_report.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/family_segments.py bin/ibd_report.py bin/mosaic_sex_loss.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -108,9 +108,9 @@ hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv ass ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/family_segments.py bin/ibd_report.py scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/mei_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh +mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/mocha_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/mosaic_sex_loss.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh @@ -173,8 +173,8 @@ vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv as vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/og bin/pgx_outside_calls.py bin/pgx_parse.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh -viral-reads-1 bin/viral_report.py modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/vntyper_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py bin/y_private_snps.py bin/y_tree_place.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/ystr_report.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/y_tree_place.py bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh diff --git a/tests/e2e/ci-e2e-2-locityper.sh b/tests/e2e/ci-e2e-2-locityper.sh index 89185a9f3..e3295e4b6 100755 --- a/tests/e2e/ci-e2e-2-locityper.sh +++ b/tests/e2e/ci-e2e-2-locityper.sh @@ -2,7 +2,7 @@ # Step 53 (Locityper) and the database it reads, the shipped scripts as a # user runs them: setup.sh --locityper-data builds the database, then # scripts/53-locityper.sh genotypes HG002 and writes the table -# bin/locityper_report.py makes. The image's smoke test repeats their +# og locityper report makes. The image's smoke test repeats their # commands; this case runs the scripts. # # The fixture holds reads over the MHC but none over FCGR or KIR, so the @@ -86,7 +86,7 @@ check "and builds nothing again" grep -q "already built" "${CASE_TMP}/again.log" run53 "${CASE_TMP}/step.log" check_eq "scripts/53-locityper.sh exits 0" "$STEP_RC" 0 TABLE="${S}/locityper/${SAMPLE}_locityper.tsv" -check_eq "the table's columns are those of bin/locityper_report.py" "$(head -n 1 "$TABLE" 2>/dev/null | tr '\t' ' ')" \ +check_eq "the table's columns are those of og locityper report" "$(head -n 1 "$TABLE" 2>/dev/null | tr '\t' ' ')" \ "locus region status genotype haplotype_1 haplotype_2 quality total_reads unexplained_reads weighted_distance warnings filter reason" check_eq "one row per locus of the database" "$(awk -F'\t' 'NR > 1 {print $1}' "$TABLE" 2>/dev/null | paste -sd' ' -)" "HLA-DQB1" l=HLA-DQB1 diff --git a/tests/e2e/exomiser-1.sh b/tests/e2e/exomiser-1.sh index 807a4ebbc..34a1379a9 100755 --- a/tests/e2e/exomiser-1.sh +++ b/tests/e2e/exomiser-1.sh @@ -147,7 +147,7 @@ awk -F'\t' 'NR == 1 {for (i = 1; i <= NF; i++) if ($i == "MOI") m = i} {o = ""; for (i = 1; i <= NF; i++) if (i != m) o = o (o == "" ? "" : "\t") $i; print o}' \ "${CASE_TMP}/bash-out/${S}_exomiser.variants.tsv" > "${CASE_TMP}/cut.variants.tsv" check "the cut file still has rows" test "$(wc -l < "${CASE_TMP}/cut.variants.tsv")" -gt 1 -OUT=$(python3 -I "${REPO}/bin/exomiser_report.py" report --sample "$S" --variants "${CASE_TMP}/cut.variants.tsv" \ +OUT=$("${REPO}/bin/og" exomiser report --sample "$S" --variants "${CASE_TMP}/cut.variants.tsv" \ --terms "${CASE_TMP}/bash-out/${S}_exomiser_terms.tsv" --out "${CASE_TMP}/cut.tsv" --text "${CASE_TMP}/cut.txt" 2>&1) RC=$? check "a variants file without MOI: the report stops (exit ${RC})" test "$RC" -ne 0 diff --git a/tests/e2e/mei-1.sh b/tests/e2e/mei-1.sh index 3d5a608c0..923194b20 100755 --- a/tests/e2e/mei-1.sh +++ b/tests/e2e/mei-1.sh @@ -21,7 +21,7 @@ # tests/fixtures/annotsv/class45.sh plus one made-up gene, FIXCHR20, over # all of chr20 (two exons at its ends): every call comes back in # FIXCHR20, with its location there, and with AnnotSV's class of an -# insertion, NA. Red control: the same VCF through bin/mei_report.py +# insertion, NA. Red control: the same VCF through og mei report # without AnnotSV's table fails those checks; # - the Nextflow pipeline (--tools mei, with --annotsv_annotations) on the # same BAM writes the same calls table as the bash step, and MEI's peak @@ -154,7 +154,7 @@ echo "AnnotSV columns (calls/in FIXCHR20/with its location/class NA): $(annotsv_ check_eq "every call is in FIXCHR20, with its location there and AnnotSV's class NA" \ "$(annotsv_cols "${M}_mei.tsv")" "${N}/${N}/${N}/${N}" # Red control: the same VCF without AnnotSV's table. -python3 -I "${REPO}/bin/mei_report.py" --sample "$H" --vcf "${M}_megane.vcf" --sites "${REPO}/assets/mei_sites.tsv" \ +"${REPO}/bin/og" mei report --sample "$H" --vcf "${M}_megane.vcf" --sites "${REPO}/assets/mei_sites.tsv" \ --depth "${M}_mei_depth.tsv" --out-calls "${T}/red_mei.tsv" --out-sites "${T}/red_sites.tsv" > /dev/null check "red control: without AnnotSV's table the same check fails" \ test "$(annotsv_cols "${T}/red_mei.tsv")" != "${N}/${N}/${N}/${N}" -a "$(annotsv_cols "${T}/red_mei.tsv")" = "${N}/0/0/0" diff --git a/tests/e2e/y-str-1.sh b/tests/e2e/y-str-1.sh index efabd18a4..97440a8f7 100755 --- a/tests/e2e/y-str-1.sh +++ b/tests/e2e/y-str-1.sh @@ -36,7 +36,7 @@ dk() { local img=$1; shift; docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } # --- the panel against the reference --------------------------------------------------- -python3 "${REPO}/bin/ystr_report.py" bed --panel "$PANEL" --out "${T}/regions.bed" +"${REPO}/bin/og" y str-bed --panel "$PANEL" --out "${T}/regions.bed" check_eq "HipSTR regions in the panel" "$(grep -c . "${T}/regions.bed")" 86 awk -F'\t' '{print $1 ":" $2 "-" $3}' "${T}/regions.bed" > "${T}/regions.txt" dk "$SAMTOOLS_IMAGE" samtools faidx -r "${T}/regions.txt" "$REF" > "${T}/regions.fa" @@ -133,7 +133,7 @@ NOREC=$(awk -F'\t' 'NR > 1 && $10 ~ /no record/' "$OUT" 2>/dev/null | wc -l | tr echo " regions without a record: ${NOREC}" if [ "$NOREC" -gt 0 ]; then : > "${T}/empty.log" - python3 "${REPO}/bin/ystr_report.py" report --panel "$PANEL" --vcf "${G}/${H}/y_str/${H}_y_str.vcf.gz" \ + "${REPO}/bin/og" y str-report --panel "$PANEL" --vcf "${G}/${H}/y_str/${H}_y_str.vcf.gz" \ --log "${T}/empty.log" --tsv "${T}/nolog.tsv" --csv "${T}/nolog.csv" > /dev/null check_eq "control: with an empty log, those rows name no reason" \ "$(awk -F'\t' 'NR > 1 && $10 ~ /its log gives no reason/' "${T}/nolog.tsv" 2>/dev/null | wc -l | tr -d ' ')" "$NOREC" From 34b9c10c5bddf814fd33d4b25a153cc6a36aea8f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:16 +0100 Subject: [PATCH 24/53] test(reports): the step report tests drive og The seven tests run og (OG), skip without it, and fail without it when OG_REQUIRED is set, as Guard's rust job does. The demo builder writes the seven tables itself, byte for byte what it wrote before. Bead: pgp-bs9u.17 --- .github/workflows/guard.yml | 7 +- tests/demo/make_demo_sample.py | 295 +++++++++++++++++---------------- tests/test_exomiser_report.py | 22 ++- tests/test_locityper_report.py | 221 +++++++++++++----------- tests/test_mei_report.py | 47 +++--- tests/test_mocha_report.py | 220 ++++++++++++------------ tests/test_report_charts.py | 4 +- tests/test_viral_report.py | 97 ++++++----- tests/test_vntyper_report.py | 107 +++++++----- tests/test_ystr_report.py | 46 +++-- 10 files changed, 594 insertions(+), 472 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index feccd7079..43937b94f 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -223,7 +223,7 @@ jobs: ./og.sh check --required # The tests of the og filters steps 04, 05 and 07 run, and og chip's - # (step 49): unit-tests has + # (step 49) and the opt-in step reports' (45, 47, 48, 50 to 53): unit-tests has # no og, so there they skip; here OG_REQUIRED turns a skip into a # failure. The step 05 tests run the real step with bin/og. - name: The og filter tests and step 05 with og @@ -235,7 +235,10 @@ jobs: fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ - tests/test_chip_report.py; do + tests/test_chip_report.py \ + tests/test_vntyper_report.py tests/test_locityper_report.py tests/test_mocha_report.py \ + tests/test_mei_report.py tests/test_viral_report.py tests/test_ystr_report.py \ + tests/test_exomiser_report.py; do rc=0 echo "::group::${t}" case "$t" in diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 9def678d8..fc94a997a 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -25,13 +25,13 @@ - step 33's mosaic rows come from bin/mosaic_sex_loss.py, step 26b's painting from bin/ancestry_fine_report.py, step 35b's table from bin/paraphase_report.py, step 39's from bin/couple_report.py and step 42's - from bin/traits_report.py, each fed made-up inputs; step 45's rows come - from bin/ystr_report.py's marker_row, step 46's from bin/lpa_kiv2.py's - assess, step 35's HBA1/HBA2 table from bin/hba_copy_number.py's assess, - step 49's depth rows from bin/chip_report.py's detection_limit, step 50's - tables from bin/mei_report.py on a made-up VCF and depth, and - step 47's from bin/viral_report.py's build_rows, with made-up counts; steps 02, 40 and 41's tables are written here in their writers' - columns. + from bin/traits_report.py, each fed made-up inputs; step 46's rows come + from bin/lpa_kiv2.py's assess, step 35's HBA1/HBA2 table from + bin/hba_copy_number.py's assess, step 49's depth rows from + bin/chip_report.py's detection_limit, with made-up counts; the tables of + steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written here, in the + columns their writers use (og for 45 to 53), with the values those + writers give for the made-up inputs named in each function. DEMO-001 carries every section, so every card is in the pictures: also the Paraphase card (a PacBio HiFi step, though the invented reads are short) and @@ -1012,39 +1012,53 @@ def bloodtype(d): def y_str(d, rng): - """Step 45: one row per marker of assets/ystr_panel.tsv from bin/ystr_report.py's - marker_row, so the multi-copy, absent and too-long markers read not callable - as in a real run; every other marker gets a made-up count at confidence high.""" - import ystr_report - rows = [] - for r in ystr_report.read_panel(os.path.join(REPO, "assets", "ystr_panel.tsv")): - row = ystr_report.marker_row(r, {}, 150) - if row["confidence"] == "no call": + """Step 45: one row per marker of assets/ystr_panel.tsv in the columns of + og y str-report. The multi-copy, absent and too-long markers (a repeat + with its 10-base flanks longer than 150 bp reads) read not callable, with + og's notes, as in a real run; every other marker gets a made-up count at + confidence high.""" + with open(os.path.join(REPO, "assets", "ystr_panel.tsv"), encoding="utf-8") as f: + lines = [x.rstrip("\n").split("\t") for x in f if x.strip() and not x.startswith("#")] + cols = ["order", "panel", "marker", "count", "confidence", "reads", "stutter_reads", "flank_indel_reads", "q", + "note"] + out = ["\t".join(cols) + "\n"] + for c in lines[1:]: + r = dict(zip(lines[0], c)) + row = dict.fromkeys(cols, "") + row.update(order=r["order"], panel=r["panel"], marker=r["marker"], confidence="not callable") + spans = [[int(x) for x in part.split("/")[0].split(":")[1].split("-")] + for part in ([] if r["regions"] == "." else r["regions"].split(";"))] + if r["kind"] == "span": # one region over the whole locus + spans = [[spans[0][0], spans[-1][1]]] + longest = max((e - b + 1 for b, e in spans), default=0) + if r["kind"] == "multicopy": + row["note"] = "not callable from short reads: a multi-copy marker in duplicated Y sequence" + elif r["kind"] == "absent": + row["note"] = "not callable: no region for it in the HipSTR marker set" + elif longest + 2 * 10 > 150: + row["note"] = (f"not callable from short reads: a {longest} bp repeat is longer than 150 bp reads can " + f"span with 10 bases on each side") + else: row.update(count=str(rng.randint(10, 30)), confidence="high", reads=str(rng.randint(18, 34)), stutter_reads="0", flank_indel_reads="0", q="0.99", note=".") - rows.append(row) - put(os.path.join(d, "y_str", f"{SAMPLE}_y_str.tsv"), - "\t".join(ystr_report.TSV_COLUMNS) + "\n" - + "".join("\t".join(row[k] for k in ystr_report.TSV_COLUMNS) + "\n" for row in rows)) + out.append("\t".join(row[k] for k in cols) + "\n") + put(os.path.join(d, "y_str", f"{SAMPLE}_y_str.tsv"), "".join(out)) def viral(d): - """Step 47: bin/viral_report.py's build_rows on made-up coverage: the lab's - phiX control, a few Epstein-Barr reads, no HHV-6, at a human depth of 31.""" - import csv - import viral_report as vr - genomes = [("NC_001664.4", "Human herpesvirus 6A, complete genome", 159378, 0, (0.0, 0.0, 0)), - ("NC_000898.1", "Human herpesvirus 6B, complete genome", 162114, 0, (0.0, 0.0, 0)), - ("NC_007605.1", "Human gammaherpesvirus 4, complete genome", 171823, 7, (0.0041, 0.006, 0)), - ("NC_001422.1", "Escherichia phage phiX174, complete genome", 5386, 1412, (1.0, 38.2, 37))] - rows = vr.build_rows([(a, t) for a, t, *_ in genomes], - {a: {"reads": n, "length": ln} for a, _, ln, n, _ in genomes}, - {a: dep for a, *_, dep in genomes}, 31.0) - os.makedirs(os.path.join(d, "viral"), exist_ok=True) - with open(os.path.join(d, "viral", f"{SAMPLE}_viral.tsv"), "w", encoding="utf-8", newline="") as f: - w = csv.DictWriter(f, fieldnames=vr.HEADER, delimiter="\t", lineterminator="\n") - w.writeheader() - w.writerows(rows) + """Step 47: what og viral report writes for made-up coverage: no HHV-6, + 7 Epstein-Barr reads (under the 10 reads of a call), and the lab's phiX + control, at a human depth of 31.""" + head = ("virus\taccession\tlength\treads\tbreadth\tmean_depth\tmode_depth\thuman_depth\trelative_depth" + "\tcopies_per_cell\tcall\tnote\n") + rows = [("HHV-6A", "NC_001664.4", "159378", "0", "0.0000", "0.000", "0", "0.000", "0.0000", "not found", ""), + ("HHV-6B", "NC_000898.1", "162114", "0", "0.0000", "0.000", "0", "0.000", "0.0000", "not found", ""), + ("Human gammaherpesvirus 4", "NC_007605.1", "171823", "7", "0.0041", "0.006", "0", "0.000", "0.0004", + "not found", ""), + ("Escherichia phage phiX174", "NC_001422.1", "5386", "1412", "1.0000", "38.200", "37", "1.194", "2.4645", + "lab control (phiX)", "Illumina adds phiX to the run as a control: these reads were added in the lab.")] + put(os.path.join(d, "viral", f"{SAMPLE}_viral.tsv"), + head + "".join("\t".join(r[:7] + ("31.00",) + r[7:]) + "\n" for r in rows)) def lpa(d, rng): @@ -1153,43 +1167,42 @@ def ancient(d, rng): def exomiser(d): - """Step 48: bin/exomiser_report.py's top_rows and words on made-up Exomiser - rows for two searched symptoms: two weak candidates of uncertain - significance, the shape of a search that finds nothing to act on. DEMO-003's - search also ranks the planted gene (PLANTED) third, at the planted hit: a - symptom search can rank the gene of a result his page keeps closed.""" - import exomiser_report as er - terms = [{"hpo_id": "HP:0001250", "label": "Seizure", "given_as": "HP:0001250"}, - {"hpo_id": "HP:0002315", "label": "Headache", "given_as": "HP:0002315"}] - - def row(rank, gene, moi, scores, contig, start, ref, alt, gt, cls, hgvs, disease_id, disease): - r = dict.fromkeys(er.NEED, "") - r.update({"#RANK": str(rank), "GENE_SYMBOL": gene, "MOI": moi, "P-VALUE": scores[3], - "EXOMISER_GENE_COMBINED_SCORE": scores[0], "EXOMISER_GENE_PHENO_SCORE": scores[1], - "EXOMISER_GENE_VARIANT_SCORE": scores[2], "CONTRIBUTING_VARIANT": "1", "CONTIG": contig, - "START": str(start), "REF": ref, "ALT": alt, "GENOTYPE": gt, "FUNCTIONAL_CLASS": cls, - "HGVS": hgvs, "EXOMISER_ACMG_CLASSIFICATION": "UNCERTAIN_SIGNIFICANCE", - "EXOMISER_ACMG_DISEASE_ID": disease_id, "EXOMISER_ACMG_DISEASE_NAME": disease, - "CLINVAR_PRIMARY_INTERPRETATION": "NOT_PROVIDED", "MAX_FREQ_SOURCE": "GNOMAD_E_NFE", - "MAX_FREQ": "0.0123"}) - return r - rows = er.top_rows([ - row(1, "SCN1A", "AD", ("0.412", "0.788", "0.534", "0.021"), "2", 165991230, "C", "T", "0/1", - "missense_variant", "SCN1A:ENST00000674923.1:c.3199G>A:p.(Ala1067Thr)", "OMIM:607208", - "Dravet syndrome"), - row(2, "CACNA1A", "AD", ("0.297", "0.701", "0.421", "0.048"), "19", 13299371, "G", "A", "0/1", - "missense_variant", "CACNA1A:ENST00000360228.11:c.2134C>T:p.(Arg712Cys)", "OMIM:141500", - "Migraine, familial hemiplegic, 1")] - + ([row(3, PLANTED["gene"], "AD", ("0.205", "0.512", "0.399", "0.061"), PLANTED["hit"][0].removeprefix("chr"), - PLANTED["hit"][1], PLANTED["hit"][2], PLANTED["hit"][3], "0/1", "missense_variant", - f"{PLANTED['gene']}:ENST00000262186.10:c.1C>T:p.(Met1?)", "OMIM:613688", "Long QT syndrome 2")] - if has("act") else []), 10) + """Step 48: what og exomiser report writes for made-up Exomiser rows for + two searched symptoms: two weak candidates of uncertain significance, the + shape of a search that finds nothing to act on. DEMO-003's search also + ranks the planted gene (PLANTED) third, at the planted hit: a symptom + search can rank the gene of a result his page keeps closed.""" + terms = [("HP:0001250", "Seizure"), ("HP:0002315", "Headache")] + # rank, gene, (exomiser, phenotype, variant, p), variant, HGVS, disease + ranks = [(1, "SCN1A", ("0.412", "0.788", "0.534", "0.021"), "2:165991230 C>T", + "SCN1A:ENST00000674923.1:c.3199G>A:p.(Ala1067Thr)", "Dravet syndrome (OMIM:607208)"), + (2, "CACNA1A", ("0.297", "0.701", "0.421", "0.048"), "19:13299371 G>A", + "CACNA1A:ENST00000360228.11:c.2134C>T:p.(Arg712Cys)", "Migraine, familial hemiplegic, 1 (OMIM:141500)")] + if has("act"): + c, pos, ref, alt = PLANTED["hit"] + ranks.append((3, PLANTED["gene"], ("0.205", "0.512", "0.399", "0.061"), + f"{c.removeprefix('chr')}:{pos} {ref}>{alt}", + f"{PLANTED['gene']}:ENST00000262186.10:c.1C>T:p.(Met1?)", "Long QT syndrome 2 (OMIM:613688)")) + freq = "GNOMAD_E_NFE 0.0123%" + top = ["rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant_score\tp_value\tvariant\tgenotype" + "\tconsequence\thgvs\tacmg\tacmg_evidence\tdisease\tclinvar\tmax_frequency\n"] + words = ["Symptoms searched: " + ", ".join(f"{label} ({t})" for t, label in terms) + "."] + for rank, gene, sc, var, hgvs, disease in ranks: + top.append("\t".join([str(rank), gene, "AD", *sc, var, "0/1", "missense_variant", hgvs, + "UNCERTAIN_SIGNIFICANCE", ".", disease, ".", freq]) + "\n") + words.append(f"{rank}. {gene} (autosomal dominant): Exomiser score {sc[0]} (symptom match {sc[1]}, variant " + f"{sc[2]}). {var} heterozygous, missense_variant {hgvs}, ACMG uncertain significance, linked " + f"disease {disease}, highest population frequency {freq}.") + words.append("A ranking to bring to a clinical geneticist, not a diagnosis. A high score means a rare variant, " + "predicted to be damaging, in a gene linked to symptoms like the ones given; it does not show that " + "the variant causes them. A low score or an empty list does not rule a genetic cause out: this " + "search reads coding and splice variants against the terms given, and not structural variants, " + "repeat expansions, mitochondrial heteroplasmy or variants outside genes.") out = os.path.join(d, "exomiser") put(os.path.join(out, f"{SAMPLE}_exomiser_terms.tsv"), - "\t".join(er.TERMS_HEADER) + "\n" + "".join("\t".join(t[k] for k in er.TERMS_HEADER) + "\n" for t in terms)) - put(os.path.join(out, f"{SAMPLE}_exomiser_top.tsv"), - "\t".join(er.OUT_HEADER) + "\n" + "".join("\t".join(r[k] for k in er.OUT_HEADER) + "\n" for r in rows)) - put(os.path.join(out, f"{SAMPLE}_exomiser.txt"), "\n".join(er.words(rows, terms)) + "\n") + "hpo_id\tlabel\tgiven_as\n" + "".join(f"{t}\t{label}\t{t}\n" for t, label in terms)) + put(os.path.join(out, f"{SAMPLE}_exomiser_top.tsv"), "".join(top)) + put(os.path.join(out, f"{SAMPLE}_exomiser.txt"), "\n".join(words) + "\n") def traits(d): @@ -1209,95 +1222,83 @@ def traits(d): def mocha(d): - """Step 51: bin/mocha_report.py's report on made-up MoChA tables for a - 30x genome with no mosaic call: chr1 to chr22 read, the sample filter met.""" - import mocha_report as mr - stats = [{"sample_id": SAMPLE, "call_rate": "0.9981", "cov_median": "31", "baf_auto": "0.0124", - "n_hets": "2100000"}] - sites = [(f"chr{i}", 10001, 10001 + 100_000_000, 120_000, 95_000, 94_000) for i in range(1, 23)] - calls = [] + """Step 51: what og mocha report writes for made-up MoChA tables of a 30x + genome (chr1 to chr22 read, 95,000 hets per chromosome of 100 Mb, the + sample filter met), with no mosaic call; DEMO-002's carries a loss of 20q + in about 12% of the blood cells, which MoChA's filters keep and the floor + lets through.""" + calls = ("chrom\tstart\tend\tlength_mb\tarms\ttype\tcell_fraction\tbdev\trel_cov\tn_hets\tlod_baf_phase" + "\tlod_lrr_baf\treported\treason\n") if has("mocha"): - # A loss of 20q in about 12% of the blood cells, which MoChA's filters keep and the floor lets through. - calls.append({"sample_id": SAMPLE, "chrom": "chr20", "beg_GRCh38": "30100000", "end_GRCh38": "42300000", - "length": "12200000", "p_arm": "N", "q_arm": "Y", "type": "Loss", "cf": "0.120", - "bdev": "0.031", "bdev_se": "0.004", "n50_hets": "350000", "lod_baf_conc": "nan", - "rel_cov": "1.86", "lod_baf_phase": "48.2", "lod_lrr_baf": "35.1", "n_hets": "4210"}) - rows, summary = mr.report(SAMPLE, calls, stats, sites, "beagle") - os.makedirs(os.path.join(d, "mosaic"), exist_ok=True) - with open(os.path.join(d, "mosaic", f"{SAMPLE}_mocha.tsv"), "w", encoding="utf-8") as f: - f.write("\t".join(mr.CALL_COLUMNS) + "\n") - for x in rows: - f.write("\t".join(str(x[k]) for k in mr.CALL_COLUMNS) + "\n") - with open(os.path.join(d, "mosaic", f"{SAMPLE}_mocha_summary.tsv"), "w", encoding="utf-8") as f: - f.write("metric\tvalue\n") - for k, v in summary.items(): - f.write(f"{k}\t{v}\n") + calls += "chr20\t30100000\t42300000\t12.20\t20q\tloss\t0.120\t0.0310\t1.8600\t4210\t48.2\t35.1\tyes\t.\n" + n = "1" if has("mocha") else "0" + summary = [("status", "assessed"), ("message", "."), ("depth", "31.0"), ("het_sites", "2090000"), + ("phased_het_sites", "2068000"), ("het_per_10mb", "9500"), ("analysed_mb", "2200.0"), + ("chromosomes", ",".join(f"chr{i}" for i in range(1, 23))), ("call_rate", "0.9981"), + ("baf_auto", "0.0124"), ("floor_read", "0.011"), ("floor_min", "0.05"), ("floor", "0.050"), + ("calls_mocha", n), ("calls_reported", n), ("phasing", "beagle")] + put(os.path.join(d, "mosaic", f"{SAMPLE}_mocha.tsv"), calls) + put(os.path.join(d, "mosaic", f"{SAMPLE}_mocha_summary.tsv"), + "metric\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in summary)) def vntyper(d): - """Step 52: bin/vntyper_report.py's reading of a made-up VNtyper run with - no MUC1 frameshift at a depth its coverage check passes.""" - import tempfile - import vntyper_report as vr - with tempfile.TemporaryDirectory() as run: - put(os.path.join(run, "kestrel", "kestrel_result.tsv"), - "Motif\tVariant\tConfidence\tFlag\nnone\tNone\tNegative\tNot flagged\n") - put(os.path.join(run, "coverage", "coverage_summary.tsv"), - "mean\tpercent_uncovered\tcoverage_qc\n212.4\t0.0\tPASS\n") - t = vr.read(run, 0) - os.makedirs(os.path.join(d, "vntyper"), exist_ok=True) - with open(os.path.join(d, "vntyper", f"{SAMPLE}_vntyper.tsv"), "w", encoding="utf-8") as f: - f.write("metric\tvalue\n") - for k in vr.FIELDS: - f.write(f"{k}\t{t[k]}\n") + """Step 52: what og vntyper report writes for a made-up VNtyper run with + no MUC1 frameshift, at a VNTR depth of 212x that its coverage check + passes.""" + rows = [("status", "negative"), ("confidence", "Negative"), ("variant", "."), ("flag", "."), + ("alt_depth", "."), ("region_depth", "."), ("depth_score", "."), ("vntr_mean_depth", "212.40"), + ("vntr_uncovered_pct", "0.00"), ("coverage_qc", "PASS"), + ("sensitivity", "Mean depth over the MUC1 VNTR: 212x. At that depth (at 100x or more) VNtyper found the " + "known variant in 160 of 161 runs (99%), in VNtyper 2.0.47's downsampling of 42 confirmed positive " + "GRCh38 exomes (repeated runs, not independent tests)."), + ("message", "VNtyper calls no MUC1 VNTR frameshift."), + ("advice", "A negative here lowers the chance of the frameshifts VNtyper looks for; it does not rule " + "ADTKD-MUC1 out. A family history of kidney failure with gout or a bland urine sediment still deserves " + "a clinical test (SNaPshot for dupC).")] + put(os.path.join(d, "vntyper", f"{SAMPLE}_vntyper.tsv"), "metric\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows)) def locityper(d, rng): - """Step 53: bin/locityper_report.py's reading of a made-up Locityper run + """Step 53: what og locityper report writes for a made-up Locityper run that called every locus of assets/locityper_loci.bed with two HPRC panel - haplotypes and passed the authors' filter.""" - import gzip - import tempfile - import locityper_report as lr - loci_path = os.path.join(REPO, "assets", "locityper_loci.bed") + haplotypes, made-up qualities and read counts, and passed the authors' + filter (no warning, a weighted distance under 30, few unexplained reads).""" panel = ["HG00438.1", "HG00733.2", "HG01891.1", "HG02257.2", "NA18906.1", "NA19240.2"] - out = os.path.join(d, "locityper") - os.makedirs(out, exist_ok=True) - with tempfile.TemporaryDirectory() as run: - for name, _ in lr.read_loci(loci_path): - res = {"genotype": ",".join(rng.sample(panel, 2)), "quality": round(rng.uniform(25, 60), 1), - "total_reads": rng.randint(800, 4000), "unexpl_reads": rng.randint(0, 30), - "weight_dist": round(rng.uniform(0.1, 3), 3)} - put(os.path.join(run, "loci", name, "success"), "") - with gzip.open(os.path.join(run, "loci", name, "res.json.gz"), "wt") as f: - json.dump(res, f) - if lr.main(["--gt-dir", run, "--loci", loci_path, "--out", os.path.join(out, f"{SAMPLE}_locityper.tsv")]): - raise SystemExit("locityper_report failed") + out = ["locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads" + "\tweighted_distance\twarnings\tfilter\treason\n"] + with open(os.path.join(REPO, "assets", "locityper_loci.bed")) as f: + for line in f: + if not line.strip() or line.startswith("#"): + continue + chrom, start, end, name = line.rstrip("\n").split("\t")[:4] + haps = rng.sample(panel, 2) + quality, total = round(rng.uniform(25, 60), 1), rng.randint(800, 4000) + unexpl, dist = rng.randint(0, 30), round(rng.uniform(0.1, 3), 3) + out.append("\t".join([name, f"{chrom}:{int(start) + 1}-{end}", "called", ",".join(haps), *haps, + f"{quality:.1f}", str(total), str(unexpl), f"{dist:.3f}", ".", "pass", "."]) + "\n") + put(os.path.join(d, "locityper", f"{SAMPLE}_locityper.tsv"), "".join(out)) def mei(d, rng): - """Step 50: bin/mei_report.py run on a made-up MEGAnE VCF (a few Alu and L1 - insertions away from the catalog) and a made-up bedcov near 31x over every - window of assets/mei_sites.tsv, so every catalog row reads "not seen".""" - import tempfile - import mei_report - sites_path = os.path.join(REPO, "assets", "mei_sites.tsv") - sites = mei_report.read_sites(sites_path) - out = os.path.join(d, "mei") - os.makedirs(out, exist_ok=True) - with tempfile.TemporaryDirectory() as t: - vcf, depth = os.path.join(t, "megane.vcf"), os.path.join(t, "depth.tsv") - recs = [("chr3", 41258120, "SINE/Alu", "AluYa5"), ("chr7", 102931455, "SINE/Alu", "AluYb8"), - ("chr11", 63044871, "LINE/L1", "L1HS")] - put(vcf, "##fileformat=VCFv4.1\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + SAMPLE + "\n" - + "".join(f"{c}\t{p}\t.\tN\t\t.\tPASS\tSVTYPE={e};MEPRED=PASS;MEI={s};CN_conf=H\tGT:CN\t0/1:1\n" - for c, p, e, s in recs)) - put(depth, "".join(f"{r['chrom']}\t{r['start']}\t{r['end']}\t{r['gene']}\t" - f"{round((int(r['end']) - int(r['start'])) * rng.uniform(28, 34))}\t500\n" for r in sites)) - if mei_report.main(["--sample", SAMPLE, "--vcf", vcf, "--sites", sites_path, "--depth", depth, - "--out-calls", os.path.join(out, f"{SAMPLE}_mei.tsv"), - "--out-sites", os.path.join(out, f"{SAMPLE}_mei_sites.tsv")]): - raise SystemExit("mei_report failed") + """Step 50: what og mei report writes for a made-up MEGAnE VCF (two Alu + and one L1 insertion, away from the catalog) and a made-up depth near 31x + over every window of assets/mei_sites.tsv, so every catalog row reads + "not seen".""" + calls = ["chrom\tpos\tend\telement\tsubfamily\tcopies\tfilter\tconfidence\tgenes\tlocation\tbenign_af" + "\tacmg_class\n"] + for c, p, e, sub in (("chr3", 41258120, "Alu", "AluYa5"), ("chr7", 102931455, "Alu", "AluYb8"), + ("chr11", 63044871, "L1", "L1HS")): + calls.append(f"{c}\t{p}\t{p}\t{e}\t{sub}\t1\tPASS\tH\t.\t.\t.\t.\n") + with open(os.path.join(REPO, "assets", "mei_sites.tsv")) as f: + lines = [x.rstrip("\n") for x in f if x.strip() and not x.startswith("#")] + sites = [lines[0] + "\tstatus\tmean_depth\tcall\n"] + for line in lines[1:]: + start, end = (int(x) for x in line.split("\t")[2:4]) + mean = round((end - start) * rng.uniform(28, 34)) / (end - start) + sites.append(f"{line}\tnot seen\t{mean:.1f}\t.\n") + put(os.path.join(d, "mei", f"{SAMPLE}_mei.tsv"), "".join(calls)) + put(os.path.join(d, "mei", f"{SAMPLE}_mei_sites.tsv"), "".join(sites)) def run_status(d, started): diff --git a/tests/test_exomiser_report.py b/tests/test_exomiser_report.py index 6e4cd2db0..b516b6a5d 100644 --- a/tests/test_exomiser_report.py +++ b/tests/test_exomiser_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/exomiser_report.py: the symptom-driven ranking of step 48 (Exomiser). +"""og exomiser prepare and og exomiser report: the symptom-driven ranking of step 48 (Exomiser). Every input is synthetic: a few-term OBO file, one- and four-sample VCF headers, a pedigree, and Exomiser variants TSVs in the column layout of @@ -22,17 +22,21 @@ and render_report.py); the module's -stub table has the same columns and reads as no gene -Run: python3 tests/test_exomiser_report.py +Without OG (the og binary) the test is skipped with a note; with +OG_REQUIRED set too, it fails (Guard's rust job sets both). + +Run: OG=bin/og python3 tests/test_exomiser_report.py """ import gzip import json import os +import shutil import subprocess import sys import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "exomiser_report.py") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) FAILS = [] @@ -119,7 +123,7 @@ def check(name, ok, detail=""): def run(*args): - return subprocess.run([sys.executable, SCRIPT] + list(args), capture_output=True, text=True) + return subprocess.run([OG, "exomiser"] + list(args), capture_output=True, text=True) def write(path, text): @@ -320,7 +324,7 @@ def case_reports(): sd = os.path.join(d, "S1", "exomiser") os.makedirs(sd) r = report(sd, ROWS) - os.rename(os.path.join(sd, "top.tsv"), os.path.join(sd, "S1_exomiser_top.tsv")) + shutil.copy(os.path.join(sd, "top.tsv"), os.path.join(sd, "S1_exomiser_top.tsv")) os.rename(os.path.join(sd, "terms.tsv"), os.path.join(sd, "S1_exomiser_terms.tsv")) p, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") check("summary: the section reads the table", p is not None and r.returncode == 0) @@ -347,11 +351,12 @@ def case_reports(): check("summary: a table without its columns is unreadable", True) # The -stub table of EXOMISER_REPORT reads as "no gene passed", not unreadable. import re - import exomiser_report as er + with open(os.path.join(sd, "top.tsv")) as f: + out_header = f.readline() nf = open(os.path.join(REPO, "modules", "local", "exomiser", "main.nf")).read() m = re.search(r"printf '([^']*)' > \$\{meta\.id\}_exomiser_top\.tsv", nf.split("stub:")[-1]) head = m.group(1).replace("\\\\t", "\t").replace("\\\\n", "\n") if m else "" - check("stub: the top table carries the report's columns", head == "\t".join(er.OUT_HEADER) + "\n", repr(head)) + check("stub: the top table carries the report's columns", head == out_header, repr(head)) write(bad, head) try: _, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") @@ -361,6 +366,9 @@ def case_reports(): if __name__ == "__main__": + if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) case_prepare() case_prepare_stops() case_family() diff --git a/tests/test_locityper_report.py b/tests/test_locityper_report.py index f2bf59324..5bbae64a9 100755 --- a/tests/test_locityper_report.py +++ b/tests/test_locityper_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/locityper_report.py: step 53's table from what Locityper genotype +"""og locityper report: step 53's table from what Locityper genotype wrote, and its section in both reports. On Locityper output folders made up for each case, in the layout Locityper @@ -19,8 +19,8 @@ column, skips '#' lines, and exits 1 on a BED it cannot read; 5. both reports: the card and the text section carry each locus's call, filter and the primary caller's call; no card without step 53's table; - 6. control: a copy of the script whose filter passes everything fails - case 2, so these checks can fail; + 6. control: a reading whose filter passes everything fails case 2, so + these checks can fail; 7. scripts/run-all.sh, under the fakes of scripts/ci/fake-docker (no Docker, no Nextflow): a default run neither lists step 53 nor passes locityper to Nextflow; TOOLS naming locityper with the database @@ -39,10 +39,13 @@ a DONE that names only the .fai, which the old guard accepted, is refused. -Run: python3 tests/test_locityper_report.py +Cases 1 to 6 run og: without OG (the og binary) they are skipped with a +note, and with OG_REQUIRED set too, they fail (Guard's rust job sets +both). Cases 7 to 9 always run. + +Run: OG=bin/og python3 tests/test_locityper_report.py """ import gzip -import importlib.util import json import os import subprocess @@ -50,9 +53,10 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "locityper_report.py") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import locityper_report as lr # noqa: E402 +COLUMNS = ["locus", "region", "status", "genotype", "haplotype_1", "haplotype_2", "quality", "total_reads", + "unexplained_reads", "weighted_distance", "warnings", "filter", "reason"] FAILS = 0 BED = ("# a comment line\n" @@ -99,7 +103,33 @@ def gt_dir(root, locus, result=None, success=True, raw=None, truncate=False, dam return root -def cases(mod, t): +def og(*args): + return subprocess.run([OG, "locityper", "report", *args], capture_output=True, text=True) + + +def og_row(name, region, root, exit_code): + """og locityper report on one locus: its row as a dict.""" + chrom, span = region.split(":") + start, end = span.split("-") + bed, out = root + ".bed", root + ".tsv" + with open(bed, "w") as f: + f.write(f"{chrom}\t{int(start) - 1}\t{end}\t{name}\n") + p = og("--gt-dir", root, "--loci", bed, "--exit-code", str(exit_code), "--out", out) + if p.returncode: + raise SystemExit(f"og locityper report failed: {p.stderr}") + lines = [line.rstrip("\n").split("\t") for line in open(out)] + return dict(zip(lines[0], lines[1])) + + +def pass_everything(*args): + """The control: a reading whose filter passes every called locus.""" + r = og_row(*args) + if r["status"] == "called": + r.update(filter="pass", reason=".") + return r + + +def cases(row_for, t): out = [] n = [0] @@ -107,7 +137,7 @@ def row(result=None, success=True, raw=None, exit_code=0, truncate=False, damage n[0] += 1 root = os.path.join(t, f"c{n[0]}") gt_dir(root, "CYP2D6", result, success, raw, truncate, damage) - return mod.row_for("CYP2D6", "chr22:42122285-42152960", root, exit_code) + return row_for("CYP2D6", "chr22:42122285-42152960", root, exit_code) r = row(res()) out.append(("1. a clean call is called and passes, both haplotypes in their columns", @@ -320,93 +350,12 @@ def module(done): def main(): with tempfile.TemporaryDirectory() as t: - for desc, ok, detail in cases(lr, os.path.join(t, "real")): - check(desc, ok, detail) - - # 4. the CLI - bed = os.path.join(t, "loci.bed") - with open(bed, "w") as f: - f.write(BED) - root = os.path.join(t, "cli") - gt_dir(root, "CYP2D6", res()) - gt_dir(root, "HLA-DQB1", res(wd=50)) - out_tsv = os.path.join(t, "cli.tsv") - p = subprocess.run([sys.executable, SCRIPT, "--gt-dir", root, "--loci", bed, "--exit-code", "0", - "--out", out_tsv], capture_output=True, text=True) - rows = [line.rstrip("\n").split("\t") for line in open(out_tsv)] if os.path.exists(out_tsv) else [] - check("4. the CLI exits 0, one row per BED locus in its order, every column", - p.returncode == 0 and rows[:1] == [lr.COLUMNS] and [r[0] for r in rows[1:]] == ["HLA-DQB1", "CYP2D6", "FCGR"] - and all(len(r) == len(lr.COLUMNS) for r in rows) and rows[1][1] == "chr6:32652604-32669077" - and [r[2] for r in rows[1:]] == ["called", "called", "not called"] - and "2 of 3 loci called, 1 pass" in p.stdout, (p, rows)) - badbed = os.path.join(t, "bad.bed") - with open(badbed, "w") as f: - f.write("chr6\tx\t10\tA\n") - p = subprocess.run([sys.executable, SCRIPT, "--gt-dir", root, "--loci", badbed, "--out", - os.path.join(t, "bad.tsv")], capture_output=True, text=True) - check("4. a BED it cannot read exits 1", p.returncode == 1 and "ERROR" in p.stderr, p) - - # 5. the reports - import collect_summary - import render_report - - sd = os.path.join(t, "s", "S") - os.makedirs(os.path.join(sd, "locityper")) - lr.main(["--gt-dir", root, "--loci", bed, "--out", os.path.join(sd, "locityper", "S_locityper.tsv")]) - s = collect_summary.collect("S", sd) - txt, html = render_report.text_report(s), render_report.html_report(s) - sec = s["sections"].get("locityper", {}) - check("5. the section is read, with the counts", - sec.get("state") == "ok" and sec["data"]["called"] == 2 and sec["data"]["passed"] == 1, sec) - check("5. each locus carries the primary caller's call", - [x.get("primary", "")[:5] for x in sec["data"]["loci"]] == ["T1K (", "pypgx", "no ot"], sec["data"]) - check("5. the text section has the calls, the filter and the primary caller", - "## Locityper Second Opinion at Hard Loci (step 53)" in txt and "HG00733.1 + NA19240.2" in txt - and "filter fail: weighted distance 50.00" in txt and "primary: no other step calls FCGR" in txt - and "not called: Locityper wrote no result" in txt, txt) - check("5. the HTML card has the table and the filter badges", - "Locityper Second Opinion at Hard Loci" in html and 'badge-green">pass<' in html - and 'badge-yellow">fail<' in html and "" in html, html[-3000:]) - os.makedirs(os.path.join(sd, "cyrius")) - with open(os.path.join(sd, "cyrius", "S_cyp2d6.tsv"), "w") as f: - f.write("Sample\tGenotype\tFilter\nS\tfixture_call\tLowQ_high_CN\n") - s = collect_summary.collect("S", sd) - cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] - check("5. a Cyrius call without Filter PASS is shown as not usable, as the CYP2D6 block does", - "Cyrius (step 21): fixture_call (not usable: Filter LowQ_high_CN)" in cyp["primary"], cyp) - os.makedirs(os.path.join(sd, "logs")) - import time - with open(os.path.join(sd, "logs", "run_status.tsv"), "w") as f: - f.write(f"meta\tstarted_epoch\t{time.time() + 60}\nstep\t53\tok\n") - s = collect_summary.collect("S", sd) - cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] - check("5. a primary call older than the last run says it is stale", "(stale: from an earlier run)" in cyp["primary"], - (s["sections"]["cyrius"]["state"], cyp)) - hla = {"loci": [{"gene": "DRB1", "alleles": ["*15:01", "*03:01"], "low_confidence": True}, - {"gene": "DQB1", "alleles": ["*06:02", "*02:01"], "low_confidence": False}]} - prim = collect_summary.locityper_primary({"hla": {"state": "ok", "data": hla}, "pypgx": {"state": "missing"}, - "cyrius": {"state": "missing"}}) - check("5. a low-confidence T1K call says so in the primary text, a confident one does not", - prim["HLA-DRB1"] == "T1K (step 08): *15:01/*03:01 (low confidence)" - and prim["HLA-DQB1"] == "T1K (step 08): *06:02/*02:01", prim) - os.makedirs(os.path.join(t, "none", "S")) - s = collect_summary.collect("S", os.path.join(t, "none", "S")) - check("5. without step 53 there is no card and no text section", - "Locityper Second Opinion" not in render_report.html_report(s) - and "Locityper Second Opinion" not in render_report.text_report(s)) - - # 6. control: a copy whose filter passes everything - src = open(SCRIPT).read() - cond = 'return ("fail" if why else "pass"), "; ".join(why)' - check("6. control: the line to break is in the script", cond in src) - broken = os.path.join(t, "locityper_report_broken.py") - with open(broken, "w") as f: - f.write(src.replace(cond, 'return "pass", ""')) - spec = importlib.util.spec_from_file_location("locityper_report_broken", broken) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - failed = [desc for desc, ok, _ in cases(mod, os.path.join(t, "broken")) if not ok] - check("6. control: the broken copy fails the filter cases", any(x.startswith("2.") for x in failed), failed) + if OG: + og_cases(t) + elif os.environ.get("OG_REQUIRED"): + check("OG names the og binary (OG_REQUIRED is set)", False) + else: + print("skip: cases 1 to 6 need OG (OG=bin/og)") # 7. run-all.sh plans step 53 only when TOOLS names it plan_checks(os.path.join(t, "plan")) @@ -418,5 +367,85 @@ def main(): return 1 if FAILS else 0 +def og_cases(t): + """Cases 1 to 6, on og.""" + for desc, ok, detail in cases(og_row, os.path.join(t, "real")): + check(desc, ok, detail) + + # 4. the CLI + bed = os.path.join(t, "loci.bed") + with open(bed, "w") as f: + f.write(BED) + root = os.path.join(t, "cli") + gt_dir(root, "CYP2D6", res()) + gt_dir(root, "HLA-DQB1", res(wd=50)) + out_tsv = os.path.join(t, "cli.tsv") + p = og("--gt-dir", root, "--loci", bed, "--exit-code", "0", "--out", out_tsv) + rows = [line.rstrip("\n").split("\t") for line in open(out_tsv)] if os.path.exists(out_tsv) else [] + check("4. the CLI exits 0, one row per BED locus in its order, every column", + p.returncode == 0 and rows[:1] == [COLUMNS] and [r[0] for r in rows[1:]] == ["HLA-DQB1", "CYP2D6", "FCGR"] + and all(len(r) == len(COLUMNS) for r in rows) and rows[1][1] == "chr6:32652604-32669077" + and [r[2] for r in rows[1:]] == ["called", "called", "not called"] + and "2 of 3 loci called, 1 pass" in p.stdout, (p, rows)) + badbed = os.path.join(t, "bad.bed") + with open(badbed, "w") as f: + f.write("chr6\tx\t10\tA\n") + p = og("--gt-dir", root, "--loci", badbed, "--out", os.path.join(t, "bad.tsv")) + check("4. a BED it cannot read exits 1", p.returncode == 1 and "ERROR" in p.stderr, p) + + # 5. the reports + import collect_summary + import render_report + + sd = os.path.join(t, "s", "S") + os.makedirs(os.path.join(sd, "locityper")) + og("--gt-dir", root, "--loci", bed, "--out", os.path.join(sd, "locityper", "S_locityper.tsv")) + s = collect_summary.collect("S", sd) + txt, html = render_report.text_report(s), render_report.html_report(s) + sec = s["sections"].get("locityper", {}) + check("5. the section is read, with the counts", + sec.get("state") == "ok" and sec["data"]["called"] == 2 and sec["data"]["passed"] == 1, sec) + check("5. each locus carries the primary caller's call", + [x.get("primary", "")[:5] for x in sec["data"]["loci"]] == ["T1K (", "pypgx", "no ot"], sec["data"]) + check("5. the text section has the calls, the filter and the primary caller", + "## Locityper Second Opinion at Hard Loci (step 53)" in txt and "HG00733.1 + NA19240.2" in txt + and "filter fail: weighted distance 50.00" in txt and "primary: no other step calls FCGR" in txt + and "not called: Locityper wrote no result" in txt, txt) + check("5. the HTML card has the table and the filter badges", + "Locityper Second Opinion at Hard Loci" in html and 'badge-green">pass<' in html + and 'badge-yellow">fail<' in html and "" in html, html[-3000:]) + os.makedirs(os.path.join(sd, "cyrius")) + with open(os.path.join(sd, "cyrius", "S_cyp2d6.tsv"), "w") as f: + f.write("Sample\tGenotype\tFilter\nS\tfixture_call\tLowQ_high_CN\n") + s = collect_summary.collect("S", sd) + cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] + check("5. a Cyrius call without Filter PASS is shown as not usable, as the CYP2D6 block does", + "Cyrius (step 21): fixture_call (not usable: Filter LowQ_high_CN)" in cyp["primary"], cyp) + os.makedirs(os.path.join(sd, "logs")) + import time + with open(os.path.join(sd, "logs", "run_status.tsv"), "w") as f: + f.write(f"meta\tstarted_epoch\t{time.time() + 60}\nstep\t53\tok\n") + s = collect_summary.collect("S", sd) + cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] + check("5. a primary call older than the last run says it is stale", "(stale: from an earlier run)" in cyp["primary"], + (s["sections"]["cyrius"]["state"], cyp)) + hla = {"loci": [{"gene": "DRB1", "alleles": ["*15:01", "*03:01"], "low_confidence": True}, + {"gene": "DQB1", "alleles": ["*06:02", "*02:01"], "low_confidence": False}]} + prim = collect_summary.locityper_primary({"hla": {"state": "ok", "data": hla}, "pypgx": {"state": "missing"}, + "cyrius": {"state": "missing"}}) + check("5. a low-confidence T1K call says so in the primary text, a confident one does not", + prim["HLA-DRB1"] == "T1K (step 08): *15:01/*03:01 (low confidence)" + and prim["HLA-DQB1"] == "T1K (step 08): *06:02/*02:01", prim) + os.makedirs(os.path.join(t, "none", "S")) + s = collect_summary.collect("S", os.path.join(t, "none", "S")) + check("5. without step 53 there is no card and no text section", + "Locityper Second Opinion" not in render_report.html_report(s) + and "Locityper Second Opinion" not in render_report.text_report(s)) + + # 6. control: a reading whose filter passes everything + failed = [desc for desc, ok, _ in cases(pass_everything, os.path.join(t, "broken")) if not ok] + check("6. control: the broken reading fails the filter cases", any(x.startswith("2.") for x in failed), failed) + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/test_mei_report.py b/tests/test_mei_report.py index 91a2adfdc..d87f1c5cb 100644 --- a/tests/test_mei_report.py +++ b/tests/test_mei_report.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""bin/mei_report.py: the Alu, L1 and SVA insertions MEGAnE calls, and the known disease insertions (step 50). +"""og mei report: the Alu, L1 and SVA insertions MEGAnE calls, and the known disease insertions (step 50). Each case writes a synthetic VCF in the format MEGAnE writes (the header and INFO fields of its MEI_final_*_genotyped.vcf), the catalog of assets/mei_sites.tsv and a samtools bedcov -c file with chosen depths, runs -the script, and checks what it writes: +og mei report, and checks what it writes: 1. an SVA planted inside the FKTN window is seen, with its position, subfamily and copies; the same VCF without it says not seen; the same @@ -17,7 +17,7 @@ copies, with MEGAnE's FILTER and copy-number confidence, in karyotype order; it leaves out MEPRED=FAILED, zero copies and other classes (DNA transposons, LTRs); the subfamily comes from MEI, else from a breakpoint - 4. damaged input stops the script: a VCF without its #CHROM line, a VCF + 4. damaged input stops og: a VCF without its #CHROM line, a VCF that is not MEGAnE's (no SVTYPE or MEPRED), a depth file that is not samtools bedcov -c output, a catalog without its columns 5. collect_summary.py's mei section and render_report.py's text block and @@ -26,8 +26,8 @@ without the tables the summary's "not assessed" list names mobile element insertions, with them it does not 6. the real catalog: every row is an SVA, its window is the site -/+ 500, - and the script reads it - 7. AnnotSV: --annotsv-input writes the listed calls, and only them, as + and og reads it + 7. AnnotSV: og mei annotsv-input writes the listed calls, and only them, as records with END=POS, an ID per call and the VCF's contigs; --annotsv joins a table in AnnotSV's columns back by that ID: genes, the location in each gene, the benign insertion frequency and the @@ -35,7 +35,10 @@ --annotsv, has "." there; a table that is not AnnotSV's stops it; an empty one (AnnotSV had nothing to annotate) leaves every call "." -Run: python3 tests/test_mei_report.py +Without OG (the og binary) the test is skipped with a note; with +OG_REQUIRED set too, it fails (Guard's rust job sets both). + +Run: OG=bin/og python3 tests/test_mei_report.py """ import csv import os @@ -45,10 +48,14 @@ import time REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "mei_report.py") +OG = os.environ.get("OG", "") CATALOG = os.path.join(REPO, "assets", "mei_sites.tsv") sys.path.insert(0, os.path.join(REPO, "bin")) +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) + FAILS = [] HEAD = """##fileformat=VCFv4.1 ##source=MEI search version "v1.0.1 2022/04/12" @@ -85,7 +92,7 @@ def check(name, got, want): def run(records, depth=DEPTH_30X, sites=SITES, vcf=None): - """(calls, sites) as lists of dicts, or the error text when the script fails.""" + """(calls, sites) as lists of dicts, or the error text when og fails.""" with tempfile.TemporaryDirectory() as d: with open(f"{d}/m.vcf", "w") as f: f.write(vcf if vcf is not None else HEAD + "".join(records)) @@ -93,7 +100,7 @@ def run(records, depth=DEPTH_30X, sites=SITES, vcf=None): f.write(sites) with open(f"{d}/depth.tsv", "w") as f: f.write(depth) - p = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", + p = subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", "--depth", f"{d}/depth.tsv", "--out-calls", f"{d}/c.tsv", "--out-sites", f"{d}/s.tsv"], capture_output=True, text=True) if p.returncode: @@ -161,7 +168,7 @@ def status(result, gene): # 4. damaged input err = run([], vcf=FKTN_SVA) -check("4 a VCF without #CHROM stops the script", isinstance(err, str) and "not a VCF" in err, True) +check("4 a VCF without #CHROM stops og", isinstance(err, str) and "not a VCF" in err, True) err = run([], vcf=HEAD + "chr9\t1\t.\tN\t\t.\tPASS\tEND=5\tCN\t1\n") check("4 a VCF that is not MEGAnE's stops it", isinstance(err, str) and "not MEGAnE's VCF" in err, True) err = run([], depth="chr9\t105639155\t105640156\n") @@ -171,7 +178,6 @@ def status(result, gene): # 5. the summary and the reports import collect_summary as cs # noqa: E402 -import mei_report as mr # noqa: E402 import render_report as rr # noqa: E402 with tempfile.TemporaryDirectory() as d: os.makedirs(f"{d}/S") @@ -193,8 +199,9 @@ def status(result, gene): ("depth.tsv", DEPTH_30X)): with open(f"{d}/{name}", "w") as f: f.write(text) - mr.main(["--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", "--depth", f"{d}/depth.tsv", - "--out-calls", f"{d}/S/mei/S_mei.tsv", "--out-sites", f"{d}/S/mei/S_mei_sites.tsv"]) + subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", + "--depth", f"{d}/depth.tsv", "--out-calls", f"{d}/S/mei/S_mei.tsv", + "--out-sites", f"{d}/S/mei/S_mei_sites.tsv"], check=True) path, data = cs.sec_mei(f"{d}/S", "S") check("5 the summary finds the table", path.endswith("mei/S_mei_sites.tsv"), True) check("5 the summary counts the calls per element", (data["calls"], data["by_element"]), (2, {"Alu": 1, "SVA": 1})) @@ -239,12 +246,12 @@ def status(result, gene): check("5 a sites table without its columns is unreadable", "Unreadable", "Unreadable") # 6. the real catalog -rows = mr.read_sites(CATALOG) +rows = list(csv.DictReader([x for x in open(CATALOG) if not x.startswith("#")], delimiter="\t")) check("6 the catalog: FKTN, TAF1 and BBS1", [x["gene"] for x in rows], ["FKTN", "TAF1", "BBS1"]) check("6 every row is an SVA", {x["element"] for x in rows}, {"SVA"}) check("6 every window is the site -/+ 500", all(int(x["start"]) == int(x["site"]) - 500 and int(x["end"]) == int(x["site"]) + 500 for x in rows), True) -check("6 the script reads it", [x["status"] for x in run([FKTN_SVA], sites=open(CATALOG).read())[1]], +check("6 og reads it", [x["status"] for x in run([FKTN_SVA], sites=open(CATALOG).read())[1]], ["seen", "not seen", "not callable"]) # 7. AnnotSV @@ -253,15 +260,15 @@ def annotsv_run(records, table): with tempfile.TemporaryDirectory() as d: with open(f"{d}/m.vcf", "w") as f: f.write(HEAD.replace("##ALT", "##contig=\n##ALT", 1) + "".join(records)) - p = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--vcf", f"{d}/m.vcf", - "--annotsv-input", f"{d}/in.vcf"], capture_output=True, text=True) + p = subprocess.run([OG, "mei", "annotsv-input", "--sample", "S", "--vcf", f"{d}/m.vcf", + "--out", f"{d}/in.vcf"], capture_output=True, text=True) if p.returncode: return p.stderr vcf = open(f"{d}/in.vcf").read() for name, text in (("sites.tsv", SITES), ("depth.tsv", DEPTH_30X), ("a.tsv", table)): with open(f"{d}/{name}", "w") as f: f.write(text) - p = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", + p = subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", "--depth", f"{d}/depth.tsv", "--annotsv", f"{d}/a.tsv", "--out-calls", f"{d}/c.tsv", "--out-sites", f"{d}/s.tsv"], capture_output=True, text=True) if p.returncode: @@ -313,8 +320,8 @@ def annotsv_run(records, table): with tempfile.TemporaryDirectory() as d: with open(f"{d}/m.vcf", "w") as f: f.write(HEAD) - p = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--vcf", f"{d}/m.vcf"], capture_output=True, text=True) -check("7 without --annotsv-input the report options are required", p.returncode == 2 and "--sites" in p.stderr, True) + p = subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf"], capture_output=True, text=True) +check("7 the report's options are required", p.returncode == 2 and "--sites" in p.stderr, True) if FAILS: print(f"{len(FAILS)} check(s) failed") diff --git a/tests/test_mocha_report.py b/tests/test_mocha_report.py index 11ea23dc0..55a94a56a 100644 --- a/tests/test_mocha_report.py +++ b/tests/test_mocha_report.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -"""bin/mocha_report.py: step 51's mosaic chromosomal alterations from MoChA's tables. +"""og mocha report: step 51's mosaic chromosomal alterations from MoChA's tables. Each case writes MoChA's --stats and --calls tables and the per-chromosome -site counts with chosen numbers, runs the script, and checks what it writes: +site counts with chosen numbers, runs og mocha report, and checks what it +writes: 1. the floor: 6 / sqrt(D x H) at depth D and H heterozygous sites per 10 Mb of the chromosomes read, computed here by hand; at 30x with 3,000 sites @@ -28,7 +29,11 @@ 7. versions.env pins the MoChA image by digest and the plugin by commit, those of the e2e case (tests/e2e/mocha-1.sh) -Run: python3 tests/test_mocha_report.py +Cases 1 to 6 run og: without OG (the og binary) they are skipped with a +note, and with OG_REQUIRED set too, they fail (Guard's rust job sets both). +Case 7 always runs. + +Run: OG=bin/og python3 tests/test_mocha_report.py """ import math import os @@ -38,10 +43,9 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "mocha_report.py") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) import collect_summary as cs # noqa: E402 -import mocha_report as mr # noqa: E402 import render_report as rr # noqa: E402 FAILS = [] @@ -68,7 +72,7 @@ def call(chrom="chr20", beg=60000, end=20000000, p="T", q="N", n50=3000, bdev=0. def run(calls, depth=30.0, hets_per_10mb=3000, call_rate=0.999, baf_auto=0.001, phased=True, stats=None, sites=None): - """Write the three inputs in a fresh folder, run the script; (exit, rows, summary, folder).""" + """Write the three inputs in a fresh folder, run og; (exit, rows, summary, folder).""" d = tempfile.mkdtemp() span = 60_000_000 h = int(hets_per_10mb * span / 1e7) @@ -79,112 +83,122 @@ def run(calls, depth=30.0, hets_per_10mb=3000, call_rate=0.999, baf_auto=0.001, with open(f"{d}/sites.tsv", "w") as f: f.write(sites or f"chr20\t1\t{span}\t{3 * h}\t{h}\t{h if phased else 0}\n") os.makedirs(f"{d}/S/mosaic") - rc = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--calls", f"{d}/calls.tsv", "--stats", + rc = subprocess.run([OG, "mocha", "report", "--sample", "S", "--calls", f"{d}/calls.tsv", "--stats", f"{d}/stats.tsv", "--sites", f"{d}/sites.tsv", "--phasing", "input", "--out-calls", f"{d}/S/mosaic/S_mocha.tsv", "--out-summary", f"{d}/S/mosaic/S_mocha_summary.tsv"], capture_output=True, text=True).returncode if rc: return rc, [], {}, d - rows = [dict(zip(mr.CALL_COLUMNS, ln.rstrip("\n").split("\t"))) for ln in open(f"{d}/S/mosaic/S_mocha.tsv")][1:] + lines = [ln.rstrip("\n").split("\t") for ln in open(f"{d}/S/mosaic/S_mocha.tsv")] + rows = [dict(zip(lines[0], ln)) for ln in lines[1:]] summ = dict(ln.rstrip("\n").split("\t") for ln in open(f"{d}/S/mosaic/S_mocha_summary.tsv")) return rc, rows, summ, d -# 1. the floor -check("1 30x, 3,000 hets per 10 Mb: the read-sampling floor by hand", round(mr.floor_read(30, 3000), 4), - round(6 / math.sqrt(30 * 3000), 4)) -_, _, s, _ = run([call()]) -check("1 at 30x the floor is the minimum, 0.05", (s["floor"], s["floor_min"], s["floor_read"]), ("0.050", "0.05", "0.020")) -_, _, s, _ = run([call()], depth=5, hets_per_10mb=300) -check("1 at 5x with 300 hets per 10 Mb the floor is the read-sampling one", s["floor"], f"{6 / math.sqrt(5 * 300):.3f}") -check("1 the het density comes from the sites table", s["het_per_10mb"], "300") - -# 2. reported, under the floor -rc, rows, s, _ = run([call()]) -check("2 a 0.15 CN-LOH is reported", [(r["type"], r["cell_fraction"], r["reported"]) for r in rows], - [("CN-LOH", "0.150", "yes")]) -check("2 with its arm", rows[0]["arms"], "20p, to the p telomere") -check("2 the summary counts it", (s["status"], s["calls_reported"], s["calls_mocha"]), ("assessed", "1", "1")) -_, rows, s, _ = run([call(bdev=0.02, cf=0.04)]) -check("2 a 0.04 CN-LOH is under the floor", (rows[0]["reported"], rows[0]["reason"]), - ("no", "cell fraction 0.040 under the floor 0.050")) -_, rows, _, _ = run([call()], depth=5, hets_per_10mb=300) -check("2 at 5x with 300 hets the same 0.15 call is under that depth's floor", rows[0]["reported"], "no") - -# 3. MoChA's filter -_, rows, s, _ = run([call(typ="CNP_Loss"), call(bdev=0.2, n50=500000, cf=0.4), call(rel=3.0, bdev=0.16, typ="Gain", - cf=0.6), call(chrom="chrX"), call(cf="nan"), call(beg=30000000, end=60000000, p="N", q="T", - typ="Loss", bdev=0.03, cf=0.12)]) -check("3 a CNP is left out", rows[0]["reason"].startswith("germline copy-number polymorphism"), True) -check("3 a constitutional event is left out", rows[1]["reason"].startswith("likely constitutional"), True) -check("3 a likely germline duplication is left out", rows[2]["reason"].startswith("likely germline duplication"), True) -check("3 chrX is step 33's", rows[3]["reason"].startswith("not an autosome"), True) -check("3 no cell fraction: not reported", rows[4]["reason"], "cell fraction not estimated") -check("3 the loss on 20q is the one reported", [(r["type"], r["arms"]) for r in rows if r["reported"] == "yes"], - [("loss", "20q, to the q telomere")]) -check("3 calls_reported 1 of 6", (s["calls_reported"], s["calls_mocha"]), ("1", "6")) - -# 4. the sample filter -for kw, why in (({"call_rate": 0.95}, "call rate"), ({"baf_auto": 0.05}, "phased BAF autocorrelation"), - ({"phased": False}, "no heterozygous site is phased"), - ({"call_rate": "nan"}, "MoChA gave no call rate or phased BAF autocorrelation"), - ({"baf_auto": "nan"}, "MoChA gave no call rate or phased BAF autocorrelation")): - _, rows, s, _ = run([call()], **kw) - check(f"4 {why}: not assessed", (s["status"], s["message"].startswith(why.split(':')[0]) or why in s["message"]), - ("not assessed", True)) - check(f"4 {why}: no call reported", [r["reported"] for r in rows], ["no"]) - -# 5. the CLI -rc, *_ = run([call()], stats="sample_id\tcall_rate\nS\t0.99\n") -check("5 a stats table without cov_median exits 2", rc, 2) -rc, *_ = run([call()], sites="chr20\t1\t100\n") -check("5 a short sites row exits 2", rc, 2) - -# 6. both reports -_, _, _, d = run([call(), call(chrom="chrX")]) -path, data = cs.sec_mocha(f"{d}/S", "S") -check("6 the section lists the reported call only", [(x["type"], x["chrom"], x["cell_fraction"]) for x in data["calls"]], - [("CN-LOH", "chr20", "0.150")]) -check("6 and counts the other", data["not_reported"], 1) -lines = rr.mosaic_mca_lines(data) -check("6 the text names the call", lines[1], "CN-LOH on 20p, to the p telomere, chr20:60000-20000000 (19.94 Mb): " - "about 0.150 of the cells") -check("6 the first line gives the floor", lines[0].startswith("Reported at a cell fraction of 0.050 or more at 30.0x"), True) -summ = cs.collect("S", f"{d}/S") -check("6 assessed: no not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], False) -os.remove(f"{d}/S/mosaic/S_mocha_summary.tsv") -summ = cs.collect("S", f"{d}/S") -check("6 without step 51: the not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], True) -_, _, _, d = run([call()], baf_auto=0.05) -data = cs.sec_mocha(f"{d}/S", "S")[1] -check("6 not assessed: the text says why", rr.mosaic_mca_lines(data)[0].startswith("Not assessed: phased BAF"), True) -check("6 and the not-assessed line stays", cs.MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) -check("6 no reported call: the text says none is reported, not that none exists", - rr.mosaic_mca_lines(cs.sec_mocha(f"{run([call(cf=0.04)])[3]}/S", "S")[1])[1].startswith("No mosaic alteration reported"), True) -check("6 the pipeline-wide not-assessed list keeps large mosaic alterations out of its chrM-only line", - [x for x in cs.NOT_ASSESSED if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) - - -def unreadable(d): - try: - cs.sec_mocha(f"{d}/S", "S") - except cs.Unreadable as e: - return str(e) - return None - - -_, _, _, d = run([call()]) -os.remove(f"{d}/S/mosaic/S_mocha.tsv") -check("6 a summary without its call table is unreadable", bool(unreadable(d)), True) -_, _, _, d = run([call()]) -with open(f"{d}/S/mosaic/S_mocha.tsv") as f: - head = f.readline() -with open(f"{d}/S/mosaic/S_mocha.tsv", "w") as f: - f.write(head) -check("6 a call table with fewer reported calls than the summary is unreadable", bool(unreadable(d)), True) -with open(f"{d}/S/mosaic/S_mocha.tsv", "w") as f: - f.write("chrom\tstart\n") -check("6 a header-only call table without MoChA's columns is unreadable", bool(unreadable(d)), True) +def cases_1_to_6(): + # 1. the floor + _, _, s, _ = run([call()]) + check("1 30x, 3,000 hets per 10 Mb: the read-sampling floor by hand", s["floor_read"], + f"{6 / math.sqrt(30 * 3000):.3f}") + check("1 at 30x the floor is the minimum, 0.05", (s["floor"], s["floor_min"], s["floor_read"]), ("0.050", "0.05", "0.020")) + _, _, s, _ = run([call()], depth=5, hets_per_10mb=300) + check("1 at 5x with 300 hets per 10 Mb the floor is the read-sampling one", s["floor"], f"{6 / math.sqrt(5 * 300):.3f}") + check("1 the het density comes from the sites table", s["het_per_10mb"], "300") + + # 2. reported, under the floor + rc, rows, s, _ = run([call()]) + check("2 a 0.15 CN-LOH is reported", [(r["type"], r["cell_fraction"], r["reported"]) for r in rows], + [("CN-LOH", "0.150", "yes")]) + check("2 with its arm", rows[0]["arms"], "20p, to the p telomere") + check("2 the summary counts it", (s["status"], s["calls_reported"], s["calls_mocha"]), ("assessed", "1", "1")) + _, rows, s, _ = run([call(bdev=0.02, cf=0.04)]) + check("2 a 0.04 CN-LOH is under the floor", (rows[0]["reported"], rows[0]["reason"]), + ("no", "cell fraction 0.040 under the floor 0.050")) + _, rows, _, _ = run([call()], depth=5, hets_per_10mb=300) + check("2 at 5x with 300 hets the same 0.15 call is under that depth's floor", rows[0]["reported"], "no") + + # 3. MoChA's filter + _, rows, s, _ = run([call(typ="CNP_Loss"), call(bdev=0.2, n50=500000, cf=0.4), call(rel=3.0, bdev=0.16, typ="Gain", + cf=0.6), call(chrom="chrX"), call(cf="nan"), call(beg=30000000, end=60000000, p="N", q="T", + typ="Loss", bdev=0.03, cf=0.12)]) + check("3 a CNP is left out", rows[0]["reason"].startswith("germline copy-number polymorphism"), True) + check("3 a constitutional event is left out", rows[1]["reason"].startswith("likely constitutional"), True) + check("3 a likely germline duplication is left out", rows[2]["reason"].startswith("likely germline duplication"), True) + check("3 chrX is step 33's", rows[3]["reason"].startswith("not an autosome"), True) + check("3 no cell fraction: not reported", rows[4]["reason"], "cell fraction not estimated") + check("3 the loss on 20q is the one reported", [(r["type"], r["arms"]) for r in rows if r["reported"] == "yes"], + [("loss", "20q, to the q telomere")]) + check("3 calls_reported 1 of 6", (s["calls_reported"], s["calls_mocha"]), ("1", "6")) + + # 4. the sample filter + for kw, why in (({"call_rate": 0.95}, "call rate"), ({"baf_auto": 0.05}, "phased BAF autocorrelation"), + ({"phased": False}, "no heterozygous site is phased"), + ({"call_rate": "nan"}, "MoChA gave no call rate or phased BAF autocorrelation"), + ({"baf_auto": "nan"}, "MoChA gave no call rate or phased BAF autocorrelation")): + _, rows, s, _ = run([call()], **kw) + check(f"4 {why}: not assessed", (s["status"], s["message"].startswith(why.split(':')[0]) or why in s["message"]), + ("not assessed", True)) + check(f"4 {why}: no call reported", [r["reported"] for r in rows], ["no"]) + + # 5. the CLI + rc, *_ = run([call()], stats="sample_id\tcall_rate\nS\t0.99\n") + check("5 a stats table without cov_median exits 2", rc, 2) + rc, *_ = run([call()], sites="chr20\t1\t100\n") + check("5 a short sites row exits 2", rc, 2) + + # 6. both reports + _, _, _, d = run([call(), call(chrom="chrX")]) + path, data = cs.sec_mocha(f"{d}/S", "S") + check("6 the section lists the reported call only", [(x["type"], x["chrom"], x["cell_fraction"]) for x in data["calls"]], + [("CN-LOH", "chr20", "0.150")]) + check("6 and counts the other", data["not_reported"], 1) + lines = rr.mosaic_mca_lines(data) + check("6 the text names the call", lines[1], "CN-LOH on 20p, to the p telomere, chr20:60000-20000000 (19.94 Mb): " + "about 0.150 of the cells") + check("6 the first line gives the floor", lines[0].startswith("Reported at a cell fraction of 0.050 or more at 30.0x"), True) + summ = cs.collect("S", f"{d}/S") + check("6 assessed: no not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], False) + os.remove(f"{d}/S/mosaic/S_mocha_summary.tsv") + summ = cs.collect("S", f"{d}/S") + check("6 without step 51: the not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], True) + _, _, _, d = run([call()], baf_auto=0.05) + data = cs.sec_mocha(f"{d}/S", "S")[1] + check("6 not assessed: the text says why", rr.mosaic_mca_lines(data)[0].startswith("Not assessed: phased BAF"), True) + check("6 and the not-assessed line stays", cs.MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) + check("6 no reported call: the text says none is reported, not that none exists", + rr.mosaic_mca_lines(cs.sec_mocha(f"{run([call(cf=0.04)])[3]}/S", "S")[1])[1].startswith("No mosaic alteration reported"), True) + check("6 the pipeline-wide not-assessed list keeps large mosaic alterations out of its chrM-only line", + [x for x in cs.NOT_ASSESSED if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) + + + def unreadable(d): + try: + cs.sec_mocha(f"{d}/S", "S") + except cs.Unreadable as e: + return str(e) + return None + + + _, _, _, d = run([call()]) + os.remove(f"{d}/S/mosaic/S_mocha.tsv") + check("6 a summary without its call table is unreadable", bool(unreadable(d)), True) + _, _, _, d = run([call()]) + with open(f"{d}/S/mosaic/S_mocha.tsv") as f: + head = f.readline() + with open(f"{d}/S/mosaic/S_mocha.tsv", "w") as f: + f.write(head) + check("6 a call table with fewer reported calls than the summary is unreadable", bool(unreadable(d)), True) + with open(f"{d}/S/mosaic/S_mocha.tsv", "w") as f: + f.write("chrom\tstart\n") + check("6 a header-only call table without MoChA's columns is unreadable", bool(unreadable(d)), True) + + +if OG: + cases_1_to_6() +elif os.environ.get("OG_REQUIRED"): + check("OG names the og binary (OG_REQUIRED is set)", False, True) +else: + print("skip: cases 1 to 6 need OG (OG=bin/og)") # 7. the pins env = open(os.path.join(REPO, "versions.env")).read() diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 15bb59205..075e36827 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -131,7 +131,7 @@ def sv_chrom(r): def mca_chrom(c): - """The chromosome of a MoChA call, chr prefix put back (mocha_report.py passes bare names through).""" + """The chromosome of a MoChA call, chr prefix put back (og mocha report passes bare names through).""" ch = str(c["chrom"]) return ch if ch.startswith("chr") else "chr" + ch @@ -317,7 +317,7 @@ def unit_checks(): "cell_fraction": "0.12"}, {"chrom": "chrM", "start": "100", "end": "900", "length_mb": ".", "type": "Del", "cell_fraction": "."}, {"chrom": "chr3", "start": ".", "end": ".", "length_mb": ".", "type": "Dup", "cell_fraction": "."}, - # mocha_report.py also passes bare names through + # og mocha report also passes bare names through {"chrom": "20", "start": "50000000", "end": "52000000", "length_mb": "2.0", "type": "Dup", "cell_fraction": "0.05"}] sm = copy.deepcopy(s) diff --git a/tests/test_viral_report.py b/tests/test_viral_report.py index 4f268b2bc..96d14505d 100644 --- a/tests/test_viral_report.py +++ b/tests/test_viral_report.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""bin/viral_report.py: the viral DNA in the reads that do not map to the human genome (step 47). +"""og viral report: the viral DNA in the reads that do not map to the human genome (step 47). Each case writes a synthetic viral FASTA, the samtools coverage and depth -aa -files with chosen depths, and a human coverage file, runs the script, and +files with chosen depths, and a human coverage file, runs og viral report, and checks the rows it writes: 1. the integrated HHV-6 pattern: the whole HHV-6B genome at half the human @@ -16,14 +16,17 @@ 5. no read on chr1 to chr22: no human depth, integration not assessed 6. the human depth is over the autosomal bases with reads: chrX, chrY and unplaced contigs are left out, and so are bases with no read - 7. damaged input stops the script: a depth file with missing positions, + 7. damaged input stops og: a depth file with missing positions, a coverage file that is not samtools coverage output; a genome with no position at all in the depth file has no read, and is not found 8. collect_summary.py's viral section and render_report.py's text block read the table; with no human depth the HHV-6 line says not assessed and the text leaves out copies per cell; a genome with 1 to 9 reads is listed -Run: python3 tests/test_viral_report.py +Without OG (the og binary) the test is skipped with a note; with +OG_REQUIRED set too, it fails (Guard's rust job sets both). + +Run: OG=bin/og python3 tests/test_viral_report.py """ import os import subprocess @@ -31,9 +34,19 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "viral_report.py") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import viral_report as vr # noqa: E402 +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) + +HEADER = ["virus", "accession", "length", "reads", "breadth", "mean_depth", "mode_depth", "human_depth", + "relative_depth", "copies_per_cell", "call", "note"] +CALL_ICI = "inherited integrated HHV-6 likely" +CALL_CONTROL = "lab control (phiX)" +CALL_FOUND = "viral DNA found" +CALL_FEW = "a few reads" +CALL_NONE = "not found" FAILS = [] COV_HEAD = "#rname\tstartpos\tendpos\tnumreads\tcovbases\tcoverage\tmeandepth\tmeanbaseq\tmeanmapq\n" @@ -76,13 +89,13 @@ def write_inputs(d, depths, reads, human=None): def run(depths=None, reads=None, human=None, raw=None): - """Rows by accession, or the error text when the script fails.""" + """Rows by accession, or the error text when og fails.""" with tempfile.TemporaryDirectory() as d: write_inputs(d, depths or {}, reads or {}, human) if raw: raw(d) out = os.path.join(d, "out.tsv") - p = subprocess.run([sys.executable, SCRIPT, "--sample", "S", "--viral-fasta", f"{d}/v.fa", + p = subprocess.run([OG, "viral", "report", "--sample", "S", "--viral-fasta", f"{d}/v.fa", "--viral-coverage", f"{d}/vcov.tsv", "--viral-depth", f"{d}/vdepth.tsv", "--human-coverage", f"{d}/hcov.tsv", "--out", out], capture_output=True, text=True) if p.returncode: @@ -90,7 +103,7 @@ def run(depths=None, reads=None, human=None, raw=None): with open(out) as f: lines = [ln.rstrip("\n").split("\t") for ln in f] head = lines[0] - check("the header is the documented one", head, vr.HEADER) + check("the header is the documented one", head, HEADER) return {r[1]: dict(zip(head, r)) for r in lines[1:]} @@ -102,64 +115,61 @@ def flat(x): # 1. integration: whole genome at half the human depth (30x human, 15x viral) ici = run({B: flat(15)}, {B: 400}) -check("1 HHV-6B at half the human depth: inherited integrated HHV-6", ici[B]["call"], vr.CALL_ICI) +check("1 HHV-6B at half the human depth: inherited integrated HHV-6", ici[B]["call"], CALL_ICI) check("1 relative depth 0.5", ici[B]["relative_depth"], "0.500") check("1 about one copy per cell", ici[B]["copies_per_cell"], "1.0000") check("1 full breadth", ici[B]["breadth"], "1.0000") check("1 the short name", ici[B]["virus"], "HHV-6B") -check("1 HHV-6A with no read is not found", ici[A]["call"], vr.CALL_NONE) +check("1 HHV-6A with no read is not found", ici[A]["call"], CALL_NONE) check("1 rows in FASTA order", list(ici), [g[0] for g in GENOMES]) check("1 the note cites the threshold", "Liu et al. 2020" in ici[B]["note"], True) above = run({B: flat(9)}, {B: 300}) -check("1 relative 0.30 is integration", (above[B]["relative_depth"], above[B]["call"]), ("0.300", vr.CALL_ICI)) +check("1 relative 0.30 is integration", (above[B]["relative_depth"], above[B]["call"]), ("0.300", CALL_ICI)) below = run({B: flat(8)}, {B: 300}) -check("1 relative 0.27 is not", (below[B]["relative_depth"], below[B]["call"]), ("0.267", vr.CALL_FOUND)) +check("1 relative 0.27 is not", (below[B]["relative_depth"], below[B]["call"]), ("0.267", CALL_FOUND)) check("1 and its note says it is below the threshold", "Below the integration threshold" in below[B]["note"], True) # The mode, not the mean: a telomere-like pile-up on a short stretch does not lift a genome with no reads elsewhere. pile = run({B: lambda p: 400 if p <= 100 else 0}, {B: 300}) check("1 a 100-base pile-up at 400x has mode 0: not integration", (pile[B]["mode_depth"], pile[B]["call"]), - ("0", vr.CALL_FEW)) + ("0", CALL_FEW)) check("1 and its mean is high", pile[B]["mean_depth"], "20.000") # 2. low level, few reads, none low = run({EBV: lambda p: 1 if p % 2 else 0}, {EBV: 40}) -check("2 EBV over half the genome: viral DNA found", low[EBV]["call"], vr.CALL_FOUND) +check("2 EBV over half the genome: viral DNA found", low[EBV]["call"], CALL_FOUND) check("2 its mode depth is 0 or 1, relative far below integration", float(low[EBV]["relative_depth"]) < 0.1, True) few = run({EBV: lambda p: 1 if p <= 100 else 0}, {EBV: 12}) -check("2 12 reads over 7% of EBV: a few reads", few[EBV]["call"], vr.CALL_FEW) +check("2 12 reads over 7% of EBV: a few reads", few[EBV]["call"], CALL_FEW) nine = run({EBV: lambda p: 1 if p <= 1000 else 0}, {EBV: 9}) -check("2 nine reads: not found", nine[EBV]["call"], vr.CALL_NONE) +check("2 nine reads: not found", nine[EBV]["call"], CALL_NONE) hcv = run({HCV: flat(2)}, {HCV: 50}) check("2 HCV DNA is flagged as an RNA virus", "RNA virus with no DNA stage" in hcv[HCV]["note"], True) check("2 HHV-6 rule never applies to EBV at integration depth", - run({EBV: flat(15)}, {EBV: 500})[EBV]["call"], vr.CALL_FOUND) + run({EBV: flat(15)}, {EBV: 500})[EBV]["call"], CALL_FOUND) # 3. phiX px = run({PHIX: flat(30)}, {PHIX: 200}) -check("3 phiX at human depth is the lab control", px[PHIX]["call"], vr.CALL_CONTROL) +check("3 phiX at human depth is the lab control", px[PHIX]["call"], CALL_CONTROL) check("3 one phiX read is still the lab control", run({PHIX: lambda p: 1 if p < 5 else 0}, {PHIX: 1})[PHIX]["call"], - vr.CALL_CONTROL) -check("3 no phiX read: not found", run()[PHIX]["call"], vr.CALL_NONE) + CALL_CONTROL) +check("3 no phiX read: not found", run()[PHIX]["call"], CALL_NONE) # 4. both species pass: the deeper one wins both = run({A: flat(10), B: flat(15)}, {A: 300, B: 400}) -check("4 HHV-6B (deeper) is the integrated call", both[B]["call"], vr.CALL_ICI) +check("4 HHV-6B (deeper) is the integrated call", both[B]["call"], CALL_ICI) check("4 HHV-6A is read as shared reads", (both[A]["call"], "Reads shared with HHV-6B" in both[A]["note"]), - (vr.CALL_FOUND, True)) + (CALL_FOUND, True)) # 5. no human depth nohum = run({B: flat(15)}, {B: 400}, human=[("chrX", 1000, 1000, 30.0)]) check("5 no autosome: human depth '.'", nohum[B]["human_depth"], ".") check("5 and integration is not assessed", (nohum[B]["call"], "integration is not assessed" in nohum[B]["note"]), - (vr.CALL_FOUND, True)) + (CALL_FOUND, True)) # 6. the human depth -hd = vr.human_depth({"chr1": {"length": 1000, "covbases": 500, "meandepth": 10.0, "reads": 0}, - "22": {"length": 1000, "covbases": 1000, "meandepth": 40.0, "reads": 0}, - "chrX": {"length": 1000, "covbases": 1000, "meandepth": 99.0, "reads": 0}, - "chrUn_KI270302v1": {"length": 1000, "covbases": 1000, "meandepth": 99.0, "reads": 0}, - "chrEBV": {"length": 1000, "covbases": 1000, "meandepth": 99.0, "reads": 0}}) -check("6 human depth over autosomal bases with reads: (10*1000 + 40*1000) / 1500", round(hd, 4), round(50000 / 1500, 4)) +hd = run(human=[("chr1", 1000, 500, 10.0), ("22", 1000, 1000, 40.0), ("chrX", 1000, 1000, 99.0), + ("chrUn_KI270302v1", 1000, 1000, 99.0), ("chrEBV", 1000, 1000, 99.0)])[B]["human_depth"] +check("6 human depth over autosomal bases with reads: (10*1000 + 40*1000) / 1500", hd, f"{50000 / 1500:.2f}") # 7. damaged input @@ -185,9 +195,9 @@ def drop_genome(d): gone = run({B: flat(15)}, {B: 400}, raw=drop_genome) check("7 a genome with no read that depth left out entirely is not found", - (gone[HCV]["call"], gone[HCV]["breadth"], gone[B]["call"]), (vr.CALL_NONE, "0.0000", vr.CALL_ICI)) + (gone[HCV]["call"], gone[HCV]["breadth"], gone[B]["call"]), (CALL_NONE, "0.0000", CALL_ICI)) err = run(raw=cut_depth) -check("7 a depth file with positions missing stops the script", isinstance(err, str) and "samtools depth -aa" in err, True) +check("7 a depth file with positions missing stops og", isinstance(err, str) and "samtools depth -aa" in err, True) err = run(raw=bad_cov) check("7 a coverage file that is not samtools coverage output stops it", isinstance(err, str) and "not samtools coverage output" in err, True) @@ -195,12 +205,19 @@ def drop_genome(d): # 8. the summary and the text report import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 + + +def og_report(d): + """og viral report on the inputs in d, into d/S/viral/S_viral.tsv.""" + subprocess.run([OG, "viral", "report", "--sample", "S", "--viral-fasta", f"{d}/v.fa", "--viral-coverage", + f"{d}/vcov.tsv", "--viral-depth", f"{d}/vdepth.tsv", "--human-coverage", f"{d}/hcov.tsv", + "--out", f"{d}/S/viral/S_viral.tsv"], check=True, capture_output=True) + + with tempfile.TemporaryDirectory() as d: write_inputs(d, {B: flat(15), EBV: flat(1), PHIX: flat(3)}, {B: 400, EBV: 30, PHIX: 20}) os.makedirs(f"{d}/S/viral") - vr.main(["--sample", "S", "--viral-fasta", f"{d}/v.fa", "--viral-coverage", f"{d}/vcov.tsv", - "--viral-depth", f"{d}/vdepth.tsv", "--human-coverage", f"{d}/hcov.tsv", - "--out", f"{d}/S/viral/S_viral.tsv"]) + og_report(d) path, data = cs.sec_viral(f"{d}/S", "S") check("8 the summary finds the table", path.endswith("viral/S_viral.tsv"), True) check("8 the summary's HHV-6 line", data["hhv6"], "inherited integrated HHV-6B likely") @@ -226,9 +243,7 @@ def drop_genome(d): with tempfile.TemporaryDirectory() as d: write_inputs(d, {EBV: flat(1)}, {EBV: 30}, human=human) os.makedirs(f"{d}/S/viral") - vr.main(["--sample", "S", "--viral-fasta", f"{d}/v.fa", "--viral-coverage", f"{d}/vcov.tsv", - "--viral-depth", f"{d}/vdepth.tsv", "--human-coverage", f"{d}/hcov.tsv", - "--out", f"{d}/S/viral/S_viral.tsv"]) + og_report(d) data = cs.sec_viral(f"{d}/S", "S")[1] check(f"8 the summary's HHV-6 line reads {want!r}", data["hhv6"], want) if human: @@ -239,12 +254,10 @@ def drop_genome(d): with tempfile.TemporaryDirectory() as d: write_inputs(d, {HCV: lambda i: 1 if i < 50 else 0}, {HCV: 5}) os.makedirs(f"{d}/S/viral") - vr.main(["--sample", "S", "--viral-fasta", f"{d}/v.fa", "--viral-coverage", f"{d}/vcov.tsv", - "--viral-depth", f"{d}/vdepth.tsv", "--human-coverage", f"{d}/hcov.tsv", - "--out", f"{d}/S/viral/S_viral.tsv"]) + og_report(d) data = cs.sec_viral(f"{d}/S", "S")[1] check("8 a genome with 5 reads is listed", [(x["virus"], x["call"], x["reads"]) for x in data["found"]], - [("Hepatitis C virus genotype 1", vr.CALL_NONE, "5")]) + [("Hepatitis C virus genotype 1", CALL_NONE, "5")]) check("8 and the text does not say no read", any(ln.startswith("No read") for ln in rr.viral_lines(data)), False) if FAILS: diff --git a/tests/test_vntyper_report.py b/tests/test_vntyper_report.py index bec92ec2e..c4341a997 100644 --- a/tests/test_vntyper_report.py +++ b/tests/test_vntyper_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/vntyper_report.py: step 52's reading of VNtyper's MUC1 VNTR result, +"""og vntyper report: step 52's reading of VNtyper's MUC1 VNTR result, and its section in both reports. On VNtyper run folders made up for each case, in the layout VNtyper 2.0.47 @@ -25,12 +25,14 @@ names the MUC1 VNTR when step 52 did not run or gave no reading, and not when it read positive or negative; no card without step 52's table; a table older than a run-all.sh run that did not run step 52 is stale; - 7. control: a copy of the script that calls every Kestrel call positive - fails case 2, so these checks can fail. + 7. control: a reading that calls every Kestrel call positive fails case + 2, so these checks can fail. -Run: python3 tests/test_vntyper_report.py +Without OG (the og binary) the test is skipped with a note; with +OG_REQUIRED set too, it fails (Guard's rust job sets both). + +Run: OG=bin/og python3 tests/test_vntyper_report.py """ -import importlib.util import os import subprocess import sys @@ -38,9 +40,14 @@ import time REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "vntyper_report.py") +OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) -import vntyper_report as vr # noqa: E402 + +FIELDS = ("status", "confidence", "variant", "flag", "alt_depth", "region_depth", "depth_score", + "vntr_mean_depth", "vntr_uncovered_pct", "coverage_qc", "sensitivity", "message", "advice") +ADVICE_NONE = ("A negative here lowers the chance of the frameshifts VNtyper looks for; it does not rule " + "ADTKD-MUC1 out. A family history of kidney failure with gout or a bland urine sediment " + "still deserves a clinical test (SNaPshot for dupC).") FAILS = 0 @@ -77,75 +84,100 @@ def call(conf="High_Precision*", flag="Not flagged", name="59dupC"): return POS_HEADER + f"2\tInsertion\t67\tG\tGC\t416\t7110\t0.0585\t{conf}\t{flag}\t3\t{name}\tCCCCACC\n" -def cases(mod, t): - """(description, ok, detail) for cases 1 to 4 run with module mod.""" +def og(*args): + return subprocess.run([OG, "vntyper", "report", *args], capture_output=True, text=True) + + +def og_read(d, exit_code=0, log_text=""): + """og vntyper report on run folder d: the table as a dict.""" + out, log = d + ".tsv", d + ".log" + with open(log, "w") as f: + f.write(log_text) + p = og("--run-dir", d, "--exit-code", str(exit_code), "--log", log, "--out", out) + if p.returncode: + raise SystemExit(f"og vntyper report failed: {p.stderr}") + return dict(line.rstrip("\n").split("\t", 1) for line in open(out).readlines()[1:]) + + +def every_call_positive(d, exit_code=0, log_text=""): + """The control: a reading that calls every Kestrel call positive.""" + r = og_read(d, exit_code, log_text) + if r["status"] == "possible": + r["status"] = "positive" + return r + + +def cases(read, t): + """(description, ok, detail) for cases 1 to 4 read with read.""" out = [] - r = mod.read(run_dir(t, "pos", call()), 0) + r = read(run_dir(t, "pos", call()), 0) out.append(("1. High_Precision*, not flagged, 1,075x: positive with the name", r["status"] == "positive" and r["variant"] == "59dupC" and r["confidence"] == "High_Precision*" and r["alt_depth"] == "416", r)) out.append(("1. the sensitivity line names the depth and the 100x band (160 of 161)", "1075x" in r["sensitivity"] and "160 of 161 runs (99%)" in r["sensitivity"], r["sensitivity"])) - r = mod.read(run_dir(t, "low", call(conf="Low_Precision")), 0) + r = read(run_dir(t, "low", call(conf="Low_Precision")), 0) out.append(("2. a Low_Precision call is possible, not positive", r["status"] == "possible" and "low precision" in r["message"], r)) - r = mod.read(run_dir(t, "flag", call(flag="False_Positive_4bp_Insertion")), 0) + r = read(run_dir(t, "flag", call(flag="False_Positive_4bp_Insertion")), 0) out.append(("2. a flagged High_Precision call is possible, with the flag", r["status"] == "possible" and "False_Positive_4bp_Insertion" in r["message"], r)) - r = mod.read(run_dir(t, "holes", call(), uncovered="62.00", qc="FAIL"), 0) + r = read(run_dir(t, "holes", call(), uncovered="62.00", qc="FAIL"), 0) out.append(("2. a call over a VNTR 62% without reads is possible", r["status"] == "possible" and "62% of the VNTR has no read" in r["message"], r)) - r = mod.read(run_dir(t, "nounc", call(), uncovered="NA"), 0) + r = read(run_dir(t, "nounc", call(), uncovered="NA"), 0) out.append(("2. a call with no uncovered share from VNtyper is possible, not positive", r["status"] == "possible" and "no share of the VNTR without reads" in r["message"], r)) - r = mod.read(run_dir(t, "neg", NEG, mean="147.94"), 0) + r = read(run_dir(t, "neg", NEG, mean="147.94"), 0) out.append(("3. no call at 148x, coverage PASS: negative, 100x band", r["status"] == "negative" and r["variant"] == "." and "160 of 161" in r["sensitivity"] - and r["advice"] == mod.ADVICE_NONE, r)) - r = mod.read(run_dir(t, "red", NEG, mean="70.00", qc="REDUCED"), 0) + and r["advice"] == ADVICE_NONE, r)) + r = read(run_dir(t, "red", NEG, mean="70.00", qc="REDUCED"), 0) out.append(("3. no call at 70x, coverage REDUCED: negative, 50x-100x band (95 of 108)", r["status"] == "negative" and "95 of 108 runs (88%)" in r["sensitivity"] and "between 50x and 100x" in r["sensitivity"], r)) - r = mod.read(run_dir(t, "thin", NEG, mean="20.00", qc="FAIL"), 0) + r = read(run_dir(t, "thin", NEG, mean="20.00", qc="FAIL"), 0) out.append(("3. no call at 20x, coverage FAIL: not assessed, never negative, below-50x band (48 of 109)", r["status"] == "not assessed" and "cannot be ruled out" in r["message"] and "48 of 109 runs (44%)" in r["sensitivity"], r)) - r = mod.read(run_dir(t, "na", NEG, mean="NA", qc="NOT_EVALUATED"), 0) + r = read(run_dir(t, "na", NEG, mean="NA", qc="NOT_EVALUATED"), 0) out.append(("3. no measured depth: not assessed, sensitivity not known", r["status"] == "not assessed" and r["sensitivity"].startswith("Sensitivity not known"), r)) - r = mod.read(run_dir(t, "rc", call()), 1) + r = read(run_dir(t, "rc", call()), 1) out.append(("4. VNtyper's exit code 1: not assessed, whatever it wrote", r["status"] == "not assessed" and "exit code 1" in r["message"], r)) - r = mod.read(run_dir(t, "empty"), 1, "ValueError: FASTQ layout 'empty' cannot be consumed without dropping reads.") + r = read(run_dir(t, "empty"), 1, "ValueError: FASTQ layout 'empty' cannot be consumed without dropping reads.") out.append(("4. exit 1 on a BAM with no read over MUC1: not assessed, saying so", r["status"] == "not assessed" and "no read in the MUC1 region" in r["message"], r)) - r = mod.read(run_dir(t, "hdr", NEG.split("\n")[0] + "\n", mean="147.94"), 0) + r = read(run_dir(t, "hdr", NEG.split("\n")[0] + "\n", mean="147.94"), 0) out.append(("4. a Kestrel file with a header and no result row at 148x: not assessed, never negative", r["status"] == "not assessed" and "no result row" in r["message"], r)) - r = mod.read(run_dir(t, "nok"), 0) + r = read(run_dir(t, "nok"), 0) out.append(("4. no kestrel_result.tsv: not assessed, naming the file", r["status"] == "not assessed" and "kestrel/kestrel_result.tsv" in r["message"], r)) return out def main(): + if not OG: + print("skip: OG is not set (OG=bin/og)") + return 1 if os.environ.get("OG_REQUIRED") else 0 with tempfile.TemporaryDirectory() as t: - for desc, ok, detail in cases(vr, os.path.join(t, "real")): + for desc, ok, detail in cases(og_read, os.path.join(t, "real")): check(desc, ok, detail) # 5. the CLI d = run_dir(t, "cli", call()) out_tsv = os.path.join(t, "cli.tsv") - p = subprocess.run([sys.executable, SCRIPT, "--run-dir", d, "--out", out_tsv], capture_output=True, text=True) + p = og("--run-dir", d, "--out", out_tsv) rows = [line.rstrip("\n").split("\t") for line in open(out_tsv)] if os.path.exists(out_tsv) else [] check("5. the CLI exits 0 and writes metric, value and every field once", p.returncode == 0 and rows[:1] == [["metric", "value"]] - and [r[0] for r in rows[1:]] == list(vr.FIELDS) and all(len(r) == 2 for r in rows), (p, rows)) + and [r[0] for r in rows[1:]] == list(FIELDS) and all(len(r) == 2 for r in rows), (p, rows)) bad = run_dir(t, "bad", "Confidence\n\"x\n") with open(os.path.join(bad, "kestrel", "kestrel_result.tsv"), "wb") as f: f.write(b"\xff\xfe\x00bad") - p = subprocess.run([sys.executable, SCRIPT, "--run-dir", bad, "--out", os.path.join(t, "bad.tsv")], - capture_output=True, text=True) + p = og("--run-dir", bad, "--out", os.path.join(t, "bad.tsv")) check("5. an unreadable result file exits 2", p.returncode == 2 and "cannot read" in p.stderr, p) # 6. the reports @@ -155,8 +187,8 @@ def main(): def sample(name, kestrel, **kw): sd = os.path.join(t, "s", name, "S") os.makedirs(os.path.join(sd, "vntyper")) - vr.main(["--run-dir", run_dir(t, "r" + name, kestrel, **kw), - "--out", os.path.join(sd, "vntyper", "S_vntyper.tsv")]) + og("--run-dir", run_dir(t, "r" + name, kestrel, **kw), + "--out", os.path.join(sd, "vntyper", "S_vntyper.tsv")) s = collect_summary.collect("S", sd) return s, render_report.text_report(s), render_report.html_report(s) @@ -193,18 +225,9 @@ def sample(name, kestrel, **kw): check("6. and both reports list it under what was not assessed", line in render_report.text_report(s) and render_report.E(line) in render_report.html_report(s)) - # 7. control: a copy that calls every Kestrel call positive - src = open(SCRIPT).read() - cond = 'conf.startswith("High_Precision") and not flagged' - check("7. control: the condition to break is in the script", cond in src) - broken = os.path.join(t, "vntyper_report_broken.py") - with open(broken, "w") as f: - f.write(src.replace(cond, "True")) - spec = importlib.util.spec_from_file_location("vntyper_report_broken", broken) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - failed = [desc for desc, ok, _ in cases(mod, os.path.join(t, "broken")) if not ok] - check("7. control: the broken copy fails the possible-not-positive cases", + # 7. control: a reading that calls every Kestrel call positive + failed = [desc for desc, ok, _ in cases(every_call_positive, os.path.join(t, "broken")) if not ok] + check("7. control: the broken reading fails the possible-not-positive cases", any(x.startswith("2.") for x in failed), failed) print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py index 47f146e87..77c49ea30 100644 --- a/tests/test_ystr_report.py +++ b/tests/test_ystr_report.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""bin/ystr_report.py: Y-STR repeat counts from HipSTR's calls (step 45). +"""og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). Each case writes a synthetic HipSTR VCF (one sample, only the records it -plants), runs the script on assets/ystr_panel.tsv, and checks the table: +plants), runs og y str-report on assets/ystr_panel.tsv, and checks the table: 1. the panel: 111 markers in FamilyTreeDNA's order (the Y-12, Y-25 and Y-37 heads checked by name), every multi-copy marker kind multicopy, @@ -29,7 +29,10 @@ bin/render_report.py lists the called markers only, with the note, and the HTML report shows an unreadable table as Unreadable -Run: python3 tests/test_ystr_report.py +Without OG (the og binary) the test is skipped with a note; with +OG_REQUIRED set too, it fails (Guard's rust job sets both). + +Run: OG=bin/og python3 tests/test_ystr_report.py """ import atexit import csv @@ -41,10 +44,12 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO, "bin", "ystr_report.py") +OG = os.environ.get("OG", "") PANEL = os.path.join(REPO, "assets", "ystr_panel.tsv") sys.path.insert(0, os.path.join(REPO, "bin")) -import ystr_report as yr # noqa: E402 +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) FAILS = [] @@ -57,6 +62,24 @@ def check(name, got, want): FAILS.append(name) +def read_panel(path): + """The panel's rows as dicts, each region as (chrom, start, end, period, copies, name); a span + marker has one region over its whole locus, its parts' copies rounded half up and summed.""" + with open(path, encoding="utf-8") as f: + lines = [line.rstrip("\n").split("\t") for line in f if line.strip() and not line.startswith("#")] + rows = [dict(zip(lines[0], c)) for c in lines[1:]] + for r in rows: + r["regions"] = [] if r["regions"] == "." else [ + (where.split(":")[0], int(where.split(":")[1].split("-")[0]), int(where.split(":")[1].split("-")[1]), + int(period), float(copies), name) + for where, period, copies, name in (x.split("/", 3) for x in r["regions"].split(";"))] + if r["kind"] == "span": # one region over the whole locus, named after the marker + regs = r["regions"] + r["regions"] = [(regs[0][0], regs[0][1], regs[-1][2], regs[0][3], + float(sum(int(x[4] + 0.5) for x in regs)), r["marker"])] + return rows + + TMP = tempfile.mkdtemp(prefix="ystr-test-") atexit.register(shutil.rmtree, TMP, ignore_errors=True) FORMAT = "GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL" @@ -64,7 +87,7 @@ def check(name, got, want): def vcf(records, samples=("S1",), gz=True): """records: {HipSTR name: (GT, GB, Q, DP, DSTUTTER, DFLANKINDEL)}.""" - panel = yr.read_panel(PANEL) + panel = read_panel(PANEL) where = {n: (c, s) for r in panel for c, s, _e, _p, _k, n in r["regions"]} lines = ["##fileformat=VCFv4.1", "##source=HipSTR", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(samples)] @@ -83,7 +106,7 @@ def vcf(records, samples=("S1",), gz=True): def run(records=None, panel=PANEL, read_length=None, samples=("S1",), want_rc=0, log=None): v = vcf(records or {}, samples) tsv, out_csv = v + ".tsv", v + ".csv" - cmd = [sys.executable, SCRIPT, "report", "--panel", panel, "--vcf", v, "--tsv", tsv, "--csv", out_csv] + cmd = [OG, "y", "str-report", "--panel", panel, "--vcf", v, "--tsv", tsv, "--csv", out_csv] if read_length: cmd += ["--read-length", str(read_length)] if log is not None: @@ -112,7 +135,7 @@ def good(gb="0", dp="20"): # --- 1. the panel -------------------------------------------------------------------- -panel = yr.read_panel(PANEL) +panel = read_panel(PANEL) names = [r["marker"] for r in panel] check("1 111 markers", len(panel), 111) check("1 Y-12 in FamilyTreeDNA's order", names[:12], @@ -132,7 +155,7 @@ def good(gb="0", dp="20"): {"DYS385a", "DYS385b", "DYS459a", "DYS459b", "DYS464a", "DYS464b", "DYS464c", "DYS464d", "YCAIIa", "YCAIIb", "CDYa", "CDYb", "DYF395S1a", "DYF395S1b", "DYS413a", "DYS413b"}) bed = os.path.join(TMP, "regions.bed") -check("1 bed exits 0", yr.main(["bed", "--panel", PANEL, "--out", bed]), 0) +check("1 bed exits 0", subprocess.run([OG, "y", "str-bed", "--panel", PANEL, "--out", bed]).returncode, 0) with open(bed) as f: bl = [line.rstrip("\n").split("\t") for line in f] want_regions = {n for r in panel for *_x, n in r["regions"]} @@ -285,8 +308,9 @@ def good(gb="0", dp="20"): sd = os.path.join(TMP, "sample") os.makedirs(os.path.join(sd, "y_str")) v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) -yr.main(["report", "--panel", PANEL, "--vcf", v, "--tsv", os.path.join(sd, "y_str", "S1_y_str.tsv"), - "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")]) +subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", + os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], + check=True, capture_output=True) path, data = cs.sec_y_str(sd, "S1") check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), From 57983211bfaffaf4b8d85dbf81ed61780dd96bf2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:16 +0100 Subject: [PATCH 25/53] refactor(reports): delete the seven Python step reports og replaces them. The docs, the changelog and the comments that named them now name og. Bead: pgp-bs9u.17 --- CHANGELOG.md | 9 + assets/mei_sites.tsv | 2 +- bin/collect_summary.py | 14 +- bin/exomiser_report.py | 451 ---------------------------------------- bin/locityper_report.py | 193 ----------------- bin/mei_report.py | 260 ----------------------- bin/mocha_report.py | 256 ----------------------- bin/viral_report.py | 227 -------------------- bin/vntyper_report.py | 160 -------------- bin/ystr_report.py | 305 --------------------------- docs/45-y-str.md | 2 +- docs/47-viral.md | 4 +- docs/48-exomiser.md | 2 +- docs/50-mei.md | 6 +- docs/51-mocha.md | 4 +- docs/52-vntyper.md | 4 +- docs/53-locityper.md | 4 +- docs/rust.md | 3 +- docs/testing.md | 2 +- rust/README.md | 1 + 20 files changed, 34 insertions(+), 1875 deletions(-) delete mode 100755 bin/exomiser_report.py delete mode 100755 bin/locityper_report.py delete mode 100755 bin/mei_report.py delete mode 100755 bin/mocha_report.py delete mode 100755 bin/viral_report.py delete mode 100755 bin/vntyper_report.py delete mode 100755 bin/ystr_report.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a197c3414..ecb0c8e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The opt-in steps 45, 47, 48 and 50 to 53 now read their tool's output with the Rust `og` binary: `og y str-bed` and `og y str-report` (step 45), `og viral report` (47), `og exomiser prepare` and `og exomiser report` (48), `og mei report` and `og mei annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53). They replace `bin/ystr_report.py`, `bin/viral_report.py`, `bin/exomiser_report.py`, `bin/mei_report.py`, `bin/mocha_report.py`, `bin/vntyper_report.py` and `bin/locityper_report.py`, which are gone. og is static, so each step still runs in the image it ran in. The tables are the same, byte for byte, on the 123 parity cases, except where a line below says otherwise. Every output is written whole or not at all, and an input a step cannot read stops it with one `ERROR:` line, not a Python traceback. + - Step 48: the summary printed at the end of `og exomiser report` has one line per rank. The Python cut it by variant count, so a rank with several variants could push the closing note in or out. The text file is unchanged. + - Step 50: the depth of each catalog window is its own. The Python looked it up by gene and chromosome, so two windows of one gene on one chromosome both got the last one's depth. The shipped catalog has no such pair, so its results do not change. + - Step 51: `calls_mocha` in the summary counts this sample's calls only. The Python counted every row of MoChA's call table, so a table of several samples gave a wrong count. A call row shorter than the header now stops the step with exit 2 and the file and line. + - Step 52: a coverage table with a negative mean depth stops the step with exit 2 and says so. The Python crashed. + - Step 53: a `res.json.gz` that holds JSON but not an object, or a genotype that is not text, now reads "not called" with the reason. The Python crashed and wrote no table. A loci BED start written in non-ASCII digits is refused with its line. + - Step 45: a panel region with a motif length under 1, or a negative copy number, is refused with the marker's name. The Python crashed on the first and used the second. + - Steps 47, 50 and 51: an unreadable input stops with `ERROR: ` on stderr, where the Python wrote `
Primary callerPrimary caller