diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 05cf8d80e..11fade087 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,9 +65,9 @@ on: - 'nextflow_schema.json' - 'versions.env' - '.github/workflows/e2e.yml' + - 'rust/crates/og-dev/src/fixture.rs' - '.github/actions/**' - 'tests/fixtures/annotsv/class45.sh' - - 'tests/fixtures/comphet/make_fixture.py' - 'tests/fixtures/consumer_export/sites.tsv.gz' - 'tests/fixtures/families/stub_families.tsv' - 'tests/fixtures/pharmcat/refblocks.g.vcf' diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index ccadb79b0..e5395b89f 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -315,6 +315,9 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + - name: Evidence offset regression in release mode + run: cargo test --release --locked -p og-dev --no-fail-fast fai_offsets_reject_overflow_and_out_of_file_before_publication + - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation) if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows run: | diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..aa6e7dcc7 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -166,6 +166,11 @@ jobs: name: archaic-1kg-sites path: sites + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --archaic and step 43 on HG002 (chromosome ${{ matrix.part }}) id: run continue-on-error: ${{ matrix.part == 'empty' }} @@ -230,47 +235,7 @@ jobs: # same-position rule of archaic_report.py decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz - python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' - import glob, gzip, os, sys - raw, split, data, out = sys.argv[1:] - tables = sorted(glob.glob(os.path.join(data, "chr*.tsv.gz"))) - chroms = {os.path.basename(t)[:-7] for t in tables} # this part's chromosomes only - def clashes(path): - """Positions of this part with two records or more, one an SNV (the VCF is sorted).""" - hit, key, n, snvs = set(), None, 0, 0 - for line in gzip.open(path, "rt"): - if line.startswith("#"): - continue - c = line.split("\t", 6) - if c[0] not in chroms: - continue - k = (c[0], int(c[1])) - if k != key: - key, n, snvs = k, 0, 0 - n += 1 - snvs += len(c[3]) == 1 and len(c[4]) == 1 - if n >= 2 and snvs >= 1: - hit.add(key) - return hit - sets = {"raw": clashes(raw), "split": clashes(split)} - on_table = {k: 0 for k in sets} - for t in tables: - chrom = os.path.basename(t)[:-7] - want = {k: {p for c, p in v if c == chrom} for k, v in sets.items()} - if not any(want.values()): - continue - for line in gzip.open(t, "rt"): - c = line.split("\t", 2) - if c[0] != chrom: - continue - p = int(c[1]) - for k in want: - if p in want[k]: - on_table[k] += 1 - with open(out, "w") as f: - for k in sets: - f.write(f"{k}_positions\t{len(sets[k])}\n{k}_on_table\t{on_table[k]}\n") - PY + bin/og dev evidence clashes "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" cat "$OUT/affected.tsv" - name: The red control failed, as it must @@ -597,6 +562,11 @@ jobs: gzip -dc "${R}/${f}.gz" > "${R}/GRCh38_no_alt_analysis_set.fasta" rm -f "${R}/${f}.gz" + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --ancient-panel and step 44 on HG002, measured env: THREADS: '4' @@ -672,57 +642,9 @@ jobs: mkdir -p "${G}/HG002/vcf" curl -fsSL --retry 5 -o "${EV}/truth.vcf.gz" "${GIAB_HG002}.vcf.gz" curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed" - python3 -I - "${EV}/truth.vcf.gz" "${EV}/truth.bed" "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \ - "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" <<'PY' - import bisect, gzip, sys - truth, bed, ref, sites, out = sys.argv[1:] - fai = {l.split("\t")[0]: [int(x) for x in l.split("\t")[1:5]] for l in open(ref + ".fai")} - fa = open(ref, "rb") - def base(c, p): - n, off, lb, lw = fai[c] - fa.seek(off + ((p - 1) // lb) * lw + (p - 1) % lb) - return fa.read(1).decode().upper() - conf = {} - for l in open(bed): - c, s, e = l.split()[:3] - conf.setdefault(c, []).append((int(s), int(e))) - starts = {c: [s for s, _ in sorted(v)] for c, v in conf.items()} - ends = {c: [e for _, e in sorted(v)] for c, v in conf.items()} - def confident(c, p): - i = bisect.bisect_right(starts.get(c, []), p - 1) - 1 - return i >= 0 and ends[c][i] >= p - covered, head, body = {}, [], [] - for l in gzip.open(truth, "rt"): - if l.startswith("##"): - head.append(l) - continue - if l.startswith("#"): - head.append('##INFO=\n') - head.append('##ALT=\n') - head.append(l) - continue - f = l.split("\t", 5) - p = int(f[1]) - covered.setdefault(f[0], set()).update(range(p, p + len(f[3]))) - body.append(l) - blocks = 0 - with open(sites) as s: - next(s) - for l in s: - _, c, p = l.split("\t")[:3] - p = int(p) - if c not in fai or p in covered.get(c, ()) or not confident(c, p): - continue - b = base(c, p) - if b not in "ACGT": - continue - body.append(f"{c}\t{p}\t.\t{b}\t<*>\t.\t.\tEND={p}\tGT\t0/0\n") - blocks += 1 - with open(out, "w") as o: - o.writelines(head) - o.writelines(body) - print(f"gVCF: {len(body) - blocks} GIAB records, {blocks} reference blocks at panel sites") - PY + /usr/bin/time -v -o "${M}/gvcf.time" bin/og dev evidence hg002-gvcf "${EV}/truth.vcf.gz" "${EV}/truth.bed" \ + "${G}/reference/GRCh38_no_alt_analysis_set.fasta" "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" + grep -E 'Maximum resident|Elapsed' "${M}/gvcf.time" docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/ev" -v "${G}:/g" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' bcftools sort -m 2G -T /ev/sort -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /ev/hg002.unsorted.vcf bcftools index -t /g/HG002/vcf/HG002.g.vcf.gz' diff --git a/.github/workflows/renovate-dry-run.yml b/.github/workflows/renovate-dry-run.yml index 5722cb025..155568c19 100644 --- a/.github/workflows/renovate-dry-run.yml +++ b/.github/workflows/renovate-dry-run.yml @@ -6,7 +6,8 @@ name: Renovate dry run # matcher that misses an image, or a versioning that cannot read a tag, shows # up here first. The full debug log is kept as an artifact for 14 days. # -# The checks (each one has a planted fault in the self-test step): +# The checks (og dev renovate-check; each one has a planted fault in its +# self-test, which runs after the checks): # count every *_IMAGE line of versions.env is a detected docker dependency # names no package name carries a ':' or '@' (a digest line once became # a package named '...@sha256') @@ -16,9 +17,10 @@ name: Renovate dry run # one-pr a tool gets the same proposal in every file it is in # pins every non-image pin (Nextflow, actionlint, gitleaks, ShellCheck, # the Renovate CLI, the HLA database, pgsc_calc, the demo's Python -# packages, Aldy, the docs workflow's Python) is detected in every -# place it is pinned, as often as the text has them. A setup-nextflow -# step counts as a pin only when its version: line is a literal +# packages, Aldy, the docs workflow's Python; one row each in +# scripts/ci/renovate-pins.tsv) is detected in every place it is +# pinned, as often as the text has them. A setup-nextflow step +# counts as a pin only when its version: line is a literal # version, quoted or not; a computed one (an expression) is not a pin # lookups no lookup failed and no tag was unreadable; the only skipped # dependencies are the ones a rule in renovate.json disables @@ -31,7 +33,8 @@ on: # Every file a manager in renovate.json reads (the workflows hold the # Nextflow, actionlint, gitleaks, ShellCheck, Renovate and docs Python pins, # the docs the NXF_VER lines and the Aldy pin, scripts/lib/common.sh the HLA - # database, the demo script its Python packages), plus the doc generator. + # database, the demo script its Python packages), plus the doc generator, + # the pins table and the checker. # The lookups take about a minute. pull_request: branches: [main] @@ -42,6 +45,8 @@ on: - 'docs/*.md' - '.github/workflows/**' - 'scripts/ci/gen-versions-doc.sh' + - 'scripts/ci/renovate-pins.tsv' + - 'rust/crates/og-dev/src/renovate.rs' - 'scripts/lib/common.sh' - 'tests/demo/retake-screenshots.sh' push: @@ -91,7 +96,7 @@ jobs: lookup: # Public repo: GitHub-hosted runners only. runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 env: RENOVATE_VERSION: 44.199.2 steps: @@ -129,222 +134,14 @@ jobs: path: ${{ runner.temp }}/renovate.log retention-days: 14 - - name: Write the checker + # The checks are og dev renovate-check (rust/crates/og-dev/src/renovate.rs), + # with the non-image pins in scripts/ci/renovate-pins.tsv. + - name: Build og as scripts/setup.sh --og does if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} run: | - cat > "${RUNNER_TEMP}/check.py" <<'PY' - import collections, copy, glob, json, os, re, sys - - ENV = "versions.env" - TWINS = ("docs/versions.md", "conf/containers.config") - RULE_SKIPS = {"disabled", "package-rules"} - # The non-image pins: (dependency, files, how a pin of it looks in that text). - # A setup-nextflow step is a pin when a version: line with a literal - # version follows its uses: line inside the same step (the lines in - # between are blank or start with anything but '-', which opens the - # next step). - SETUP_NEXTFLOW = r'uses: ["\x27]?nf-core/setup-nextflow@[^\n]*\n(?:[ \t]*(?:[^\s-][^\n]*)?\n)*?[ \t]+version: ["\x27]?\d' - DEMO = "tests/demo/retake-screenshots.sh" - PINS = ( - ("nextflow-io/nextflow", [ENV], r'^NEXTFLOW_VERSION="'), - ("nextflow-io/nextflow", sorted(glob.glob(".github/workflows/*.yml")), SETUP_NEXTFLOW), - ("nextflow-io/nextflow", sorted(glob.glob("docs/*.md")), r"NXF_VER=\d"), - ("rhysd/actionlint", sorted(glob.glob(".github/workflows/*.yml")), r"ACTIONLINT_VERSION=\d"), - ("gitleaks/gitleaks", sorted(glob.glob(".github/workflows/*.yml")), r"GITLEAKS_VERSION=\d"), - ("koalaman/shellcheck", [".github/workflows/lint.yml"], r"^[ \t]+version: v\d"), - ("renovate", [".github/workflows/renovate-dry-run.yml"], r"RENOVATE_VERSION: \d"), - ("renovate", ["docs/updating.md"], r"renovate@\d"), - ("python", [".github/workflows/docs.yml"], r"python-version: ['\"]?\d"), - ("ANHIG/IMGTHLA", ["scripts/lib/common.sh"], r"^HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-\d"), - ("PGScatalog/pgsc_calc", [ENV], r'^PGSC_CALC_VERSION="'), - ("PGScatalog/pgsc_calc", ["conf/containers.config"], r"ext\.pipeline_version = '"), - ("playwright", [DEMO], r"playwright==\d"), - ("pillow", [DEMO], r"pillow==\d"), - ("pyobjc-framework-Vision", [DEMO], r"pyobjc-framework-Vision==\d"), - ("aldy", ["docs/21-cyrius.md"], r"aldy==\d|\[Aldy\]\([^)]*\) v\d"), - ) - # Only the files a pin is listed for are counted: python is also the - # docker image of versions.env, which the image checks cover. - PIN_FILES = collections.defaultdict(set) - for _name, _files, _ in PINS: - PIN_FILES[_name].update(_files) - - def pin_counts(): - want = collections.Counter() - for name, files, pattern in PINS: - for f in files: - n = len(re.findall(pattern, open(f).read(), re.M)) - if n: - want[(name, f)] += n - return want - - def load(log): - files = None - for line in open(log): - try: - rec = json.loads(line) - except ValueError: - continue - if rec.get("msg") == "packageFiles with updates": - files = rec["config"] - if files is None: - sys.exit("FAIL: no 'packageFiles with updates' record in %s" % log) - return [(pf["packageFile"], dep) for entries in files.values() for pf in entries for dep in pf.get("deps", [])] - - def env_images(text): - out = [] - for m in re.finditer(r'^[A-Z0-9_]+_IMAGE="([^"]*)"', text, re.M): - # name:tag@sha256:..., name@sha256:... or name:tag -> name - out.append(re.sub(r":[^:/]*$", "", m.group(1).split("@", 1)[0])) - return out - - def key(dep): - return (dep.get("depName"), dep.get("currentValue"), dep.get("currentDigest")) - - def proposals(dep): - return sorted((u.get("newValue") or "", u.get("newDigest") or "") for u in dep.get("updates", [])) - - def check(deps, env_text): - fails = [] - docker = [(f, d) for f, d in deps if d.get("datasource") == "docker"] - by_file = collections.defaultdict(list) - for f, d in docker: - by_file[f].append(d) - want = env_images(env_text) - got = [d.get("depName") for d in by_file[ENV]] - if len(got) != len(want): - fails.append("count: versions.env has %d *_IMAGE lines, Renovate found %d docker dependencies there" % (len(want), len(got))) - for name in sorted(set(want) - set(got)): - fails.append("count: %s (versions.env) is not a detected dependency" % name) - for f, d in docker: - if re.search(r"[:@]", d.get("depName") or ":"): - fails.append("names: %s: package name %r" % (f, d.get("depName"))) - env_keys = collections.Counter(key(d) for d in by_file[ENV]) - for f in TWINS: - keys = collections.Counter(key(d) for d in by_file[f]) - if f == "docs/versions.md" and keys != env_keys: - for k in sorted((keys - env_keys) | (env_keys - keys), key=str): - fails.append("twins: docs/versions.md and versions.env differ on %s" % (k,)) - else: - for k in sorted(set(keys) - set(env_keys), key=str): - fails.append("twins: %s has %s, which versions.env does not" % (f, k)) - seen = {} - for f, d in deps: - name = (d.get("datasource"), d.get("depName"), d.get("currentValue"), d.get("currentDigest")) - p = proposals(d) - if name in seen and seen[name][1] != p: - fails.append("one-pr: %s gets %s in %s but %s in %s" % (d.get("depName"), seen[name][1], seen[name][0], p, f)) - seen.setdefault(name, (f, p)) - for f, d in deps: - if d.get("warnings"): - fails.append("lookups: %s (%s): %s" % (d.get("depName"), f, "; ".join(w.get("message", str(w)) for w in d["warnings"]))) - if d.get("skipReason") and d["skipReason"] not in RULE_SKIPS: - fails.append("lookups: %s (%s) skipped: %s" % (d.get("depName"), f, d["skipReason"])) - got = collections.Counter((d.get("depName"), f) for f, d in deps - if f in PIN_FILES.get(d.get("depName"), ())) - for k in sorted(set(want_pins) | set(got)): - if want_pins[k] != got[k]: - fails.append("pins: %s in %s: the text pins it %d times, Renovate found %d" % (k[0], k[1], want_pins[k], got[k])) - return fails - - def self_test(deps, env_text): - ok = True - base = set(check(deps, env_text)) - def plant(label, fn, pattern): - nonlocal ok - d = copy.deepcopy(deps) - try: - fn(d) - except StopIteration: - print("self-test %-8s NOT PLANTED: no dependency to plant it on" % label) - ok = False - return - # Only a failure the plant added counts as caught. - hits = [x for x in check(d, env_text) if x.startswith(pattern) and x not in base] - print("self-test %-8s %s" % (label, "caught: " + hits[0] if hits else "NOT CAUGHT")) - ok = ok and bool(hits) - def first(d, f): - return next(x for g, x in d if g == f and x.get("datasource") == "docker") - def drop(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("datasource") == "docker")) - def digest_name(d): - first(d, ENV)["depName"] += "@sha256" - def stale_twin(d): - first(d, "docs/versions.md")["currentValue"] = "0.0.0-planted" - def split_pr(d): - first(d, "conf/containers.config")["updates"] = [{"newValue": "9.9.9-planted"}] - def drop_pin(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("depName") == "nextflow-io/nextflow")) - def broken_lookup(d): - first(d, ENV)["skipReason"] = "invalid-value" - plant("count", drop, "count:") - plant("names", digest_name, "names:") - plant("twins", stale_twin, "twins:") - plant("one-pr", split_pr, "one-pr:") - plant("pins", drop_pin, "pins:") - plant("lookups", broken_lookup, "lookups:") - # The setup-nextflow pattern itself, on planted workflow text. - step = (' - uses: %snf-core/setup-nextflow@%s%s # v3.0.1\n' - '%s' - ' if: steps.scope.outputs.run == \'true\'\n' - ' with:\n' - ' version: %s\n' - '\n' - ' - uses: actions/setup-java@%s # v6\n' - ' with:\n' - ' java-version: "17"\n') - for label, quote, gap, version, want in ( - ("computed", "", "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed, uses quoted", '"', "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed after a blank line", "", "\n", "$" + "{{ steps.nf.outputs.version }}", 0), - ("literal", "", "", '"25.10.8"', 1), - ("literal after a blank line", "", "\n", '"25.10.8"', 1), - ("single-quoted literal", "'", "", "'25.10.8'", 1), - ("bare literal, uses quoted", '"', "", "25.10.8", 1)): - text = step % (quote, "a" * 40, quote, gap, version, "b" * 40) - n = len(re.findall(SETUP_NEXTFLOW, text, re.M)) - print("self-test pin-text %-27s counted %d, want %d" % (label, n, want)) - ok = ok and n == want - clean = check(deps, env_text) - print("self-test clean %s" % ("passes" if not clean else "fails: %s" % clean[0])) - return ok - - def summary(deps): - rows, seen = [], set() - files = collections.defaultdict(set) - for f, d in deps: - files[(d.get("datasource"), d.get("depName"))].add(f) - for f, d in deps: - k = (d.get("datasource"), d.get("depName")) - if k in seen: - continue - seen.add(k) - cur = d.get("currentValue") or d.get("currentDigest") or "" - ups = ", ".join("%s (%s)" % (u.get("newValue") or u.get("newDigest"), u.get("updateType")) - for u in d.get("updates", [])) or "none" - note = d.get("skipReason") or "" - rows.append("| `%s` | `%s` | %s | %s | %d |" % (d.get("depName"), cur, ups, note, len(files[k]))) - return "\n".join(["| Dependency | Current | Proposed update | Skipped | Files |", "|---|---|---|---|---|"] + rows) - - deps = load(sys.argv[1]) - env_text = open(ENV).read() - want_pins = pin_counts() - table = summary(deps) - print(table) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: - out.write("## Renovate dry run\n\n" + table + "\n") - fails = check(deps, env_text) - for x in fails: - print("FAIL " + x) - if not self_test(deps, env_text): - fails.append("the self-test did not catch a planted fault") - print("FAIL the self-test did not catch a planted fault") - if fails: - sys.exit(1) - print("OK: %d dependencies; every check passed" % len(deps)) - PY + scripts/setup.sh --og + rust/og.sh check --required - name: Check what Renovate found if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} - run: python3 "${RUNNER_TEMP}/check.py" "${RUNNER_TEMP}/renovate.log" + run: bin/og dev renovate-check "${RUNNER_TEMP}/renovate.log" diff --git a/CHANGELOG.md b/CHANGELOG.md index 286d71a9c..02b24beab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- CI evidence rejects FASTA offset overflow and offsets outside the file before reading or publishing. Guard tests this in both debug and release builds. Every new dev verb fails on stdout or stderr write or flush errors. A truth VCF header after the first data record fails without replacing the destination. Additional deliberate refusals are documented and tested: zero-byte gzip inputs, zero-width unused FAI rows, non-ASCII or out-of-range annotation limits, non-writable staging parents, and gzip FASTA files with long FNAME headers. Publication replaces a file and leaves other hard links on the old inode. + +- CI evidence and Renovate inputs now enforce their original explicit gzip or plain format above strict UTF-8 and universal-newline handling. Renovate normalizes repository sources too and accepts falsy warnings and packages with no dependencies. Output publication follows destination symlinks, preserves existing permissions, rejects read-only files and directories, and uses the umask for new files. VNtyper uses private temporary directories with failure and signal cleanup, and GNU `mv -T` prevents publication inside a directory. Comphet stages and validates all destinations before publishing. Three renames are still not one transaction. Stricter parsing is deliberate and tested: BED refuses unsupported Unicode/control whitespace, Renovate refuses wrong-typed present values, PharmCAT refuses truthy non-list containers and non-object alleles, evidence positions must be ASCII u64 values, truth VCF rows need five columns, blank FAI rows fail, text records have a 16 MiB limit, and clash rows need a trailing VCF column after ALT. Truth VCF records get a final LF. `docs/rust.md` lists the error classes. + +- `og dev evidence` now rejects invalid UTF-8, truncated selected records, blank BED rows and missing panel headers. It reads LF, CRLF and CR line endings alike, with a 16 MiB limit per record. Single-file evidence and fixture outputs are published by rename after validation. Comphet stages and validates all three outputs before its first rename. Three renames are still not one transaction. A crash or a filesystem failure between renames can publish only part of the new set. `og dev renovate-check` rejects malformed log records and present fields of the wrong type, and fails if its configured step summary cannot be written. The PharmCAT trimmer consumes source genes as it trims them. The VNtyper downloader also accepts output paths that contain a newline. + + +- The Python that CI and the fixtures ran outside the pipeline now runs in the Rust `og` binary. The public evidence workflow runs `og dev evidence clashes` and `og dev evidence hg002-gvcf` instead of two inline Python programs; the Renovate dry run checks what Renovate found with `og dev renovate-check`, whose list of non-image pins is now data in `scripts/ci/renovate-pins.tsv`; `og dev fixture comphet` writes the compound-het fixture of the e2e case `comphet-phasing-1-reads`, byte for byte; and `og dev fixture trim-pharmcat` trims a PharmCAT report for `tests/fixtures/pharmcat`. `tests/fixtures/comphet/make_fixture.py`, `tests/fixtures/pharmcat/trim_report.py` and `tests/smoke/vntyper_testdata.py` are gone: VNtyper's two test BAMs now come from `tests/smoke/vntyper_testdata.sh`, one HTTP range request per BAM with curl, checked by gzip against the archive's CRC-32 and size and then against the pinned sha256. On GIAB's HG002 benchmark VCF (4,048,342 records) with the chr20-22 reference, `og dev evidence hg002-gvcf` writes the same gVCF in 3.5 s and 97 MB instead of 11.8 s and 3.8 GB, because it keeps the spans the records cover instead of every covered position. No pipeline output changes. What changes on purpose: + - `og dev evidence hg002-gvcf` skips a panel site past the end of its contig. The Python read the byte after the contig, and at the end of the file wrote a reference block with an empty REF. + - `og dev renovate-check` stops with exit 2 when a pin's file is missing or its glob matches no file. The Python counted such a pin in no file, so it could never fail. + - `og dev fixture trim-pharmcat` names itself in the trimmed report's `_fixture_note`. + - Renovate skips only malformed JSON lines. JSON reader limits (signed 128-bit integers, Unicode scalar strings and 1000 nesting levels) now fail before an earlier config can pass the check. Evidence, fixture and Renovate refusals preserve existing output bytes, modes and symlink targets. [The refusal rule](docs/rust.md#input-and-destination-refusal-rule) states the numeric and size limits, with a shared table of negative CLI controls. - The shared `og-io` line reader bounds memory by the longest record for CR-only files as well as LF and CRLF files. Smoke coordinates require signed integers, so decimal and nonfinite coordinates fail. Trace integer cells accept whitespace, signs and underscores but deliberately refuse Unicode digits and negative resources. Integers outside the target range fail. Duplicate task rows retain concrete exits. Smoke JSON checks require one document, numeric counts and copy numbers, array catalogs and string genotypes. Numeric depth strings stay supported; booleans are refused. The array and genotype rules deliberately reject malformed values Python accepted. Smoke fixtures build a missing default `og` on first use. Setup output goes to stderr so captured PharmCAT counts and CSV values stay clean. - CI readers now use `og dev pharmcat diplotypes`, `nf-trace` and `smoke`. Unreadable PharmCAT reports fail with exit 2 and are marked not assessed. Resource checks refuse trace globs and malformed numeric cells. The smoke mini-reference uses a fixed Rust seed for the planted bases. Gzip smoke reads have deterministic headers. Smoke table checks report concise property results. - Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. diff --git a/docs/rust.md b/docs/rust.md index f55e41b15..94103db1c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -111,6 +111,128 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to [Testing](testing.md) has the rest of CI. +## CI evidence and fixture verbs + +`og dev evidence clashes` reads two gzip VCFs and gzip archaic tables. +`og dev evidence hg002-gvcf` reads a gzip truth VCF, plain BED, plain FASTA +and FAI, and a plain sites table. `og dev renovate-check` reads a plain log +and plain repository sources. Text uses strict UTF-8 and universal newlines +(LF, CRLF and CR). Compression is explicit, even when a filename is misleading. + +Evidence, `og dev fixture comphet`, `og dev fixture trim-pharmcat` and the +Renovate summary stage output beside its resolved destination. A file symlink +stays a symlink, existing permission bits are preserved, and read-only files +and directory destinations fail. New files use the caller's umask. +The VNtyper downloader follows the same destination policy, uses a private +`mktemp -d` directory, and removes it on handled failures and signals. It needs +GNU `mv -T` (`gmv` on macOS) so a directory can never receive a temporary BAM. + +Comphet stages all three files and checks all three destinations before the +first rename. A bad later destination leaves previous outputs intact. +Three renames are still not one transaction. A crash or a filesystem failure +between renames can publish only part of the new set. Unhandled SIGKILL can +leave temporary files. Single-file outputs are replaced by one rename. Publication replaces the file +and does not update other hard links. Those links still refer to the old inode. + +The shared runner fails the command if stdout or stderr cannot be written or +flushed. This includes the evidence status and Renovate table. A configured +Renovate summary must also be written and flushed successfully. +Guard runs the FASTA offset regression in debug and release profiles. + +### Input and destination refusal rule + +These evidence, fixture and Renovate verbs may refuse any input that is +malformed, outside the numeric ranges or size limits below, or not representable +by the JSON reader. They may also refuse any destination beside which they +cannot stage a sibling temporary file. A refusal always exits nonzero with a +specific message. It leaves every existing output byte-identical, with its mode +and link target unchanged. The crash and multi-file publication limits above +still apply. + +- Evidence numeric fields use ASCII unsigned integers from 0 through + 18446744073709551615. A reference lookup needs a position of at least 1. + Spans must fit that range. Calculated FASTA offsets must fit 0 through + 9223372036854775807 and address a byte inside the file. +- Every FAI row needs at least five columns and nonzero bases per line. +- `MAX_ANNOTATIONS` is an ASCII signed integer from -9223372036854775808 + through 9223372036854775807. +- Evidence and Renovate text records have a 16 MiB (16777216-byte) limit. + The PharmCAT reader has no separate file-size limit and materializes JSON. +- The JSON reader accepts signed 128-bit integers (-2^127 through 2^127-1), + Unicode scalar strings and at most 1000 nested arrays or objects. + It cannot represent lone surrogates, including in unused fields. +- Destinations need a writable parent and a regular writable target when one + exists. Resolution follows at most 40 final symlinks. Temporary sibling names + must fit the filesystem's name limit. A 255-byte basename is refused cleanly + before any output write because its temporary sibling name is longer. + +Inputs produced by the real tools are never refused: bcftools/DeepVariant VCF, +samtools faidx, header-free BED intervals, PharmCAT report JSON and Renovate +debug log. A BED file with a UCSC column header, a `track` line or a `browser` +line is refused, as the Python refused it: remove those lines first. + +Valid JSON nested up to the reader's 1000 levels is accepted. The Python stopped +with a RecursionError at a few hundred levels when it copied or wrote such a +value. That was a limit of the interpreter, not a check on the input, so the +Rust commands do not copy it. + +`refusals_preserve_bytes_modes_and_link_targets` pins the rule through each CLI. +Each table row runs with a regular output and with a symlink. It checks nonzero +exit, the specific diagnostic, unchanged bytes, mode and link target, and no +leftover temporary file. Adding a refusal case needs one row. + +Examples of refusals follow. This is not an exhaustive input grammar. +The final-LF normalization has a positive byte test. + +- BED fields use spaces and tabs as separators. Other Python whitespace + (Unicode White_Space and U+001C through U+001F) fails with `unsupported whitespace`. +- Renovate present fields of the wrong type fail with `not a map of package file lists`. + This includes map-valued updates and numeric dependency names, current values, + or update values. Falsy warnings and packages with no dependencies remain valid. +- PharmCAT truthy strings or objects where lists belong fail with `is not a list`. +- PharmCAT truthy non-object alleles fail with `is not an object`. +- Evidence positions with underscores, non-ASCII digits, negative values or + values outside u64 fail with `not a number`. +- Four-field truth VCF rows and blank FAI rows fail with `fewer than 5 columns`. +- Text records over 16 MiB, including ignored headers or log lines, fail with + `record exceeds 16777216 bytes`. +- Clash VCF rows need a tab after ALT. A newline in the ALT field of a short + row fails with `missing trailing VCF columns`. +- A truth VCF record without its final LF gets one in the output. Valid VCF ends with LF. + +- Zero-byte gzip VCFs or selected gzip tables fail with `wrong compression (expected gzip)`. +- Every FAI row needs nonzero bases per line, including unused rows. + A zero width fails with `a row has 0 bases per line`. +- `MAX_ANNOTATIONS` must parse as a signed 64-bit ASCII integer. + Underscores, non-ASCII digits and out-of-range values fail with + `MAX_ANNOTATIONS ... is not a whole number` (exit 2). +- Staging needs a writable parent directory, even for an existing writable file. + A parent with no write bits fails with `parent directory is not writable`. + Other access failures report the operating system error. +- The FASTA reader accepts plain files only. A gzip magic prefix fails with + `wrong compression (expected plain)`, including gzip with a long FNAME. +- A VCF header after the first truth record fails with + `VCF header after first data record`. The previous destination stays intact. +- FASTA offsets use checked subtraction, division, multiplication, addition + and remainder. Overflow, a position beyond the file, or an offset outside + the signed file-offset range fails with `invalid FASTA offset`. + +- A nonblank unused FAI row with fewer than five columns fails with + `a row has fewer than 5 columns`. +- A truth position that fits u64 but whose REF span does not fails with + `position span overflows`. +- A destination whose temporary sibling name is too long fails with the + filesystem diagnostic `File name too long`. +- JSON integers outside signed 128-bit range and escaped lone surrogates fail + with `Integer beyond 128 bits` and `Lone surrogate`, even in unused fields. + Excess nesting fails with `nested deeper than 1000 levels`. + +Renovate skips genuine JSON syntax errors only. Reader limitations fail the +command before an earlier matching config or the summary can be used. +`TooDeep` has its own error variant. The reader currently puts integer and +surrogate limitations in `JsonError::Decode`, so the loader distinguishes those +static reasons inside that variant. Other decode errors remain skippable. + ## CI readers `og dev pharmcat diplotypes REPORT` prints sorted, unique `GENE\tA1/A2` rows from flat or source-nested reports. `--called` prints the called count and names on stderr. `--called-lines` prints one called gene per line for e2e checks. An unreadable report exits 2 without a count. A valid empty report succeeds with a zero count. diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs new file mode 100644 index 000000000..a3e2f12db --- /dev/null +++ b/rust/crates/og-dev/src/evidence.rs @@ -0,0 +1,1140 @@ +//! `og dev evidence`: the measurements of `.github/workflows/public-evidence.yml` +//! on public data (GIAB's HG002), which were Python inside the workflow. +//! +//! - `og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv`: the +//! positions with two records or more, one of them an SNV, in the VCF as +//! GIAB writes it and split with `bcftools norm -m-`, on the chromosomes +//! of the archaic tables in DATA_DIR (`chr*.tsv.gz`), and how many rows of +//! those tables sit at such a position. OUT holds `raw_positions`, +//! `raw_on_table`, `split_positions` and `split_on_table`. +//! - `og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf`: +//! a gVCF of GIAB's genotypes plus a one-base reference block at every +//! panel site inside the benchmark regions that no GIAB record covers, +//! with the reference base from the FASTA (through its `.fai`). The +//! output is unsorted: the workflow sorts it with bcftools. +//! +//! Both stream their VCFs. hg002-gvcf keeps, per chromosome, the merged +//! spans the GIAB records cover instead of every covered position, and +//! reads one reference base per site with a positioned read. +//! +//! Exit 0 when written, 1 when an input cannot be read or parsed, 2 on a +//! usage error. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::os::unix::fs::{FileExt, PermissionsExt}; +use std::path::Path; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +const USAGE: &str = "usage: og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv\n og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf\n"; + +/// Runs `og dev evidence ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev evidence: error: {msg}"); + 2 + }; + let rest = args.get(1..).unwrap_or(&[]); + let result = match args.first().map(String::as_str) { + Some("clashes") if rest.len() == 4 => clashes(&rest[0], &rest[1], &rest[2], &rest[3]), + Some("hg002-gvcf") if rest.len() == 5 => { + hg002_gvcf(&rest[0], &rest[1], &rest[2], &rest[3], &rest[4]).map(|(records, blocks)| { + let _ = writeln!( + out, + "gVCF: {records} GIAB records, {blocks} reference blocks at panel sites" + ); + }) + } + Some("clashes" | "hg002-gvcf") => return usage(err, "wrong number of arguments"), + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown evidence '{other}'")), + None => return usage(err, "the following arguments are required: evidence"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev evidence: {e}"); + 1 + } + } +} + +fn open(path: &str, gzip: bool) -> Result, String> { + let mut file = BufReader::new(File::open(path).map_err(|e| format!("{path}: {e}"))?); + let head = file.fill_buf().map_err(|e| format!("{path}: {e}"))?; + if head.starts_with(&[0x1f, 0x8b]) != gzip { + return Err(format!( + "{path}: wrong compression (expected {})", + if gzip { "gzip" } else { "plain" } + )); + } + if gzip { + og_io::reader(file).map_err(|e| format!("{path}: {e}")) + } else { + Ok(Box::new(file)) + } +} + +/// Plain repository text with Python's universal-newline normalization. +pub(crate) fn read_text(path: &Path) -> Result { + use std::io::Read; + let mut text = String::new(); + open(&path.to_string_lossy(), false)? + .read_to_string(&mut text) + .map_err(|e| format!("{}: invalid text: {e}", path.display()))?; + Ok(text.replace("\r\n", "\n").replace('\r', "\n")) +} + +/// Strict UTF-8 and universal newlines, with a 16 MiB record limit. This +/// local reader can use og-io's bounded reader once that API is available. +pub(crate) fn lines(path: &str, f: impl FnMut(&[u8]) -> Result<(), String>) -> Result<(), String> { + input_lines(path, false, f) +} + +fn input_lines( + path: &str, + gzip: bool, + mut f: impl FnMut(&[u8]) -> Result<(), String>, +) -> Result<(), String> { + const MAX_LINE: usize = 16 << 20; + let mut r = open(path, gzip)?; + let mut line = Vec::new(); + let mut after_cr = false; + loop { + let chunk = r.fill_buf().map_err(|e| format!("{path}: {e}"))?; + if chunk.is_empty() { + if !line.is_empty() { + std::str::from_utf8(&line).map_err(|e| format!("{path}: invalid UTF-8: {e}"))?; + f(&line)?; + } + return Ok(()); + } + if after_cr { + after_cr = false; + if chunk[0] == b'\n' { + r.consume(1); + continue; + } + } + let end = chunk.iter().position(|b| matches!(b, b'\r' | b'\n')); + let n = end.unwrap_or(chunk.len()); + if line.len().checked_add(n).is_none_or(|len| len > MAX_LINE) { + return Err(format!("{path}: record exceeds {MAX_LINE} bytes")); + } + line.extend_from_slice(&chunk[..n]); + if end.is_some() { + after_cr = chunk[n] == b'\r'; + r.consume( + n.checked_add(1) + .ok_or_else(|| format!("{path}: record size overflows"))?, + ); + std::str::from_utf8(&line).map_err(|e| format!("{path}: invalid UTF-8: {e}"))?; + f(&line)?; + line.clear(); + } else { + r.consume(n); + } + } +} + +/// Follow the final symlink, including relative and dangling link targets. +fn resolve_destination(destination: &Path) -> Result { + let mut path = destination.to_owned(); + for _ in 0..40 { + match fs::read_link(&path) { + Ok(target) => { + path = if target.is_absolute() { + target + } else { + path.parent().unwrap_or(Path::new(".")).join(target) + } + } + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound + ) => + { + return Ok(path); + } + Err(e) => return Err(format!("{}: {e}", path.display())), + } + } + Err(format!( + "{}: too many destination symlinks", + destination.display() + )) +} + +fn validate_destination(path: &Path) -> Result, String> { + match fs::metadata(path) { + Ok(meta) => { + if !meta.is_file() { + return Err(format!( + "{}: destination is not a regular file", + path.display() + )); + } + let permissions = meta.permissions(); + if permissions.mode() & 0o222 == 0 { + return Err(format!("{}: read-only destination", path.display())); + } + // Check ACLs and effective-user access without truncating the old file. + File::options() + .write(true) + .open(path) + .map_err(|e| format!("{}: {e}", path.display()))?; + Ok(Some(permissions)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(format!("{}: {e}", path.display())), + } +} + +/// A unique, exclusively created sibling. Dropping it on any read, write +/// or rename error removes this temporary file. A multi-file set is staged +/// together, but its renames are separate transactions. +pub(crate) struct PendingOutput { + path: PathBuf, + destination: PathBuf, +} + +impl PendingOutput { + pub(crate) fn new(destination: &Path) -> Result<(Self, File), String> { + static NEXT: AtomicU64 = AtomicU64::new(0); + let destination = resolve_destination(destination)?; + let permissions = validate_destination(&destination)?; + let parent = destination + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let meta = fs::metadata(parent).map_err(|e| format!("{}: {e}", parent.display()))?; + if meta.permissions().mode() & 0o222 == 0 { + return Err(format!( + "{}: parent directory is not writable", + destination.display() + )); + } + let name = destination.file_name().ok_or("output has no file name")?; + loop { + let mut tmp = std::ffi::OsString::from("."); + tmp.push(name); + tmp.push(format!( + ".tmp-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let path = destination.with_file_name(tmp); + match File::options().write(true).create_new(true).open(&path) { + Ok(file) => { + let pending = Self { + path, + destination: destination.to_owned(), + }; + if let Some(permissions) = permissions { + file.set_permissions(permissions) + .map_err(|e| format!("{}: {e}", destination.display()))?; + } + return Ok((pending, file)); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(format!("{}: {e}", destination.display())), + } + } + } + + pub(crate) fn commit(self) -> Result<(), String> { + if let Some(permissions) = validate_destination(&self.destination)? { + // A write can clear set-user-ID or set-group-ID bits. Apply the + // final mode after every byte is staged, immediately before rename. + fs::set_permissions(&self.path, permissions) + .map_err(|e| format!("{}: {e}", self.destination.display()))?; + } + fs::rename(&self.path, &self.destination) + .map_err(|e| format!("{}: {e}", self.destination.display())) + } +} + +impl Drop for PendingOutput { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +pub(crate) fn publish(path: &Path, bytes: &[u8]) -> Result<(), String> { + let (pending, mut file) = PendingOutput::new(path)?; + file.write_all(bytes) + .and_then(|()| file.flush()) + .map_err(|e| format!("{}: {e}", path.display()))?; + drop(file); + pending.commit() +} + +fn pos(field: &[u8], path: &str) -> Result { + std::str::from_utf8(field) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + format!( + "{path}: position '{}' is not a number", + String::from_utf8_lossy(field) + ) + }) +} + +/// The positions of `path` on `chroms` with two records or more, one of +/// them an SNV. The VCF is sorted, so a position's records are adjacent. +fn clash_positions( + path: &str, + chroms: &BTreeSet>, +) -> Result, u64)>, String> { + let mut hit = BTreeSet::new(); + let (mut key, mut seen, mut multiple, mut snv): (Option<(Vec, u64)>, bool, bool, bool) = + (None, false, false, false); + input_lines(path, true, |line| { + if line.first() == Some(&b'#') { + return Ok(()); + } + let c: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if !chroms.contains(c[0]) { + return Ok(()); + } + if c.len() < 6 { + return Err(format!( + "{path}: a selected record has fewer than 5 columns or missing trailing VCF columns" + )); + } + let k = (c[0].to_vec(), pos(c[1], path)?); + if key.as_ref() != Some(&k) { + key = Some(k.clone()); + (seen, multiple, snv) = (false, false, false); + } + multiple |= seen; + seen = true; + let one = |a: &[u8]| std::str::from_utf8(a).map_or(a.len(), |s| s.chars().count()) == 1; + snv |= one(c[3]) && one(c[4]); + if multiple && snv { + hit.insert(k); + } + Ok(()) + })?; + Ok(hit) +} + +fn clashes(raw: &str, split: &str, data: &str, out: &str) -> Result<(), String> { + let mut tables = BTreeMap::new(); + for e in fs::read_dir(data).map_err(|e| format!("{data}: {e}"))? { + let name = e + .map_err(|e| format!("{data}: {e}"))? + .file_name() + .into_string() + .unwrap_or_default(); + if let Some(chrom) = name + .strip_suffix(".tsv.gz") + .filter(|c| c.starts_with("chr")) + { + tables.insert(chrom.as_bytes().to_vec(), Path::new(data).join(&name)); + } + } + let chroms: BTreeSet> = tables.keys().cloned().collect(); + let sets = [ + ("raw", clash_positions(raw, &chroms)?), + ("split", clash_positions(split, &chroms)?), + ]; + let mut on_table = [0u64; 2]; + for (chrom, table) in &tables { + let want: Vec> = sets + .iter() + .map(|(_, s)| { + s.iter() + .filter(|(c, _)| c == chrom) + .map(|(_, p)| *p) + .collect() + }) + .collect(); + if want.iter().all(BTreeSet::is_empty) { + continue; + } + let path = table.to_string_lossy().into_owned(); + input_lines(&path, true, |line| { + let c: Vec<&[u8]> = line.splitn(3, |&b| b == b'\t').collect(); + if c[0] != chrom.as_slice() { + return Ok(()); + } + if c.len() < 2 { + return Err(format!("{path}: a selected row has fewer than 2 columns")); + } + let p = pos(c[1], &path)?; + for (k, w) in want.iter().enumerate() { + let count = on_table + .get_mut(k) + .ok_or("table counter index out of range")?; + *count = count + .checked_add(u64::from(w.contains(&p))) + .ok_or_else(|| format!("{path}: table count overflows"))?; + } + Ok(()) + })?; + } + let mut text = String::new(); + for (k, (name, s)) in sets.iter().enumerate() { + text += &format!( + "{name}_positions\t{}\n{name}_on_table\t{}\n", + s.len(), + on_table.get(k).ok_or("table counter index out of range")? + ); + } + publish(Path::new(out), text.as_bytes()) +} + +/// One `.fai` row: length, byte offset, bases per line, bytes per line. +struct Fai { + len: u64, + offset: u64, + line_bases: u64, + line_bytes: u64, +} + +fn read_fai(path: &str) -> Result, Fai>, String> { + let mut fai = HashMap::new(); + lines(path, |line| { + let f: Vec<&[u8]> = line.split(|&b| b == b'\t').collect(); + let n = |i: usize| { + f.get(i) + .ok_or_else(|| format!("{path}: a row has fewer than 5 columns")) + .and_then(|x| pos(x, path)) + }; + let row = Fai { + len: n(1)?, + offset: n(2)?, + line_bases: n(3)?, + line_bytes: n(4)?, + }; + if row.line_bases == 0 { + return Err(format!("{path}: a row has 0 bases per line")); + } + fai.insert(f[0].to_vec(), row); + Ok(()) + })?; + Ok(fai) +} + +/// Sorted, merged half-open spans `[start, end)` of 1-based positions. +fn merge(mut spans: Vec<(u64, u64)>) -> Vec<(u64, u64)> { + spans.sort_unstable(); + let mut out: Vec<(u64, u64)> = Vec::with_capacity(spans.len()); + for (s, e) in spans { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + out +} + +/// Is `p` inside one of the sorted, disjoint spans? +fn inside(spans: &[(u64, u64)], p: u64) -> bool { + let i = spans.partition_point(|&(s, _)| s <= p); + i.checked_sub(1) + .and_then(|j| spans.get(j)) + .is_some_and(|&(_, end)| p < end) +} + +fn hg002_gvcf( + truth: &str, + bed: &str, + reference: &str, + sites: &str, + out: &str, +) -> Result<(u64, u64), String> { + let fai = read_fai(&format!("{reference}.fai"))?; + let fasta = File::open(reference).map_err(|e| format!("{reference}: {e}"))?; + let fasta_len = fasta + .metadata() + .map_err(|e| format!("{reference}: {e}"))? + .len(); + let mut magic = [0; 2]; + if fasta + .read_at(&mut magic, 0) + .map_err(|e| format!("{reference}: {e}"))? + == 2 + && magic == [0x1f, 0x8b] + { + return Err(format!("{reference}: wrong compression (expected plain)")); + } + // The benchmark regions, BED 0-based: (start, end) per chromosome, + // sorted. A site p (1-based) is confident when the region with the + // largest start below p ends at p or later, as the Python's bisect did. + let mut conf: HashMap, Vec<(u64, u64)>> = HashMap::new(); + lines(bed, |line| { + let f: Vec<&[u8]> = line + .split(|b| matches!(b, b' ' | b'\t')) + .filter(|x| !x.is_empty()) + .collect(); + if f.iter().any(|field| { + std::str::from_utf8(field) + .expect("strict reader") + .chars() + .any(|ch| ch.is_whitespace() || matches!(ch, '\u{1c}'..='\u{1f}')) + }) { + return Err(format!("{bed}: unsupported whitespace in a field")); + } + if f.len() < 3 { + return Err(format!("{bed}: a row has fewer than 3 columns")); + } + conf.entry(f[0].to_vec()) + .or_default() + .push((pos(f[1], bed)?, pos(f[2], bed)?)); + Ok(()) + })?; + for v in conf.values_mut() { + v.sort_unstable(); + } + let confident = |c: &[u8], p: u64| { + conf.get(c).is_some_and(|v| { + let i = v.partition_point(|&(s, _)| s < p); + i.checked_sub(1) + .and_then(|j| v.get(j)) + .is_some_and(|&(_, end)| end >= p) + }) + }; + + let (pending, file) = PendingOutput::new(Path::new(out))?; + let mut w = BufWriter::with_capacity(1 << 20, file); + let werr = |e: std::io::Error| format!("{out}: {e}"); + let mut covered: HashMap, Vec<(u64, u64)>> = HashMap::new(); + let mut records = 0u64; + input_lines(truth, true, |line| { + if line.first() == Some(&b'#') && records != 0 { + return Err(format!("{truth}: VCF header after first data record")); + } + if line.starts_with(b"##") { + } else if line.first() == Some(&b'#') { + w.write_all(b"##INFO=\n##ALT=\n") + .map_err(werr)?; + } else { + let f: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if f.len() < 5 { + return Err(format!("{truth}: a record has fewer than 5 columns")); + } + let p = pos(f[1], truth)?; + covered.entry(f[0].to_vec()).or_default().push(( + p, + p.checked_add( + std::str::from_utf8(f[3]) + .expect("strict reader") + .chars() + .count() + .try_into() + .map_err(|_| format!("{truth}: allele length overflows"))?, + ) + .ok_or_else(|| format!("{truth}: position span overflows"))?, + )); + records = records + .checked_add(1) + .ok_or_else(|| format!("{truth}: record count overflows"))?; + } + w.write_all(line) + .and_then(|()| w.write_all(b"\n")) + .map_err(werr) + })?; + let covered: HashMap, Vec<(u64, u64)>> = + covered.into_iter().map(|(c, v)| (c, merge(v))).collect(); + + let mut blocks = 0u64; + let mut first = true; + let mut base = [0u8; 1]; + lines(sites, |line| { + if std::mem::take(&mut first) { + return Ok(()); + } + let f: Vec<&[u8]> = line.splitn(4, |&b| b == b'\t').collect(); + if f.len() < 3 { + return Err(format!("{sites}: a row has fewer than 3 columns")); + } + let (c, p) = (f[1], pos(f[2], sites)?); + let Some(row) = fai.get(c) else { return Ok(()) }; + if p == 0 || p > row.len || covered.get(c).is_some_and(|v| inside(v, p)) || !confident(c, p) + { + return Ok(()); + } + let invalid = || format!("{reference}: invalid FASTA offset: overflow or outside file"); + let zero = p.checked_sub(1).ok_or_else(invalid)?; + let at = zero + .checked_div(row.line_bases) + .and_then(|line| line.checked_mul(row.line_bytes)) + .and_then(|relative| row.offset.checked_add(relative)) + .and_then(|start| { + zero.checked_rem(row.line_bases) + .and_then(|col| start.checked_add(col)) + }) + .ok_or_else(invalid)?; + if at > i64::MAX as u64 || at.checked_add(1).is_none_or(|end| end > fasta_len) { + return Err(invalid()); + } + fasta + .read_exact_at(&mut base, at) + .map_err(|e| format!("{reference}: {e}"))?; + std::str::from_utf8(&base).map_err(|e| format!("{reference}: invalid UTF-8: {e}"))?; + let b = base[0].to_ascii_uppercase(); + if !b"ACGT".contains(&b) { + return Ok(()); + } + blocks = blocks + .checked_add(1) + .ok_or_else(|| format!("{sites}: block count overflows"))?; + w.write_all(c) + .and_then(|()| { + writeln!( + w, + "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t0/0", + char::from(b) + ) + }) + .map_err(werr) + })?; + if first { + return Err(format!("{sites}: missing header")); + } + let file = w.into_inner().map_err(|e| werr(e.into_error()))?; + drop(file); + pending.commit()?; + Ok((records, blocks)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hg002_gvcf_keeps_truth_and_adds_only_valid_reference_blocks() { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let inputs = repo.join("tests/parity/inputs/dev-evidence-hg002-gvcf"); + let temp = std::env::temp_dir().join(format!( + "og-evidence-gvcf-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&temp).unwrap(); + let output = temp.join("out.vcf"); + let args = [ + "hg002-gvcf".to_owned(), + inputs.join("truth.vcf.gz").to_str().unwrap().to_owned(), + inputs + .join("truth-past-end.bed") + .to_str() + .unwrap() + .to_owned(), + inputs.join("ref.fasta").to_str().unwrap().to_owned(), + inputs + .join("sites-past-end.tsv") + .to_str() + .unwrap() + .to_owned(), + output.to_str().unwrap().to_owned(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 0, "{err:?}"); + assert_eq!( + out, + b"gVCF: 4 GIAB records, 1 reference blocks at panel sites\n" + ); + assert!(err.is_empty()); + let actual = fs::read(&output).unwrap(); + fs::remove_dir_all(&temp).unwrap(); + assert_eq!( + actual, + include_bytes!( + "../../../../tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf" + ) + ); + } + + struct Case { + dir: std::path::PathBuf, + kind: &'static str, + } + + impl Case { + fn new(kind: &'static str, label: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "og-evidence-{kind}-{label}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/inputs") + .join(format!("dev-evidence-{kind}")); + for e in fs::read_dir(src).unwrap() { + let e = e.unwrap(); + fs::copy(e.path(), dir.join(e.file_name())).unwrap(); + } + Self { dir, kind } + } + fn text(&self, name: &str) -> Vec { + use std::io::Read; + let mut bytes = Vec::new(); + open(self.dir.join(name).to_str().unwrap(), name.ends_with(".gz")) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + bytes + } + fn put(&self, name: &str, bytes: &[u8]) { + if name.ends_with(".gz") { + let mut gz = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + fs::write(self.dir.join(name), gz.finish().unwrap()).unwrap(); + } else { + fs::write(self.dir.join(name), bytes).unwrap(); + } + } + fn args(&self) -> Vec { + let p = |n: &str| self.dir.join(n).to_str().unwrap().to_owned(); + if self.kind == "clashes" { + vec![ + "clashes".to_owned(), + p("raw.vcf.gz"), + p("split.vcf.gz"), + self.dir.to_str().unwrap().to_owned(), + p("out"), + ] + } else { + vec![ + "hg002-gvcf".to_owned(), + p("truth.vcf.gz"), + p("truth.bed"), + p("ref.fasta"), + p("sites.tsv"), + p("out"), + ] + } + } + fn run(&self) -> Result<(), String> { + let (mut out, mut err) = (Vec::new(), Vec::new()); + match cli(&self.args(), &mut out, &mut err) { + 0 => Ok(()), + 1 => Err(String::from_utf8(err).unwrap()), + code => panic!("unexpected CLI exit {code}: {err:?}"), + } + } + fn fails_without_publication(&self) { + fs::write(self.dir.join("out"), b"PREVIOUS OUTPUT\n").unwrap(); + assert!(self.run().is_err(), "{} must fail", self.dir.display()); + assert_eq!( + fs::read(self.dir.join("out")).unwrap(), + b"PREVIOUS OUTPUT\n" + ); + let leftovers: Vec<_> = fs::read_dir(&self.dir) + .unwrap() + .filter_map(|e| { + let name = e.unwrap().file_name(); + name.to_str() + .filter(|n| n.starts_with(".out.")) + .map(str::to_owned) + }) + .collect(); + assert!(leftovers.is_empty(), "{leftovers:?}"); + } + } + impl Drop for Case { + fn drop(&mut self) { + fs::remove_dir_all(&self.dir).unwrap(); + } + } + + #[test] + fn evidence_status_uses_checked_runner() { + for kind in ["clashes", "hg002-gvcf"] { + let c = Case::new(kind, "streams"); + crate::stream_tests::assert_status_errors(cli, &c.args(), kind == "hg002-gvcf"); + } + } + + #[test] + fn fai_offsets_reject_overflow_and_out_of_file_before_publication() { + for (offset, bases, bytes, position) in [ + (18446744073709551598u64, 1, 2, 13), + (1, 1, u64::MAX, 3), + (u64::MAX, 1, 1, 1), + (u64::MAX - 1, 50, 51, 3), + (1000, 50, 51, 1), + ] { + let c = Case::new("hg002-gvcf", &format!("offset-{offset}-{position}")); + c.put( + "ref.fasta.fai", + format!("chrA\t50\t{offset}\t{bases}\t{bytes}\n").as_bytes(), + ); + c.put( + "sites.tsv", + format!("id\tchr\tpos\nx\tchrA\t{position}\n").as_bytes(), + ); + c.put("truth.bed", b"chrA\t0\t50\n"); + c.put( + "truth.vcf.gz", + b"##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\n", + ); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains(": invalid FASTA offset:")); + } + } + + #[test] + fn late_vcf_headers_fail_before_publication() { + for header in [b"##late=yes\n".as_slice(), b"#CHROM\tPOS\n"] { + let c = Case::new("hg002-gvcf", &format!("late-{}", header.len())); + c.put( + "truth.vcf.gz", + &[c.text("truth.vcf.gz"), header.to_vec()].concat(), + ); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": VCF header after first data record") + ); + } + } + + #[test] + fn empty_gzip_inputs_and_unused_zero_width_fai_rows_fail() { + for (kind, name) in [ + ("clashes", "raw.vcf.gz"), + ("clashes", "split.vcf.gz"), + ("clashes", "chrA.tsv.gz"), + ("hg002-gvcf", "truth.vcf.gz"), + ] { + let c = Case::new(kind, name); + fs::write(c.dir.join(name), b"").unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected gzip)") + ); + } + let c = Case::new("hg002-gvcf", "unused-row"); + c.put( + "ref.fasta.fai", + &[c.text("ref.fasta.fai"), b"chrUnused\t0\t0\t0\t0\n".to_vec()].concat(), + ); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": a row has 0 bases per line") + ); + } + + #[test] + fn gzip_fasta_with_ascii_fname_is_refused_by_format_guard() { + let c = Case::new("hg002-gvcf", "fname"); + let mut gz = flate2::GzBuilder::new() + .filename(vec![b'A'; 200]) + .write(Vec::new(), flate2::Compression::default()); + gz.write_all(&c.text("ref.fasta")).unwrap(); + fs::write(c.dir.join("ref.fasta"), gz.finish().unwrap()).unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected plain)") + ); + } + + #[test] + fn non_writable_parents_preserve_evidence_destinations() { + for kind in ["clashes", "hg002-gvcf"] { + let c = Case::new(kind, "parent-mode"); + fs::write(c.dir.join("out"), b"PREVIOUS OUTPUT\n").unwrap(); + fs::set_permissions(&c.dir, fs::Permissions::from_mode(0o555)).unwrap(); + let result = c.run(); + fs::set_permissions(&c.dir, fs::Permissions::from_mode(0o755)).unwrap(); + assert!( + result + .unwrap_err() + .contains(": parent directory is not writable") + ); + assert_eq!(fs::read(c.dir.join("out")).unwrap(), b"PREVIOUS OUTPUT\n"); + } + } + + #[test] + fn selected_truncated_clash_records_fail() { + for name in ["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"] { + let c = Case::new("clashes", name); + c.put( + name, + if name == "chrA.tsv.gz" { + b"chrA" + } else { + b"chrA\t3\n" + }, + ); + c.fails_without_publication(); + } + } + + #[test] + fn missing_panel_header_and_blank_bed_fail() { + for name in ["sites.tsv", "truth.bed"] { + let c = Case::new("hg002-gvcf", name); + let bytes = if name == "sites.tsv" { + vec![] + } else { + [c.text(name), b"\n".to_vec()].concat() + }; + c.put(name, &bytes); + c.fails_without_publication(); + } + } + + #[test] + fn every_evidence_text_input_rejects_invalid_utf8() { + for (kind, names) in [ + ("clashes", vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"]), + ( + "hg002-gvcf", + vec!["truth.vcf.gz", "truth.bed", "ref.fasta.fai", "sites.tsv"], + ), + ] { + for name in names { + let c = Case::new(kind, name); + // Invalid bytes in a header or filtered-out record must still fail. + let mut bytes = c.text(name); + bytes.extend_from_slice(b"#ignored\xff\n"); + c.put(name, &bytes); + c.fails_without_publication(); + } + } + } + + #[test] + fn every_evidence_text_input_has_universal_newlines() { + for (kind, names) in [ + ( + "clashes", + vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz", "chrB.tsv.gz"], + ), + ( + "hg002-gvcf", + vec!["truth.vcf.gz", "truth.bed", "ref.fasta.fai", "sites.tsv"], + ), + ] { + for name in names { + for ending in [b"\r".as_slice(), b"\r\n".as_slice()] { + let c = Case::new(kind, &format!("{name}-{}", ending.len())); + c.run().unwrap(); + let expected = fs::read(c.dir.join("out")).unwrap(); + let bytes = c + .text(name) + .split(|b| *b == b'\n') + .collect::>() + .join(ending); + c.put(name, &bytes); + c.run().unwrap(); + assert_eq!( + fs::read(c.dir.join("out")).unwrap(), + expected, + "{kind}/{name}/{ending:?}" + ); + } + } + } + } + + #[test] + fn failed_gvcf_inputs_preserve_existing_output() { + for name in ["truth.vcf.gz", "sites.tsv"] { + for fault in ["missing", "directory", "malformed", "truncated"] { + let c = Case::new("hg002-gvcf", &format!("{name}-{fault}")); + let path = c.dir.join(name); + match fault { + "missing" => fs::remove_file(path).unwrap(), + "directory" => { + fs::remove_file(&path).unwrap(); + fs::create_dir(path).unwrap(); + } + "malformed" => { + let mut bytes = c.text(name); + bytes.extend_from_slice(b"bad\n"); + c.put(name, &bytes); + } + _ if name.ends_with(".gz") => { + let mut bytes = fs::read(&path).unwrap(); + bytes.truncate(bytes.len() - 5); + fs::write(path, bytes).unwrap(); + } + _ => c.put(name, b"header\nbad\n"), + } + c.fails_without_publication(); + } + } + } + + #[test] + fn explicit_compression_is_required_at_every_input() { + for (kind, names) in [ + ("clashes", vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"]), + ( + "hg002-gvcf", + vec![ + "truth.vcf.gz", + "truth.bed", + "ref.fasta.fai", + "ref.fasta", + "sites.tsv", + ], + ), + ] { + for name in names { + let c = Case::new(kind, &format!("compression-{name}")); + let bytes = c.text(name); + let wrong = if name.ends_with(".gz") { + bytes + } else { + let mut gz = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(&bytes).unwrap(); + gz.finish().unwrap() + }; + fs::write(c.dir.join(name), wrong).unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected "), + "{name}" + ); + } + } + } + + #[test] + fn bed_rejects_python_whitespace_inside_fields_and_as_separators() { + for ch in [ + '\u{2028}', '\u{85}', '\u{b}', '\u{a0}', '\u{1c}', '\u{1d}', '\u{1e}', '\u{1f}', + ] { + for embedded in [true, false] { + let c = Case::new("hg002-gvcf", &format!("space-{}-{embedded}", ch as u32)); + let text = String::from_utf8(c.text("truth.bed")).unwrap(); + let text = if embedded { + text.replacen("chrA", &format!("chr{ch}A"), 1) + } else { + text.replace('\t', &ch.to_string()) + }; + c.put("truth.bed", text.as_bytes()); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("unsupported whitespace")); + } + } + } + + #[test] + fn publication_preserves_permissions_and_follows_symlinks() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let c = Case::new("clashes", "publication"); + let target = c.dir.join("target"); + let link = c.dir.join("link"); + fs::write(&target, b"old").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o600)).unwrap(); + symlink("target", &link).unwrap(); + publish(&link, b"new").unwrap(); + assert!( + fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(fs::read(&target).unwrap(), b"new"); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o600 + ); + fs::set_permissions(&target, fs::Permissions::from_mode(0o4600)).unwrap(); + publish(&link, b"new").unwrap(); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o7777, + 0o4600 + ); + fs::set_permissions(&target, fs::Permissions::from_mode(0o444)).unwrap(); + assert!(publish(&link, b"bad").unwrap_err().contains("read-only")); + assert_eq!(fs::read(&target).unwrap(), b"new"); + let dir_link = c.dir.join("dir-link"); + symlink(&c.dir, &dir_link).unwrap(); + for path in [&c.dir, &dir_link] { + assert!( + publish(path, b"bad") + .unwrap_err() + .contains("not a regular file") + ); + } + } + + #[test] + fn approved_strict_evidence_shapes_fail_with_stable_errors() { + for value in ["3_0", "٣", "-3", "18446744073709551616"] { + let c = Case::new("hg002-gvcf", value); + c.put( + "truth.vcf.gz", + format!("chrA\t{value}\t.\tA\tC\n").as_bytes(), + ); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("not a number")); + } + let c = Case::new("hg002-gvcf", "short-vcf"); + c.put("truth.vcf.gz", b"chrA\t3\t.\tA\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("fewer than 5 columns")); + let c = Case::new("hg002-gvcf", "blank-fai"); + c.put("ref.fasta.fai", b"\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("fewer than 5 columns")); + let c = Case::new("clashes", "newline-alt"); + c.put("raw.vcf.gz", b"chrA\t3\t.\tA\tC\nchrA\t3\t.\tAA\tA\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("VCF columns")); + } + + #[test] + fn approved_record_limit_and_final_lf_are_pinned() { + let c = Case::new("clashes", "record-limit"); + c.put("raw.vcf.gz", &vec![b'#'; (16 << 20) + 1]); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains("record exceeds 16777216 bytes") + ); + let c = Case::new("hg002-gvcf", "final-lf"); + c.run().unwrap(); + let expected = fs::read(c.dir.join("out")).unwrap(); + let mut truth = c.text("truth.vcf.gz"); + assert_eq!(truth.pop(), Some(b'\n')); + c.put("truth.vcf.gz", &truth); + c.run().unwrap(); + let result = fs::read(c.dir.join("out")).unwrap(); + assert_eq!(result, expected); + assert_eq!(result.last(), Some(&b'\n')); + } + + #[test] + fn spans_merge_and_answer_inside() { + let m = merge(vec![(10, 12), (5, 6), (11, 15), (15, 16)]); + assert_eq!(m, [(5, 6), (10, 16)]); + assert!( + inside(&m, 5) && !inside(&m, 6) && inside(&m, 15) && !inside(&m, 16) && !inside(&m, 4) + ); + } +} diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs new file mode 100644 index 000000000..ec74f8dd3 --- /dev/null +++ b/rust/crates/og-dev/src/fixture.rs @@ -0,0 +1,930 @@ +//! `og dev fixture`: writers of committed or generated test fixtures. +//! +//! - `og dev fixture comphet OUT_DIR [--swap]`: the compound-het phasing +//! fixture of `tests/e2e/comphet-phasing-1-reads.sh` (made-up reads on a +//! made-up contig). It replaces `tests/fixtures/comphet/make_fixture.py` +//! and writes the same bytes: the contig is Python's `random.Random(31)` +//! choosing among `ACGT`, so [`Mt19937`] is CPython's generator. +//! - `og dev fixture trim-pharmcat REPORT OUT [MAX_ANNOTATIONS]`: a PharmCAT +//! `report.json` cut down to the fields `og pgx` reads. It replaces +//! `tests/fixtures/pharmcat/trim_report.py` and writes the same bytes. +//! +//! Exit 0 when written, 1 when an input cannot be read or has the wrong +//! shape, 2 on a usage error. + +use crate::py_os_error; +use indexmap::IndexMap; +use pycompat::json::{self, Dump, Value}; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::Path; + +const USAGE: &str = "usage: og dev fixture comphet OUT_DIR [--swap]\n og dev fixture trim-pharmcat REPORT.json OUT.json [MAX_ANNOTATIONS]\n"; + +/// Runs `og dev fixture ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev fixture: error: {msg}"); + 2 + }; + let result = match args.first().map(String::as_str) { + Some("comphet") => { + let rest: Vec<&String> = args[1..].iter().filter(|a| *a != "--swap").collect(); + if rest.len() != 1 { + return usage(err, "comphet takes one OUT_DIR and an optional --swap"); + } + comphet::write(Path::new(rest[0]), args[1..].iter().any(|a| a == "--swap")) + } + Some("trim-pharmcat") => { + let rest = &args[1..]; + if !(2..=3).contains(&rest.len()) { + return usage( + err, + "trim-pharmcat takes REPORT.json OUT.json [MAX_ANNOTATIONS]", + ); + } + let max = match rest.get(2).map(|m| m.trim().parse::()) { + None => 4, + Some(Ok(n)) => n, + Some(Err(_)) => { + return usage( + err, + &format!("MAX_ANNOTATIONS '{}' is not a whole number", rest[2]), + ); + } + }; + trim_pharmcat(Path::new(&rest[0]), Path::new(&rest[1]), max) + } + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown fixture '{other}'")), + None => return usage(err, "the following arguments are required: fixture"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev fixture: {e}"); + 1 + } + } +} + +/// CPython's Mersenne Twister (`random.Random`), only as far as +/// `Random(int).choice(seq)` needs it. +pub struct Mt19937 { + state: [u32; 624], + index: usize, +} + +impl Mt19937 { + /// `random.Random(seed)` for a seed below 2^32: `init_by_array([seed])`. + pub fn new(seed: u32) -> Self { + let mut mt = Mt19937 { + state: [0; 624], + index: 624, + }; + mt.init_genrand(19_650_218); + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624.max(key.len()) { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_566_083_941)).wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + } + mt.state[0] = 0x8000_0000; + mt + } + + fn init_genrand(&mut self, s: u32) { + self.state[0] = s; + for i in 1..624 { + let prev = self.state[i - 1] ^ (self.state[i - 1] >> 30); + self.state[i] = 1_812_433_253u32.wrapping_mul(prev).wrapping_add(i as u32); + } + self.index = 624; + } + + /// `genrand_uint32`. + pub fn next_u32(&mut self) -> u32 { + if self.index >= 624 { + for k in 0..624 { + let y = (self.state[k] & 0x8000_0000) | (self.state[(k + 1) % 624] & 0x7fff_ffff); + let mag = if y & 1 == 1 { 0x9908_b0df } else { 0 }; + self.state[k] = self.state[(k + 397) % 624] ^ (y >> 1) ^ mag; + } + self.index = 0; + } + let mut y = self.state[self.index]; + self.index += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + /// `_randbelow(n)` for `0 < n < 2^32`: rejection sampling on + /// `getrandbits(n.bit_length())`. + pub fn below(&mut self, n: u32) -> u32 { + let k = 32 - n.leading_zeros(); + loop { + let r = self.next_u32() >> (32 - k); + if r < n { + return r; + } + } + } +} + +pub mod comphet { + //! The compound-het phasing fixture: sample CHPH on contig chrP (80 kb + //! of seeded random sequence), read pairs of 100 bp from 500 bp + //! fragments, and three pairs of heterozygous variants: + //! + //! - GENEC: SNV at 1000 and a 2-base deletion at 1003, both on copy 1 + //! (cis); single reads cover both. `--swap` moves the deletion to + //! copy 2 (trans). + //! - GENET: SNVs at 5000 and 5400, one on each copy (trans); only a read + //! and its mate cover both. + //! - GENEF: SNVs at 20000 and 70000, one on each copy, 50 kb apart: no + //! read pair covers both (unknown). + //! + //! Files: `reads.sam`, `vep.vcf` (with a VEP CSQ, for step 31) and + //! `comphet.vcf` (the pairs as slivar names them, for `og comphet + //! phase`). + + use super::Mt19937; + use std::fmt::Write as _; + use std::fs; + use std::path::Path; + + pub const SAMPLE: &str = "CHPH"; + pub const CONTIG: &str = "chrP"; + pub const LENGTH: usize = 80_000; + const READ: usize = 100; + const FRAG: usize = 500; + const FIELDS: &str = "Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|Existing_variation|MAX_AF|gnomADe_AF|gnomADg_AF|CLIN_SIG"; + + /// One variant: 1-based position, alleles, consequence and the copy + /// (0 or 1) that carries it. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct Variant { + pub gene: &'static str, + pub pos: usize, + pub alt_ref: String, + pub alt: String, + pub consequence: &'static str, + pub copy: u8, + } + + /// The contig: `random.Random(31)` choosing among `ACGT` 80,000 times. + pub fn reference() -> Vec { + let mut rng = Mt19937::new(31); + (0..LENGTH) + .map(|_| b"ACGT"[rng.below(4) as usize]) + .collect() + } + + /// The base before `b` in `ACGT`, wrapping (`A` gives `T`). + fn other(b: u8) -> u8 { + let i = b"ACGT" + .iter() + .position(|&x| x == b) + .expect("a base of ACGT"); + b"ACGT"[(i + 3) % 4] + } + + pub fn variants(seq: &[u8], swap: bool) -> Vec { + let s = |a: usize, b: usize| String::from_utf8_lossy(&seq[a..b]).into_owned(); + let snv = |gene, pos: usize, consequence, copy| Variant { + gene, + pos, + alt_ref: s(pos - 1, pos), + alt: (other(seq[pos - 1]) as char).to_string(), + consequence, + copy, + }; + vec![ + snv("GENEC", 1000, "stop_gained", 0), + Variant { + gene: "GENEC", + pos: 1003, + alt_ref: s(1002, 1005), + alt: s(1002, 1003), + consequence: "frameshift_variant", + copy: u8::from(swap), + }, + snv("GENET", 5000, "stop_gained", 0), + snv("GENET", 5400, "stop_gained", 1), + snv("GENEF", 20000, "stop_gained", 0), + snv("GENEF", 70000, "stop_gained", 1), + ] + } + + /// The sequence and CIGAR of a read of `READ` bases from 0-based + /// `start` on one copy. + fn read_on(seq: &[u8], vs: &[Variant], copy: u8, start: usize) -> (String, String) { + let mine = || vs.iter().filter(|v| v.copy == copy); + let snv_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() == v.alt.len() && v.pos - 1 == p) + .map(|v| v.alt.as_bytes()[0]) + }; + // A deletion is keyed on the first deleted base. + let del_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() > v.alt.len() && v.pos + v.alt.len() - 1 == p) + .map(|v| v.alt_ref.len() - v.alt.len()) + }; + let (mut out, mut ops, mut p) = + (Vec::with_capacity(READ), Vec::<(u8, usize)>::new(), start); + while out.len() < READ { + if let Some(n) = del_at(p) { + ops.push((b'D', n)); + p += n; + continue; + } + out.push(snv_at(p).unwrap_or(seq[p])); + ops.push((b'M', 1)); + p += 1; + } + let mut cigar = String::new(); + let mut i = 0; + while i < ops.len() { + let (op, mut n) = ops[i]; + let mut j = i + 1; + while j < ops.len() && ops[j].0 == op { + n += ops[j].1; + j += 1; + } + let _ = write!(cigar, "{n}{}", op as char); + i = j; + } + (String::from_utf8_lossy(&out).into_owned(), cigar) + } + + pub fn sam(seq: &[u8], vs: &[Variant]) -> String { + let mut s = format!( + "@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:{CONTIG}\tLN:{LENGTH}\n@RG\tID:{SAMPLE}\tSM:{SAMPLE}\n" + ); + // Fragment starts (0-based) around each pair: read 1 covers the + // first position, its mate the second one when 400 bp apart. + let starts: [(&str, std::iter::StepBy>); 4] = [ + ("c", (920..991).step_by(7)), + ("t", (4905..4996).step_by(10)), + ("f", (19905..19996).step_by(10)), + ("g", (69905..69996).step_by(10)), + ]; + let q = "I".repeat(READ); + for (region, rs) in starts { + for copy in 0..2u8 { + for st in rs.clone() { + let m = st + FRAG - READ; + let (s1, c1) = read_on(seq, vs, copy, st); + let (s2, c2) = read_on(seq, vs, copy, m); + let name = format!("{region}{copy}_{st}"); + let _ = writeln!( + s, + "{name}\t99\t{CONTIG}\t{}\t60\t{c1}\t=\t{}\t{FRAG}\t{s1}\t{q}\tRG:Z:{SAMPLE}", + st + 1, + m + 1 + ); + let _ = writeln!( + s, + "{name}\t147\t{CONTIG}\t{}\t60\t{c2}\t=\t{}\t-{FRAG}\t{s2}\t{q}\tRG:Z:{SAMPLE}", + m + 1, + st + 1 + ); + } + } + } + s + } + + fn vcf_head(info: &str) -> String { + format!( + "##fileformat=VCFv4.2\n##contig=\n##FILTER=\n{info}\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n" + ) + } + + pub fn vep_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head(&format!( + "##INFO=\n" + )); + for v in vs { + let (g, p) = (v.gene, v.pos); + let _ = writeln!( + out, + "{CONTIG}\t{p}\t.\t{}\t{}\t50\tPASS\tCSQ={}|{}|HIGH|{g}|ENSG{g}|Transcript|ENST{p}|protein_coding|YES|NM_{p}.1|rs{p}|0.0001|0.0001|0.0001|\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.alt_ref, v.alt, v.alt, v.consequence + ); + } + out + } + + pub fn comphet_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head( + "##INFO=\n", + ); + for (i, v) in vs.iter().enumerate() { + let mate = if i % 2 == 0 { &vs[i + 1] } else { &vs[i - 1] }; + let pid = match v.gene { + "GENEC" => 1, + "GENET" => 2, + _ => 3, + }; + let _ = writeln!( + out, + "{CONTIG}\t{}\t.\t{}\t{}\t50\tPASS\tslivar_comphet={SAMPLE}/{}/{pid}/{CONTIG}/{}/{}/{}\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.pos, v.alt_ref, v.alt, v.gene, mate.pos, mate.alt_ref, mate.alt + ); + } + out + } + + /// Writes the three files into `dir`, made if missing. + pub fn write(dir: &Path, swap: bool) -> Result<(), String> { + let seq = reference(); + let vs = variants(&seq, swap); + fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let mut staged = Vec::new(); + for (name, text) in [ + ("reads.sam", sam(&seq, &vs)), + ("vep.vcf", vep_vcf(&vs)), + ("comphet.vcf", comphet_vcf(&vs)), + ] { + let path = dir.join(name); + let (pending, mut file) = crate::evidence::PendingOutput::new(&path)?; + std::io::Write::write_all(&mut file, text.as_bytes()) + .map_err(|e| format!("{}: {e}", path.display()))?; + std::io::Write::flush(&mut file).map_err(|e| format!("{}: {e}", path.display()))?; + drop(file); + staged.push(pending); + } + // Every file is staged and every destination checked before publication. + // Three renames are still not one transaction. + for pending in staged { + pending.commit()?; + } + Ok(()) + } +} + +/// `x.get(k)`: the value, or JSON null. +fn get(x: &IndexMap, k: &str) -> Value { + x.get(k).cloned().unwrap_or(Value::Null) +} + +/// Python's truth of a JSON value. +fn truthy(v: &Value) -> bool { + pycompat::value::truthy(v) +} + +/// The old trimmer used `v or []`: null and every false value are empty. +/// `v or []` as a list; a truthy value that is not a list is an error. +fn list<'a>(v: Option<&'a Value>, what: &str) -> Result<&'a [Value], String> { + match v { + Some(Value::Array(a)) => Ok(a), + Some(x) if truthy(x) => Err(format!("{what} is not a list")), + _ => Ok(&[]), + } +} + +fn object<'a>(v: &'a Value, what: &str) -> Result<&'a IndexMap, String> { + match v { + Value::Object(o) => Ok(o), + _ => Err(format!("{what} is not an object")), + } +} + +/// The keys of `x` among `keys`, in that order. +fn pick(x: &IndexMap, keys: &[&str]) -> IndexMap { + keys.iter() + .filter_map(|k| x.get(*k).map(|v| ((*k).to_owned(), v.clone()))) + .collect() +} + +/// One listed diplotype with the fields `og pgx` reads. +fn diplotype(x: &Value) -> Result { + let Value::Object(x) = x else { + // The Python dip() explicitly passes non-object listed diplotypes through. + return Ok(x.clone()); + }; + let mut out = pick( + x, + &[ + "gene", + "label", + "phenotypes", + "activityScore", + "outsidePhenotype", + ], + ); + for a in ["allele1", "allele2"] { + let Some(v) = x.get(a) else { continue }; + let kept = match v { + Value::Object(o) if !o.is_empty() => { + Value::Object(pick(o, &["gene", "name", "function"])) + } + v if !truthy(v) => v.clone(), + _ => return Err(format!("diplotype {a} is not an object")), + }; + out.insert(a.to_owned(), kept); + } + Ok(Value::Object(out)) +} + +/// `items[:n]` with Python's slice rule for a negative `n`. +#[allow( + clippy::manual_saturating_arithmetic, + reason = "Python slices clamp a checked negative end at zero" +)] +fn head(items: &[Value], n: i64) -> &[Value] { + // Clamp before slicing, including i64::MIN, without narrowing lengths. + let count = usize::try_from(n.unsigned_abs()).unwrap_or(usize::MAX); + let end = if n < 0 { + items.len().checked_sub(count).unwrap_or_default() + } else { + count.min(items.len()) + }; + &items[..end] +} + +/// The trimmed report: every gene with its listed diplotypes and related +/// drugs, and for each drug of each guidance source up to `max` of the +/// annotations PharmCAT matched, with only the fields `og pgx` reads. +pub fn trim(d: Value, max: i64) -> Result { + let Value::Object(mut d) = d else { + return Err("the report is not an object".to_owned()); + }; + let mut genes = IndexMap::new(); + let Some(Value::Object(input_genes)) = d.swap_remove("genes") else { + return Err("genes is not an object".to_owned()); + }; + // Consume one source gene at a time: do not retain a second full report. + for (g, v) in input_genes { + let v = object(&v, &format!("gene {g}"))?; + let mut gene = pick(v, &["geneSymbol", "chr", "phased", "callSource"]); + let mut related = Vec::new(); + for r in list(v.get("relatedDrugs"), "relatedDrugs")? { + let r = object(r, "a related drug")?; + let name = r.get("name").ok_or("a related drug has no name")?.clone(); + related.push(Value::Object(IndexMap::from([ + ("name".to_owned(), name), + ("id".to_owned(), get(r, "id")), + ]))); + } + gene.insert("relatedDrugs".to_owned(), Value::Array(related)); + // sourceDiplotypes is what og pgx reads; recommendationDiplotypes + // only when sourceDiplotypes is absent, so it is kept only then. + let k = if v.get("sourceDiplotypes").is_some_and(truthy) { + "sourceDiplotypes" + } else { + "recommendationDiplotypes" + }; + if v.contains_key(k) { + let dips = list(v.get(k), k)? + .iter() + .map(diplotype) + .collect::>()?; + gene.insert(k.to_owned(), Value::Array(dips)); + } + genes.insert(g, Value::Object(gene)); + } + let mut drugs: IndexMap = IndexMap::new(); + let Some(Value::Object(input_drugs)) = d.swap_remove("drugs") else { + return Err("drugs is not an object".to_owned()); + }; + for (source, by) in input_drugs { + for (name, rep) in object(&by, &format!("drugs of {source}"))? { + let rep = object(rep, &format!("drug {name}"))?; + let mut gls = Vec::new(); + for gl in list(rep.get("guidelines"), "guidelines")? { + let gl = object(gl, "a guideline")?; + let mut anns = Vec::new(); + for a in head(list(gl.get("annotations"), "annotations")?, max) { + let a = object(a, "an annotation")?; + let mut gts = Vec::new(); + for gt in list(a.get("genotypes"), "genotypes")? { + let gt = object(gt, "a genotype")?; + let dips = list(gt.get("diplotypes"), "diplotypes")? + .iter() + .filter_map(|x| match x { + Value::Object(x) => Some(Value::Object(IndexMap::from([ + ("gene".to_owned(), get(x, "gene")), + ("label".to_owned(), get(x, "label")), + ]))), + // The Python comprehension has `if isinstance(x, dict)`. + _ => None, + }) + .collect(); + gts.push(Value::Object(IndexMap::from([( + "diplotypes".to_owned(), + Value::Array(dips), + )]))); + } + anns.push(Value::Object(IndexMap::from([ + ("phenotypes".to_owned(), get(a, "phenotypes")), + ("classification".to_owned(), get(a, "classification")), + ( + "drugRecommendation".to_owned(), + get(a, "drugRecommendation"), + ), + ("genotypes".to_owned(), Value::Array(gts)), + ]))); + } + if !anns.is_empty() { + gls.push(Value::Object(IndexMap::from([ + ("name".to_owned(), get(gl, "name")), + ("source".to_owned(), get(gl, "source")), + ("annotations".to_owned(), Value::Array(anns)), + ]))); + } + } + if !gls.is_empty() { + let entry = drugs + .entry(source.clone()) + .or_insert_with(|| Value::Object(IndexMap::new())); + if let Value::Object(e) = entry { + e.insert( + name.clone(), + Value::Object(IndexMap::from([ + ("name".to_owned(), get(rep, "name")), + ("id".to_owned(), get(rep, "id")), + ("guidelines".to_owned(), Value::Array(gls)), + ])), + ); + } + } + } + } + let mut out = pick( + &d, + &["title", "timestamp", "pharmcatVersion", "dataVersion"], + ); + out.insert("genes".to_owned(), Value::Object(genes)); + out.insert("drugs".to_owned(), Value::Object(drugs)); + let mut note = String::new(); + let _ = write!( + note, + "Trimmed by og dev fixture trim-pharmcat: every gene, and up to {max} matched annotations per drug guideline, with only the fields og pgx reads." + ); + out.insert("_fixture_note".to_owned(), Value::Str(note)); + Ok(Value::Object(out)) +} + +fn trim_pharmcat(src: &Path, dst: &Path, max: i64) -> Result<(), String> { + let text = fs::read(src).map_err(|e| py_os_error(&e, &src.to_string_lossy()))?; + let text = String::from_utf8(text).map_err(|_| format!("{}: not UTF-8", src.display()))?; + let d = json::loads(&text).map_err(|e| format!("{}: {e}", src.display()))?; + drop(text); + let out = trim(d, max)?; + let opts = Dump { + separators: Some((",", ":")), + ..Dump::default() + }; + let text = json::dumps(&out, &opts).map_err(|e| e.to_string())?; + crate::evidence::publish(dst, text.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixture_verbs_use_checked_runner_even_with_empty_status() { + let d = std::env::temp_dir().join(format!("og-fixture-streams-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for args in [ + vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ], + vec!["comphet".into(), d.to_str().unwrap().into()], + ] { + crate::stream_tests::assert_status_errors(cli, &args, false); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn annotation_slices_clamp_extreme_counts_without_narrowing() { + let items = vec![Value::Null; 3]; + for (n, expected) in [ + (i64::MIN, 0), + (-4, 0), + (-3, 0), + (-2, 1), + (-1, 2), + (0, 0), + (1, 1), + (i64::MAX, 3), + ] { + assert_eq!(head(&items, n).len(), expected); + } + } + + #[test] + fn non_ascii_or_out_of_range_annotation_limits_fail_at_cli() { + let d = std::env::temp_dir().join(format!("og-limits-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for limit in ["1_0", "٤", "18446744073709551616"] { + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + limit.into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 2); + let err = String::from_utf8(err).unwrap(); + assert!( + err.contains("error: MAX_ANNOTATIONS") && err.contains("is not a whole number"), + "{err}" + ); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn non_writable_parents_preserve_fixture_destinations() { + use std::os::unix::fs::PermissionsExt; + let d = std::env::temp_dir().join(format!("og-parent-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for name in ["out", "reads.sam", "vep.vcf", "comphet.vcf"] { + fs::write(d.join(name), b"old").unwrap(); + } + for args in [ + vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ], + vec!["comphet".into(), d.to_str().unwrap().into()], + ] { + fs::set_permissions(&d, fs::Permissions::from_mode(0o555)).unwrap(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + fs::set_permissions(&d, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(code, 1); + assert!( + String::from_utf8(err) + .unwrap() + .contains(": parent directory is not writable") + ); + for name in ["out", "reads.sam", "vep.vcf", "comphet.vcf"] { + assert_eq!(fs::read(d.join(name)).unwrap(), b"old"); + } + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn pharmcat_input_is_plain_json() { + let d = + std::env::temp_dir().join(format!("og-pharmcat-compression-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(br#"{"genes":{},"drugs":{}}"#).unwrap(); + fs::write(d.join("in"), gz.finish().unwrap()).unwrap(); + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 1); + assert!(String::from_utf8_lossy(&err).contains("not UTF-8")); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn comphet_validates_later_destinations_before_publishing() { + let d = std::env::temp_dir().join(format!("og-comphet-preflight-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("comphet.vcf")).unwrap(); + for name in ["reads.sam", "vep.vcf"] { + fs::write(d.join(name), b"PREVIOUS OUTPUT").unwrap(); + } + assert!(comphet::write(&d, false).is_err()); + for name in ["reads.sam", "vep.vcf"] { + assert_eq!(fs::read(d.join(name)).unwrap(), b"PREVIOUS OUTPUT"); + } + assert_eq!(fs::read_dir(&d).unwrap().count(), 3); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn downloader_preflights_destinations_and_cleans_handled_failures() { + use std::os::unix::fs::{PermissionsExt, symlink}; + use std::process::Command; + let d = std::env::temp_dir().join(format!("og-download-safety-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("bin")).unwrap(); + let stub = d.join("bin/curl"); + fs::write(&stub, "#!/bin/sh\nprintf called >> \"$CALLED\"\nwhile [ \"$1\" != -o ]; do shift; done\nprintf partial > \"$2\"\nexit 22\n").unwrap(); + fs::set_permissions(&stub, fs::Permissions::from_mode(0o755)).unwrap(); + let script = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../tests/smoke/vntyper_testdata.sh"); + for fault in [ + "directory", + "directory-link", + "curl", + "crc", + "sha", + "signal", + ] { + let out = d.join(fault); + fs::create_dir(&out).unwrap(); + let dst = out.join("negative.bam"); + match fault { + "directory" => fs::create_dir(&dst).unwrap(), + "directory-link" => symlink(&d, &dst).unwrap(), + _ => { + fs::write(&dst, b"PREVIOUS OUTPUT").unwrap(); + symlink("negative.bam", out.join("negative.bam.part")).unwrap(); + } + } + if fault == "crc" { + fs::write(&stub, "#!/bin/sh\nprintf called >> \"$CALLED\"\nwhile [ \"$1\" != -o ]; do shift; done\ndd if=/dev/zero of=\"$2\" bs=3376349 count=1 2>/dev/null\n").unwrap(); + } else if fault == "sha" { + // Supply an inflater with correct size but the wrong SHA. + fs::write(d.join("bin/gzip"), "#!/bin/sh\nprintf wrong-sha\n").unwrap(); + fs::set_permissions(d.join("bin/gzip"), fs::Permissions::from_mode(0o755)).unwrap(); + } else if fault == "signal" { + fs::remove_file(d.join("bin/gzip")).unwrap(); + fs::write(&stub, "#!/bin/sh\nkill -TERM \"$PPID\"\nexit 22\n").unwrap(); + } + let called = d.join("called"); + let _ = fs::remove_file(&called); + let result = Command::new("sh") + .arg(&script) + .arg(&out) + .arg("negative.bam") + .env( + "PATH", + format!( + "{}:{}", + d.join("bin").display(), + std::env::var("PATH").unwrap() + ), + ) + .env("CALLED", &called) + .output() + .unwrap(); + assert!(!result.status.success(), "{fault}"); + if fault.starts_with("directory") { + assert!(!called.exists(), "must reject before curl: {fault}"); + } else { + assert_eq!(fs::read(&dst).unwrap(), b"PREVIOUS OUTPUT", "{fault}"); + let entries: Vec<_> = fs::read_dir(&out) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(entries.len(), 2, "{fault}: {entries:?}"); + } + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn approved_strict_pharmcat_containers_fail_at_the_cli() { + let d = std::env::temp_dir().join(format!("og-pharmcat-shapes-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + for (input, class) in [ + ( + r#"{"genes":{"G":{"sourceDiplotypes":"x"}},"drugs":{}}"#, + "sourceDiplotypes is not a list", + ), + ( + r#"{"genes":{"G":{"recommendationDiplotypes":"x"}},"drugs":{}}"#, + "recommendationDiplotypes is not a list", + ), + ( + r#"{"genes":{"G":{"sourceDiplotypes":[{"allele1":"x"}]}},"drugs":{}}"#, + "allele1 is not an object", + ), + ( + r#"{"genes":{"G":{"sourceDiplotypes":[{"allele2":["x"]}]}},"drugs":{}}"#, + "allele2 is not an object", + ), + ( + r#"{"genes":{},"drugs":{"S":{"D":{"guidelines":[{"annotations":[{"genotypes":[{"diplotypes":{"x":1}}]}]}]}}}}"#, + "diplotypes is not a list", + ), + ] { + fs::write(d.join("in"), input).unwrap(); + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 1); + assert!(String::from_utf8_lossy(&err).contains(class), "{err:?}"); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn mt19937_matches_cpython() { + // random.Random(31): [r.getrandbits(32) for _ in range(3)] and the + // first ten of "".join(r.choice("ACGT") ...) from a fresh seed. + let mut r = Mt19937::new(31); + let first: Vec = (0..3).map(|_| r.next_u32()).collect(); + assert_eq!(first, [52_734_659, 2_016_822_975, 482_749_413]); + assert_eq!(&comphet::reference()[..10], b"ATATCACACC"); + } + + #[test] + fn swap_moves_only_the_deletion() { + let seq = comphet::reference(); + let (a, b) = ( + comphet::variants(&seq, false), + comphet::variants(&seq, true), + ); + assert_eq!(a[1].copy, 0); + assert_eq!(b[1].copy, 1); + assert_eq!(a[0], b[0]); + assert_eq!(a[2..], b[2..]); + } + + #[test] + fn a_read_over_the_deletion_has_a_d_in_its_cigar() { + let seq = comphet::reference(); + let vs = comphet::variants(&seq, false); + let sam = comphet::sam(&seq, &vs); + let line = sam + .lines() + .find(|l| l.starts_with("c0_920\t99")) + .expect("read c0_920"); + assert!(line.contains("\t83M2D17M\t"), "{line}"); + let line = sam + .lines() + .find(|l| l.starts_with("c1_920\t99")) + .expect("read c1_920"); + assert!(line.contains("\t100M\t"), "{line}"); + } + + #[test] + fn trim_keeps_the_fields_og_pgx_reads() { + let d = json::loads( + r#"{"title":"t","extra":1,"genes":{"G":{"geneSymbol":"G","x":1,"relatedDrugs":[{"name":"d"}], + "sourceDiplotypes":[],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1","y":2},"allele2":null,"z":3},"str"]}}, + "drugs":{"CPIC":{"d":{"name":"d","guidelines":[{"name":"g","annotations":[{"genotypes":[{"diplotypes":[{"gene":"G"},5]}]},{},{}]},{"annotations":[]}]}}}}"#, + ) + .expect("json"); + let out = trim(d.clone(), 2).expect("trim"); + let text = json::dumps( + &out, + &Dump { + separators: Some((",", ":")), + ..Dump::default() + }, + ) + .expect("dumps"); + assert!(text.starts_with(r#"{"title":"t","genes":{"G":{"geneSymbol":"G","relatedDrugs":[{"name":"d","id":null}],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1"},"allele2":null},"str"]}}"#), "{text}"); + assert!(text.contains(r#""annotations":[{"phenotypes":null,"classification":null,"drugRecommendation":null,"genotypes":[{"diplotypes":[{"gene":"G","label":null}]}]},{"phenotypes":null"#), "{text}"); + assert_eq!(text.matches("drugRecommendation").count(), 2, "{text}"); + // A negative count is Python's slice: all but the last. + let out = trim(d, -2).expect("trim"); + let text = json::dumps(&out, &Dump::default()).expect("dumps"); + assert_eq!(text.matches("drugRecommendation").count(), 1, "{text}"); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index a310762d7..004440e3e 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -15,16 +15,25 @@ //! (`scripts/ci/check-container-helpers.sh`). //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev fixture` ([`fixture`]): writers of test fixtures (the +//! compound-het reads, a trimmed PharmCAT report). +//! - `og dev evidence` ([`evidence`]): the measurements of the public +//! evidence workflow on GIAB's HG002. +//! - `og dev renovate-check LOG` ([`renovate`]): what a Renovate dry run +//! found, against every pin in the repository. //! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): //! rebuild a committed asset table from its pinned downloads. pub mod build; pub mod contract; +pub mod evidence; +pub mod fixture; pub mod helpers; pub mod nf_trace; pub mod parity; pub mod pharmcat; pub mod remnants; +pub mod renovate; pub mod report_only; pub mod schema; pub mod shlex; @@ -33,7 +42,7 @@ mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n\n og dev build cn-intervals|callability|sv-rule-regions [args]"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -53,11 +62,74 @@ pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { /// Runs a verb that writes to stdout and stderr, flushing both. fn run(f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, args: &[String]) -> u8 { - let mut out = std::io::stdout().lock(); - let mut err = std::io::stderr().lock(); + use std::os::fd::AsFd; + // Stdout/StdoutLock suppress EBADF on Unix. Duplicating the descriptor + // preserves its access mode and exposes write errors without unsafe code. + let out = match std::io::stdout().as_fd().try_clone_to_owned() { + Ok(fd) => std::fs::File::from(fd), + Err(e) => { + let _ = writeln!(std::io::stderr().lock(), "og dev: stdout: {e}"); + return 1; + } + }; + let mut err = match std::io::stderr().as_fd().try_clone_to_owned() { + Ok(fd) => std::fs::File::from(fd), + Err(_) => return 1, + }; + let mut out = std::io::BufWriter::new(out); + run_with_writers(f, args, &mut out, &mut err) +} + +pub(crate) fn run_with_writers( + f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, + args: &[String], + out: &mut dyn Write, + err: &mut dyn Write, +) -> u8 { + let mut out = CheckedWriter { + inner: out, + failure: None, + }; + let mut err = CheckedWriter { + inner: err, + failure: None, + }; let code = f(args, &mut out, &mut err); let _ = out.flush(); - code + if let Some(reason) = &out.failure { + let _ = writeln!(err, "og dev: stdout: {reason}"); + } + let _ = err.flush(); + if code == 0 && (out.failure.is_some() || err.failure.is_some()) { + 1 + } else { + code + } +} + +/// Remember errors even when an older verb ignores its Write result. +struct CheckedWriter<'a> { + inner: &'a mut dyn Write, + failure: Option, +} +impl Write for CheckedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let result = match self.inner.write(bytes) { + Ok(0) if !bytes.is_empty() => Err(std::io::Error::from(std::io::ErrorKind::WriteZero)), + result => result, + }; + if let Err(e) = &result { + self.failure.get_or_insert_with(|| e.to_string()); + } + result + } + fn flush(&mut self) -> std::io::Result<()> { + let result = self.inner.flush(); + if let Err(e) = &result { + self.failure.get_or_insert_with(|| e.to_string()); + } + result + } } /// Runs `og dev ARGS`; returns the exit code. @@ -68,6 +140,9 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("fixture") => run(fixture::cli, &args[1..]), + Some("evidence") => run(evidence::cli, &args[1..]), + Some("renovate-check") => run(renovate::cli, &args[1..]), Some("pharmcat") => run(pharmcat::cli, &args[1..]), Some("nf-trace") => run(nf_trace::cli, &args[1..]), Some("smoke") => run(smoke::cli, &args[1..]), @@ -89,3 +164,109 @@ pub fn main(args: &[String]) -> u8 { } } } + +#[cfg(test)] +mod stream_tests { + use super::*; + struct Fails { + flush_only: bool, + } + impl Write for Fails { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.flush_only { + Ok(bytes.len()) + } else { + Err(std::io::Error::other("write control")) + } + } + fn flush(&mut self) -> std::io::Result<()> { + if self.flush_only { + Err(std::io::Error::other("flush control")) + } else { + Ok(()) + } + } + } + #[test] + fn stdio_process_control() { + if let Ok(verb) = std::env::var("OG_DEV_STREAM_CONTROL") { + std::process::exit(i32::from(main(&[verb, "--help".into()]))); + } + } + + #[test] + fn read_only_stdout_descriptor_makes_added_verbs_fail() { + let path = std::env::temp_dir().join(format!("og-stream-fd-{}", std::process::id())); + std::fs::write(&path, b"previous bytes").unwrap(); + for verb in ["fixture", "evidence", "renovate-check"] { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "stream_tests::stdio_process_control", + "--nocapture", + ]) + .env("OG_DEV_STREAM_CONTROL", verb) + .stdout(std::fs::File::open(&path).unwrap()) + .output() + .unwrap(); + assert!(!result.status.success(), "{verb} silently lost stdout"); + assert!( + String::from_utf8_lossy(&result.stderr).contains("og dev: stdout:"), + "{verb}: {:?}", + result.stderr + ); + assert_eq!(std::fs::read(&path).unwrap(), b"previous bytes"); + } + std::fs::remove_file(path).unwrap(); + } + + pub(crate) fn assert_status_errors( + verb: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, + args: &[String], + writes: bool, + ) { + for flush_only in [false, true] { + let mut bad = Fails { flush_only }; + let mut err = Vec::new(); + assert_eq!( + run_with_writers(verb, args, &mut bad, &mut err), + u8::from(writes || flush_only), + "{}", + String::from_utf8_lossy(&err) + ); + } + } + + #[test] + fn added_verbs_propagate_write_and_flush_errors() { + let verbs = [fixture::cli, evidence::cli, renovate::cli]; + for verb in verbs { + for flush_only in [false, true] { + let mut bad = Fails { flush_only }; + let mut err = Vec::new(); + assert_eq!( + run_with_writers(verb, &["--help".into()], &mut bad, &mut err), + 1 + ); + assert!(String::from_utf8(err).unwrap().contains("stdout:")); + } + } + } + #[test] + fn shared_runner_checks_stderr_and_an_empty_stdout() { + fn clean(_: &[String], _: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "status"); + 0 + } + for flush_only in [false, true] { + assert_eq!( + run_with_writers(clean, &[], &mut Vec::new(), &mut Fails { flush_only }), + 1 + ); + } + assert_eq!( + run_with_writers(clean, &[], &mut Fails { flush_only: true }, &mut Vec::new()), + 1 + ); + } +} diff --git a/rust/crates/og-dev/src/renovate.rs b/rust/crates/og-dev/src/renovate.rs new file mode 100644 index 000000000..f279010b1 --- /dev/null +++ b/rust/crates/og-dev/src/renovate.rs @@ -0,0 +1,1707 @@ +//! `og dev renovate-check LOG [--pins FILE] [--repo DIR]`: checks what a +//! Renovate dry run (`renovate --platform=local --dry-run=lookup`, JSON +//! debug log) found against every pin in the repository. It replaces the +//! Python the `renovate-dry-run.yml` workflow wrote into a file and ran. +//! +//! The checks, each with a planted fault in the self-test that runs after +//! them: +//! +//! - `count`: every `*_IMAGE` line of `versions.env` is a detected docker +//! dependency; +//! - `names`: no docker package name carries a `:` or `@`; +//! - `twins`: `docs/versions.md` holds exactly the images of `versions.env`, +//! and `conf/containers.config` holds only those; +//! - `one-pr`: a dependency gets the same proposals in every file it is in; +//! - `pins`: every non-image pin of the pins table +//! (`scripts/ci/renovate-pins.tsv`) is detected as often as its file pins +//! it; +//! - `lookups`: no lookup failed, no tag was unreadable, and the only +//! skipped dependencies are the ones a rule in `renovate.json` disables. +//! +//! The pins table has three tab-separated columns: the dependency, a file +//! or a glob of files (relative to the repository; `*` and `?` in the last +//! part), and a regular expression for one pin of it in that text, read +//! with `^` and `$` at line ends. A file that is missing, or a glob that +//! matches nothing, is an error (exit 2): a pin counted in no file could +//! never fail. +//! +//! Output: the Markdown table of every dependency and its proposed update +//! (also appended to `$GITHUB_STEP_SUMMARY`), one `FAIL : ...` line +//! per problem, the self-test lines, then `OK: N dependencies; every check +//! passed`. Exit 0 when clean, 1 on a finding, a self-test miss, or a log +//! without its `packageFiles with updates` record, 2 on a usage error or a +//! pins table it cannot use. + +use indexmap::IndexMap; +use pycompat::json::{self, Value}; +use pycompat::value::{py_repr, py_str, truthy}; +use regex::Regex; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::io::Write; +use std::path::Path; + +const USAGE: &str = "usage: og dev renovate-check LOG [--pins FILE] [--repo DIR]\n"; +const ENV: &str = "versions.env"; +const DOCS: &str = "docs/versions.md"; +const CONFIG: &str = "conf/containers.config"; +const TWINS: [&str; 2] = [DOCS, CONFIG]; +const RULE_SKIPS: [&str; 2] = ["disabled", "package-rules"]; +const PINS: &str = "scripts/ci/renovate-pins.tsv"; + +/// One dependency Renovate found: the file it is in and its record. +type Dep = (String, IndexMap); + +/// The pins table, expanded: per (dependency, file) how many pins the text +/// holds, and per dependency the files listed for it. +struct Pins { + want: BTreeMap<(String, String), usize>, + files: HashMap>, + setup_nextflow: Option, +} + +/// Runs `og dev renovate-check ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let summary = std::env::var_os("GITHUB_STEP_SUMMARY").filter(|p| !p.is_empty()); + cli_with_summary(args, out, err, summary.as_deref().map(Path::new)) +} + +fn cli_with_summary( + args: &[String], + out: &mut dyn Write, + err: &mut dyn Write, + summary_path: Option<&Path>, +) -> u8 { + let (mut log, mut pins, mut repo) = (None, PINS.to_owned(), ".".to_owned()); + let mut it = args.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "-h" | "--help" => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + "--pins" | "--repo" => { + let Some(v) = it.next() else { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: {a} needs a value" + ); + return 2; + }; + if a == "--pins" { + pins = v.clone(); + } else { + repo = v.clone(); + } + } + _ if log.is_none() => log = Some(a.clone()), + _ => { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: unexpected argument '{a}'" + ); + return 2; + } + } + } + let Some(log) = log else { + let _ = writeln!( + err, + "{USAGE}og dev renovate-check: error: the following arguments are required: LOG" + ); + return 2; + }; + let repo = Path::new(&repo); + let pins = match read_pins(repo, &repo.join(&pins)) { + Ok(p) => p, + Err(e) => { + let _ = writeln!(err, "og dev renovate-check: {e}"); + return 2; + } + }; + let env_text = match crate::evidence::read_text(&repo.join(ENV)) { + Ok(t) => t, + Err(e) => { + let _ = writeln!(err, "og dev renovate-check: {ENV}: {e}"); + return 2; + } + }; + let deps = match load(&log) { + Ok(d) => d, + Err(e) => { + let _ = writeln!(err, "{e}"); + return 1; + } + }; + let table = summary(&deps); + let _ = writeln!(out, "{table}"); + if let Some(path) = summary_path { + let appended = (|| -> Result<(), String> { + let (pending, mut file) = crate::evidence::PendingOutput::new(path)?; + match fs::File::open(path) { + Ok(mut old) => { + std::io::copy(&mut old, &mut file).map_err(|e| e.to_string())?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.to_string()), + } + writeln!(file, "## Renovate dry run\n\n{table}").map_err(|e| e.to_string())?; + file.flush().map_err(|e| e.to_string())?; + drop(file); + pending.commit() + })(); + if let Err(e) = appended { + let _ = writeln!(err, "og dev renovate-check: GITHUB_STEP_SUMMARY: {e}"); + return 1; + } + } + let mut fails = check(&deps, &env_text, &pins); + for x in &fails { + let _ = writeln!(out, "FAIL {x}"); + } + if !self_test(&deps, &env_text, &pins, out) { + fails.push("the self-test did not catch a planted fault".to_owned()); + let _ = writeln!(out, "FAIL the self-test did not catch a planted fault"); + } + if !fails.is_empty() { + return 1; + } + let _ = writeln!(out, "OK: {} dependencies; every check passed", deps.len()); + 0 +} + +/// `fnmatch` for `*` and `?`, case-sensitive. +fn fnmatch(pat: &[u8], name: &[u8]) -> bool { + match pat.split_first() { + None => name.is_empty(), + Some((b'*', rest)) => (0..=name.len()).any(|i| fnmatch(rest, &name[i..])), + Some((b'?', rest)) => !name.is_empty() && fnmatch(rest, &name[1..]), + Some((c, rest)) => name.first() == Some(c) && fnmatch(rest, &name[1..]), + } +} + +/// The files of `pattern` under `repo`, relative and sorted, as Python's +/// `sorted(glob.glob(pattern))` lists them: a wildcard never matches a name +/// that starts with a dot unless the pattern does. +fn glob(repo: &Path, pattern: &str) -> Result, String> { + let (dir, last) = pattern.rsplit_once('/').unwrap_or(("", pattern)); + if dir.contains(['*', '?']) { + return Err(format!( + "{pattern}: a wildcard is allowed only in the last part" + )); + } + if !last.contains(['*', '?']) { + return if repo.join(pattern).is_file() { + Ok(vec![pattern.to_owned()]) + } else { + Err(format!("{pattern} is not a file")) + }; + } + let mut out = Vec::new(); + let entries = fs::read_dir(repo.join(if dir.is_empty() { "." } else { dir })) + .map_err(|e| format!("{pattern}: {e}"))?; + for e in entries { + let name = e.map_err(|e| format!("{pattern}: {e}"))?.file_name(); + let name = name.to_string_lossy(); + if (name.starts_with('.') && !last.starts_with('.')) + || !fnmatch(last.as_bytes(), name.as_bytes()) + { + continue; + } + out.push(if dir.is_empty() { + name.into_owned() + } else { + format!("{dir}/{name}") + }); + } + out.sort(); + if out.is_empty() { + return Err(format!("{pattern} matches no file")); + } + Ok(out) +} + +fn read_pins(repo: &Path, path: &Path) -> Result { + let text = crate::evidence::read_text(path)?; + let mut pins = Pins { + want: BTreeMap::new(), + files: HashMap::new(), + setup_nextflow: None, + }; + let mut texts: HashMap = HashMap::new(); + for (i, line) in text.lines().enumerate() { + if line.is_empty() || line.starts_with('#') { + continue; + } + let number = i.checked_add(1).ok_or("pin line number overflows")?; + let at = |msg: &str| format!("{}:{number}: {msg}", path.display()); + let cols: Vec<&str> = line.split('\t').collect(); + let [dep, files, pattern] = cols[..] else { + return Err(at( + "want three tab-separated columns: dependency, file or glob, pattern", + )); + }; + let re = Regex::new(&format!("(?m){pattern}")).map_err(|e| at(&e.to_string()))?; + if pattern.contains("setup-nextflow") { + pins.setup_nextflow = Some(re.clone()); + } + for f in glob(repo, files).map_err(|e| at(&e))? { + if !texts.contains_key(&f) { + let t = crate::evidence::read_text(&repo.join(&f)).map_err(|e| at(&e))?; + texts.insert(f.clone(), t); + } + let n = re.find_iter(&texts[&f]).count(); + if n > 0 { + add_count(pins.want.entry((dep.to_owned(), f.clone())).or_default(), n)?; + } + pins.files.entry(dep.to_owned()).or_default().insert(f); + } + } + Ok(pins) +} + +/// Every dependency of the log's last `packageFiles with updates` record. +fn load(log: &str) -> Result, String> { + let mut files = None; + crate::evidence::lines(log, |line| { + let value = match json::loads(std::str::from_utf8(line).expect("strict reader")) { + Ok(value) => value, + // TooDeep has a distinct kind. The reader still encodes its two + // other representation limits as static Decode reasons. They are + // valid JSON, so neither may hide a record or an earlier config. + Err( + e @ (json::JsonError::TooDeep + | json::JsonError::Decode( + "Integer beyond 128 bits" + | "Lone surrogate (Python keeps it; this port cannot)", + _, + )), + ) => return Err(format!("FAIL: {log}: JSON reader limitation: {e}")), + Err(json::JsonError::Decode(..)) => return Ok(()), + Err(e) => return Err(format!("FAIL: {log}: {e}")), + }; + let Value::Object(mut rec) = value else { + return Err(format!("FAIL: {log}: a log record is not an object")); + }; + if rec.get("msg") == Some(&Value::Str("packageFiles with updates".to_owned())) { + files = Some( + rec.swap_remove("config") + .ok_or_else(|| format!("FAIL: {log}: missing config"))?, + ); + } + Ok(()) + })?; + let Some(files) = files else { + return Err(format!( + "FAIL: no 'packageFiles with updates' record in {log}" + )); + }; + let shape = || { + format!( + "FAIL: the 'packageFiles with updates' record of {log} is not a map of package file lists" + ) + }; + let Value::Object(files) = files else { + return Err(shape()); + }; + let mut deps = Vec::new(); + for entries in files.into_values() { + let Value::Array(entries) = entries else { + return Err(shape()); + }; + for pf in entries { + let Value::Object(mut pf) = pf else { + return Err(shape()); + }; + let ds = match pf.swap_remove("deps") { + None => Vec::new(), + Some(Value::Array(ds)) => ds, + _ => return Err(shape()), + }; + let name = match pf.swap_remove("packageFile") { + None => None, + Some(Value::Str(name)) => Some(name), + // An unused field may be absent, but a present wrong type is + // still a deliberate input error. + _ => return Err(shape()), + }; + if ds.is_empty() { + continue; + } + let name = name.ok_or_else(shape)?; + for d in ds { + let Value::Object(d) = d else { + return Err(shape()); + }; + // Optional means absent. Null or a scalar is not an empty list. + if let Some(u) = d.get("updates") { + let Value::Array(us) = u else { + return Err(shape()); + }; + for u in us { + let Value::Object(u) = u else { + return Err(shape()); + }; + for k in ["newValue", "newDigest", "updateType"] { + if u.get(k) + .is_some_and(|v| !matches!(v, Value::Null | Value::Str(_))) + { + return Err(shape()); + } + } + } + } + for k in [ + "datasource", + "depName", + "currentValue", + "currentDigest", + "skipReason", + ] { + if d.get(k) + .is_some_and(|v| !matches!(v, Value::Null | Value::Str(_))) + { + return Err(shape()); + } + } + if let Some(ws) = d.get("warnings").filter(|v| truthy(v)) { + let Value::Array(ws) = ws else { + return Err(shape()); + }; + if ws.iter().any(|v| !matches!(v, Value::Object(_))) { + return Err(shape()); + } + } + deps.push((name.clone(), d)); + } + } + } + Ok(deps) +} + +fn get<'a>(d: &'a IndexMap, k: &str) -> &'a Value { + d.get(k).unwrap_or(&Value::Null) +} + +/// Python's `"%s" % v`. +fn s(v: &Value) -> String { + py_str(v).unwrap_or_else(|_| format!("{v:?}")) +} + +/// Python's `repr(v)`. +fn r(v: &Value) -> String { + py_repr(v).unwrap_or_else(|_| format!("{v:?}")) +} + +/// `repr` of a tuple of values. +fn tuple(vs: &[&Value]) -> String { + let items: Vec = vs.iter().map(|v| r(v)).collect(); + format!("({})", items.join(", ")) +} + +fn is_docker(d: &IndexMap) -> bool { + get(d, "datasource") == &Value::Str("docker".to_owned()) +} + +/// The image names of `versions.env`: `name:tag@sha256:...`, +/// `name@sha256:...` and `name:tag` all give `name`. +fn env_images(text: &str) -> Vec { + let line = Regex::new(r#"(?m)^[A-Z0-9_]+_IMAGE="([^"]*)""#).expect("a valid pattern"); + let tag = Regex::new(r":[^:/]*$").expect("a valid pattern"); + line.captures_iter(text) + .map(|c| { + let v = c[1].split('@').next().unwrap_or(""); + tag.replace(v, "").into_owned() + }) + .collect() +} + +/// `(depName, currentValue, currentDigest)` as Python prints the tuple. +fn key(d: &IndexMap) -> String { + tuple(&[ + get(d, "depName"), + get(d, "currentValue"), + get(d, "currentDigest"), + ]) +} + +/// `v or ""`. +fn or_empty(v: &Value) -> Value { + if truthy(v) { + v.clone() + } else { + Value::Str(String::new()) + } +} + +/// The updates as sorted `(newValue or "", newDigest or "")` pairs, printed +/// as Python prints the list. +fn proposals(d: &IndexMap) -> String { + let mut ps: Vec<(Value, Value)> = updates(d) + .iter() + .map(|u| match u { + Value::Object(u) => (or_empty(get(u, "newValue")), or_empty(get(u, "newDigest"))), + _ => (Value::Null, Value::Null), + }) + .collect(); + ps.sort_by(|a, b| (s(&a.0), s(&a.1)).cmp(&(s(&b.0), s(&b.1)))); + let items: Vec = ps.iter().map(|(a, b)| tuple(&[a, b])).collect(); + format!("[{}]", items.join(", ")) +} + +fn updates(d: &IndexMap) -> &[Value] { + match d.get("updates") { + Some(Value::Array(u)) => u, + _ => &[], + } +} + +fn add_count(count: &mut usize, n: usize) -> Result<(), String> { + *count = count + .checked_add(n) + .ok_or("arithmetic: dependency count overflows")?; + Ok(()) +} + +fn counter<'a>( + it: impl Iterator>, +) -> Result, String> { + let mut c = BTreeMap::new(); + for d in it { + add_count(c.entry(key(d)).or_default(), 1)?; + } + Ok(c) +} + +/// The docker dependencies of one file. +fn in_file<'a>( + docker: &'a [&'a Dep], + f: &'a str, +) -> impl Iterator> { + docker.iter().filter(move |(g, _)| g == f).map(|(_, d)| d) +} + +/// Every problem, as `check: ...` lines. +fn check(deps: &[Dep], env_text: &str, pins: &Pins) -> Vec { + check_counts(deps, env_text, pins).unwrap_or_else(|e| vec![e]) +} + +fn check_counts(deps: &[Dep], env_text: &str, pins: &Pins) -> Result, String> { + let mut fails = Vec::new(); + let docker: Vec<&Dep> = deps.iter().filter(|(_, d)| is_docker(d)).collect(); + let want = env_images(env_text); + let got: Vec<&Value> = in_file(&docker, ENV).map(|d| get(d, "depName")).collect(); + if got.len() != want.len() { + fails.push(format!( + "count: versions.env has {} *_IMAGE lines, Renovate found {} docker dependencies there", + want.len(), + got.len() + )); + } + let want_set: BTreeSet<&String> = want.iter().collect(); + for name in want_set { + if !got.contains(&&Value::Str(name.clone())) { + fails.push(format!( + "count: {name} (versions.env) is not a detected dependency" + )); + } + } + for (f, d) in &docker { + let name = get(d, "depName"); + let text = if truthy(name) { + s(name) + } else { + ":".to_owned() + }; + if text.contains([':', '@']) { + fails.push(format!("names: {f}: package name {}", r(name))); + } + } + let env_keys = counter(in_file(&docker, ENV))?; + for f in TWINS { + let keys = counter(in_file(&docker, f))?; + if f == DOCS && keys != env_keys { + let all: BTreeSet<&String> = keys.keys().chain(env_keys.keys()).collect(); + for k in all { + if keys.get(k) != env_keys.get(k) { + fails.push(format!( + "twins: docs/versions.md and versions.env differ on {k}" + )); + } + } + } else { + for k in keys.keys().filter(|k| !env_keys.contains_key(*k)) { + fails.push(format!("twins: {f} has {k}, which versions.env does not")); + } + } + } + let mut seen: HashMap = HashMap::new(); + for (f, d) in deps { + let name = tuple(&[ + get(d, "datasource"), + get(d, "depName"), + get(d, "currentValue"), + get(d, "currentDigest"), + ]); + let p = proposals(d); + match seen.get(&name) { + Some((first, q)) if *q != p => fails.push(format!( + "one-pr: {} gets {q} in {first} but {p} in {f}", + s(get(d, "depName")) + )), + Some(_) => {} + None => { + seen.insert(name, (f, p)); + } + } + } + for (f, d) in deps { + let warnings = get(d, "warnings"); + if truthy(warnings) { + let msgs: Vec = match warnings { + Value::Array(ws) => ws + .iter() + .map(|w| match w { + Value::Object(o) if o.contains_key("message") => s(get(o, "message")), + w => s(w), + }) + .collect(), + w => vec![s(w)], + }; + fails.push(format!( + "lookups: {} ({f}): {}", + s(get(d, "depName")), + msgs.join("; ") + )); + } + let skip = get(d, "skipReason"); + if truthy(skip) && !RULE_SKIPS.contains(&s(skip).as_str()) { + fails.push(format!( + "lookups: {} ({f}) skipped: {}", + s(get(d, "depName")), + s(skip) + )); + } + } + let mut got: BTreeMap<(String, String), usize> = BTreeMap::new(); + for (f, d) in deps { + if let Value::Str(name) = get(d, "depName") + && pins.files.get(name).is_some_and(|fs| fs.contains(f)) + { + add_count(got.entry((name.clone(), f.clone())).or_default(), 1)?; + } + } + let all: BTreeSet<&(String, String)> = pins.want.keys().chain(got.keys()).collect(); + for k in all { + let (w, g) = ( + pins.want.get(k).copied().unwrap_or(0), + got.get(k).copied().unwrap_or(0), + ); + if w != g { + fails.push(format!( + "pins: {} in {}: the text pins it {w} times, Renovate found {g}", + k.0, k.1 + )); + } + } + Ok(fails) +} + +/// Plants one fault of each kind in a copy of the dependencies and shows +/// that the check it belongs to catches it; true when every one is caught. +fn self_test(deps: &[Dep], env_text: &str, pins: &Pins, out: &mut dyn Write) -> bool { + let mut ok = true; + let base: BTreeSet = check(deps, env_text, pins).into_iter().collect(); + let first = |d: &[Dep], f: &str| d.iter().position(|(g, x)| g == f && is_docker(x)); + type Plant = fn(&mut Vec, usize); + let plants: [(&str, &str, Option<&str>, Plant); 6] = [ + ("count", "count:", Some(ENV), |d, i| { + d.remove(i); + }), + ("names", "names:", Some(ENV), |d, i| { + let name = format!("{}@sha256", s(get(&d[i].1, "depName"))); + d[i].1.insert("depName".to_owned(), Value::Str(name)); + }), + ("twins", "twins:", Some(DOCS), |d, i| { + d[i].1.insert( + "currentValue".to_owned(), + Value::Str("0.0.0-planted".to_owned()), + ); + }), + ("one-pr", "one-pr:", Some(CONFIG), |d, i| { + let u = IndexMap::from([( + "newValue".to_owned(), + Value::Str("9.9.9-planted".to_owned()), + )]); + d[i].1 + .insert("updates".to_owned(), Value::Array(vec![Value::Object(u)])); + }), + ("pins", "pins:", None, |d, i| { + d.remove(i); + }), + ("lookups", "lookups:", Some(ENV), |d, i| { + d[i].1.insert( + "skipReason".to_owned(), + Value::Str("invalid-value".to_owned()), + ); + }), + ]; + for (label, pattern, file, plant) in plants { + let at = match file { + Some(f) => first(deps, f), + None => deps.iter().position(|(g, x)| { + g == ENV && get(x, "depName") == &Value::Str("nextflow-io/nextflow".to_owned()) + }), + }; + let Some(i) = at else { + let _ = writeln!( + out, + "self-test {label:<8} NOT PLANTED: no dependency to plant it on" + ); + ok = false; + continue; + }; + let mut d = deps.to_vec(); + plant(&mut d, i); + // Only a failure the plant added counts as caught. + let hit = check(&d, env_text, pins) + .into_iter() + .find(|x| x.starts_with(pattern) && !base.contains(x)); + match hit { + Some(h) => { + let _ = writeln!(out, "self-test {label:<8} caught: {h}"); + } + None => { + let _ = writeln!(out, "self-test {label:<8} NOT CAUGHT"); + ok = false; + } + } + } + // The setup-nextflow pattern itself, on planted workflow text. + let Some(setup) = &pins.setup_nextflow else { + let _ = writeln!( + out, + "self-test pin-text NOT PLANTED: the pins table has no setup-nextflow pattern" + ); + return false; + }; + let computed = "${{ steps.nf.outputs.version }}"; + let cases: [(&str, &str, &str, &str, usize); 7] = [ + ("computed", "", "", computed, 0), + ("computed, uses quoted", "\"", "", computed, 0), + ("computed after a blank line", "", "\n", computed, 0), + ("literal", "", "", "\"25.10.8\"", 1), + ("literal after a blank line", "", "\n", "\"25.10.8\"", 1), + ("single-quoted literal", "'", "", "'25.10.8'", 1), + ("bare literal, uses quoted", "\"", "", "25.10.8", 1), + ]; + for (label, quote, gap, version, want) in cases { + let text = format!( + " - uses: {quote}nf-core/setup-nextflow@{a}{quote} # v3.0.1\n{gap} if: steps.scope.outputs.run == 'true'\n with:\n version: {version}\n\n - uses: actions/setup-java@{b} # v6\n with:\n java-version: \"17\"\n", + a = "a".repeat(40), + b = "b".repeat(40), + ); + let n = setup.find_iter(&text).count(); + let _ = writeln!( + out, + "self-test pin-text {label:<27} counted {n}, want {want}" + ); + ok = ok && n == want; + } + let clean = check(deps, env_text, pins); + let verdict = clean + .first() + .map_or_else(|| "passes".to_owned(), |c| format!("fails: {c}")); + let _ = writeln!(out, "self-test clean {verdict}"); + ok +} + +/// The Markdown table of every dependency, once per datasource and name. +fn summary(deps: &[Dep]) -> String { + let id = |d: &IndexMap| tuple(&[get(d, "datasource"), get(d, "depName")]); + let mut files: HashMap> = HashMap::new(); + for (f, d) in deps { + files.entry(id(d)).or_default().insert(f); + } + let mut rows = vec![ + "| Dependency | Current | Proposed update | Skipped | Files |".to_owned(), + "|---|---|---|---|---|".to_owned(), + ]; + let mut seen = BTreeSet::new(); + for (_, d) in deps { + let k = id(d); + if !seen.insert(k.clone()) { + continue; + } + let cur = [get(d, "currentValue"), get(d, "currentDigest")] + .into_iter() + .find(|v| truthy(v)) + .map_or_else(String::new, s); + let ups: Vec = updates(d) + .iter() + .map(|u| { + let u = match u { + Value::Object(u) => u, + _ => return s(u), + }; + let new = if truthy(get(u, "newValue")) { + get(u, "newValue") + } else { + get(u, "newDigest") + }; + format!("{} ({})", s(new), s(get(u, "updateType"))) + }) + .collect(); + let ups = if ups.is_empty() { + "none".to_owned() + } else { + ups.join(", ") + }; + let skip = get(d, "skipReason"); + let note = if truthy(skip) { s(skip) } else { String::new() }; + rows.push(format!( + "| `{}` | `{cur}` | {ups} | {note} | {} |", + s(get(d, "depName")), + files[&k].len() + )); + } + rows.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fnmatch_and_env_images() { + assert!( + fnmatch(b"*.yml", b"e2e.yml") + && !fnmatch(b"*.yml", b"e2e.yaml") + && fnmatch(b"a?c", b"abc") + ); + let env = "A_IMAGE=\"x/y:1.2@sha256:ab\"\nB_IMAGE=\"z@sha256:cd\"\nC_IMAGE=\"host:5000/w:3\"\nD=\"1\"\n"; + assert_eq!(env_images(env), ["x/y", "z", "host:5000/w"]); + } + + #[test] + fn keys_and_proposals_print_as_python() { + let d: IndexMap = match json::loads( + r#"{"depName":"it's","currentValue":"1","updates":[{"newValue":"3"},{"newValue":"2","newDigest":"d"}]}"#, + ) { + Ok(Value::Object(o)) => o, + other => panic!("{other:?}"), + }; + assert_eq!(key(&d), r#"("it's", '1', None)"#); + assert_eq!(proposals(&d), "[('2', 'd'), ('3', '')]"); + } + + /// A repository of the files the check reads, and a log that matches it. + fn planted(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("og-renovate-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + for sub in ["docs", "conf", "scripts/ci", ".github/workflows"] { + fs::create_dir_all(d.join(sub)).expect("mkdir"); + } + let w = |p: &str, t: &str| fs::write(d.join(p), t).expect("write"); + w( + ENV, + "NEXTFLOW_VERSION=\"25.10.8\"\nA_IMAGE=\"x/a:1@sha256:aa\"\nB_IMAGE=\"x/b:2\"\n", + ); + w(DOCS, "| a | x/a:1 |\n"); + w(CONFIG, "container = 'x/a:1'\n"); + w( + ".github/workflows/ci.yml", + " - uses: nf-core/setup-nextflow@abc # v3\n with:\n version: \"25.10.8\"\n", + ); + w( + ".github/workflows/.hidden.yml", + " - uses: nf-core/setup-nextflow@abc # v3\n with:\n version: \"25.10.8\"\n", + ); + w( + PINS, + "# comment\nnextflow-io/nextflow\tversions.env\t^NEXTFLOW_VERSION=\"\nnextflow-io/nextflow\t.github/workflows/*.yml\tuses: [\"\\x27]?nf-core/setup-nextflow@[^\\n]*\\n(?:[ \\t]*(?:[^\\s-][^\\n]*)?\\n)*?[ \\t]+version: [\"\\x27]?\\d\n", + ); + let dep = |n: &str, v: &str, ds: &str| { + format!(r#"{{"depName":"{n}","currentValue":"{v}","datasource":"{ds}"}}"#) + }; + let log = format!( + "not json\n{{\"msg\":\"other\"}}\n{{\"msg\":\"packageFiles with updates\",\"config\":{{\"regex\":[{{\"packageFile\":\"versions.env\",\"deps\":[{},{},{}]}},{{\"packageFile\":\"docs/versions.md\",\"deps\":[{},{}]}},{{\"packageFile\":\"conf/containers.config\",\"deps\":[{}]}},{{\"packageFile\":\".github/workflows/ci.yml\",\"deps\":[{}]}}]}}}}\n", + dep("x/a", "1", "docker"), + dep("x/b", "2", "docker"), + dep("nextflow-io/nextflow", "25.10.8", "github-releases"), + dep("x/a", "1", "docker"), + dep("x/b", "2", "docker"), + dep("x/a", "1", "docker"), + dep("nextflow-io/nextflow", "25.10.8", "github-releases"), + ); + w("renovate.log", &log); + d + } + + fn run(d: &Path) -> (u8, String) { + let args: Vec = [ + d.join("renovate.log").to_string_lossy().into_owned(), + "--repo".to_owned(), + d.to_string_lossy().into_owned(), + ] + .into(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8_lossy(&out).into_owned() + &String::from_utf8_lossy(&err), + ) + } + + #[test] + fn a_clean_tree_passes_and_every_plant_is_caught() { + let d = planted("clean"); + let (code, out) = run(&d); + assert_eq!(code, 0, "{out}"); + assert!( + out.contains("OK: 7 dependencies; every check passed"), + "{out}" + ); + assert!(out.contains("| `x/a` | `1` | none | | 3 |"), "{out}"); + assert_eq!(out.matches("caught: ").count(), 6, "{out}"); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn malformed_json_lines_are_skipped_before_and_after_the_config() { + let d = planted("syntax-errors"); + let log = fs::read(d.join("renovate.log")).unwrap(); + let bad = b"{broken}\n{\"x\":}\n\"\\uZZZZ\"\n"; + fs::write(d.join("renovate.log"), [bad.as_slice(), &log, bad].concat()).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 0, "{out}"); + assert!(out.contains("OK: 7 dependencies; every check passed")); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn json_reader_limits_never_fall_back_to_an_earlier_config() { + let huge = "340282366920938463463374607431768211456"; + let deep = format!( + "{}{}", + "[".repeat(json::MAX_DEPTH + 1), + "]".repeat(json::MAX_DEPTH + 1) + ); + let records = [ + ( + "integer-prefix", + format!("{huge}\n"), + false, + "Integer beyond 128 bits", + ), + ( + "surrogate-prefix", + "\"\\ud800\"\n".into(), + false, + "Lone surrogate", + ), + ( + "final-config", + format!( + "{{\"msg\":\"packageFiles with updates\",\"config\":{{\"regex\":[{{\"packageFile\":\"unused\",\"deps\":null}}]}},\"extra\":{huge}}}\n" + ), + true, + "Integer beyond 128 bits", + ), + ( + "nesting-prefix", + format!("{deep}\n"), + false, + "nested deeper than 1000 levels", + ), + ]; + std::thread::Builder::new() + .stack_size(8 << 20) + .spawn(move || { + let mut misses = Vec::new(); + for (label, record, append, diagnostic) in records { + let d = planted(label); + let old = fs::read_to_string(d.join("renovate.log")).unwrap(); + let log = if append { old + &record } else { record + &old }; + fs::write(d.join("renovate.log"), log).unwrap(); + let (code, out) = run(&d); + if code != 1 || !out.contains(diagnostic) || out.contains("every check passed") + { + misses.push(format!("{label}: exit {code}, {out}")); + } + fs::remove_dir_all(d).unwrap(); + } + assert!(misses.is_empty(), "{}", misses.join("\n")); + }) + .unwrap() + .join() + .unwrap(); + } + + /// The refusal contract applies to every row, including unused input fields. + /// Run each CLI with both a regular output and a symlink to that output. + #[test] + fn refusals_preserve_bytes_modes_and_link_targets() { + // Match the main-thread stack used by og for the nesting-limit control. + std::thread::Builder::new() + .stack_size(8 << 20) + .spawn(refusal_table) + .unwrap() + .join() + .unwrap(); + } + + fn refusal_table() { + use std::os::unix::fs::{PermissionsExt, symlink}; + #[derive(Clone, Copy, Debug)] + enum Verb { + Gvcf, + Clashes, + Trim, + Renovate, + Comphet, + } + enum Fault { + Replace(&'static str, Vec), + Append(&'static str, Vec), + Prefix(Vec), + Limit(&'static str), + LongName, + ReadOnly, + ParentReadOnly, + Directory, + } + use Fault::*; + use Verb::*; + let bytes = |s: &str| s.as_bytes().to_vec(); + let huge = "340282366920938463463374607431768211456"; + let deep = format!( + "{}{}", + "[".repeat(json::MAX_DEPTH + 1), + "]".repeat(json::MAX_DEPTH + 1) + ); + let final_record = format!( + r#"{{"msg":"packageFiles with updates","config":{{"regex":[{{"packageFile":"unused","deps":null}}]}},"extra":{huge}}}"# + ); + // Add a refusal by adding one row: CLI, input fault, diagnostic. + let mut cases = vec![ + ( + Renovate, + Prefix(bytes(&format!("{huge}\n"))), + "Integer beyond 128 bits", + ), + (Renovate, Prefix(bytes("\"\\ud800\"\n")), "Lone surrogate"), + ( + Renovate, + Append("renovate.log", bytes(&format!("{final_record}\n"))), + "Integer beyond 128 bits", + ), + ( + Renovate, + Prefix(bytes(&format!("{deep}\n"))), + "nested deeper than 1000 levels", + ), + ( + Renovate, + Prefix(bytes("42\n")), + "a log record is not an object", + ), + (Renovate, Prefix(vec![0xff, b'\n']), "invalid UTF-8"), + ( + Renovate, + Prefix(vec![b'x'; (16 << 20) + 1]), + "record exceeds 16777216 bytes", + ), + ( + Renovate, + Replace(PINS, bytes("dep\tdocs/*.absent\tx\n")), + "matches no file", + ), + ( + Renovate, + Replace(PINS, bytes("dep\tabsent\tx\n")), + "is not a file", + ), + ( + Gvcf, + Append("ref.fasta.fai", bytes("chrUnused\t1\n")), + "a row has fewer than 5 columns", + ), + ( + Gvcf, + Replace( + "truth.vcf.gz", + bytes("chrA\t18446744073709551615\t.\tA\tC\n"), + ), + "position span overflows", + ), + ( + Gvcf, + Replace("ref.fasta.fai", bytes("\n")), + "a row has fewer than 5 columns", + ), + ( + Gvcf, + Append("ref.fasta.fai", bytes("chrUnused\t0\t0\t0\t0\n")), + "a row has 0 bases per line", + ), + ( + Gvcf, + Replace("truth.vcf.gz", bytes("chrA\t3\t.\tA\n")), + "a record has fewer than 5 columns", + ), + ( + Gvcf, + Replace("truth.bed", bytes("\n")), + "a row has fewer than 3 columns", + ), + ( + Gvcf, + Replace("truth.bed", bytes("chrA\u{85}0\u{85}50\n")), + "unsupported whitespace", + ), + (Gvcf, Replace("sites.tsv", bytes("")), "missing header"), + ( + Gvcf, + Append("truth.vcf.gz", bytes("##late=yes\n")), + "VCF header after first data record", + ), + ( + Clashes, + Replace("raw.vcf.gz", bytes("chrA\t3\t.\tA\tC\n")), + "missing trailing VCF columns", + ), + ( + Trim, + Replace( + "report.json", + bytes(&format!(r#"{{"genes":{{}},"drugs":{{}},"title":{huge}}}"#)), + ), + "Integer beyond 128 bits", + ), + ( + Trim, + Replace( + "report.json", + bytes(r#"{"genes":{},"drugs":{},"title":"\ud800"}"#), + ), + "Lone surrogate", + ), + ( + Trim, + Replace("report.json", bytes(&deep)), + "nested deeper than 1000 levels", + ), + ( + Trim, + Replace("report.json", bytes("{broken}")), + "Expecting property name", + ), + (Trim, Replace("report.json", vec![0xff]), "not UTF-8"), + ( + Trim, + Replace("report.json", bytes("[]")), + "the report is not an object", + ), + ( + Trim, + Replace("report.json", bytes(r#"{"genes":[],"drugs":{}}"#)), + "genes is not an object", + ), + ( + Trim, + Replace("report.json", bytes(r#"{"genes":{},"drugs":[]}"#)), + "drugs is not an object", + ), + ( + Trim, + Replace( + "report.json", + bytes(r#"{"genes":{"G":{"sourceDiplotypes":"x"}},"drugs":{}}"#), + ), + "sourceDiplotypes is not a list", + ), + ( + Trim, + Replace( + "report.json", + bytes(r#"{"genes":{"G":{"recommendationDiplotypes":{ "x":1 }}},"drugs":{}}"#), + ), + "recommendationDiplotypes is not a list", + ), + ( + Trim, + Replace( + "report.json", + bytes(r#"{"genes":{"G":{"sourceDiplotypes":[{"allele1":"x"}]}},"drugs":{}}"#), + ), + "allele1 is not an object", + ), + ( + Trim, + Replace( + "report.json", + bytes(r#"{"genes":{"G":{"sourceDiplotypes":[{"allele2":["x"]}]}},"drugs":{}}"#), + ), + "allele2 is not an object", + ), + ( + Trim, + Replace( + "report.json", + bytes( + r#"{"genes":{},"drugs":{"S":{"D":{"guidelines":[{"annotations":[{"genotypes":[{"diplotypes":{"x":1}}]}]}]}}}}"#, + ), + ), + "diplotypes is not a list", + ), + ]; + for limit in ["1_0", "٤", "18446744073709551616"] { + cases.push((Trim, Limit(limit), "is not a whole number")); + } + for value in ["3_0", "٣", "-3", "18446744073709551616"] { + cases.push(( + Gvcf, + Replace("truth.vcf.gz", bytes(&format!("chrA\t{value}\t.\tA\tC\n"))), + "is not a number", + )); + } + for row in [ + "chrA\t50\t18446744073709551598\t1\t2\n", + "chrA\t50\t1\t1\t18446744073709551615\n", + "chrA\t50\t1000\t50\t51\n", + ] { + cases.push(( + Gvcf, + Replace("ref.fasta.fai", bytes(row)), + "invalid FASTA offset", + )); + } + for bad in [ + r#"{"packageFile":"unused","deps":null}"#, + r#"{"packageFile":[],"deps":[]}"#, + r#"{"packageFile":"unused","deps":[{"updates":{}}]}"#, + r#"{"packageFile":"unused","deps":[{"depName":0}]}"#, + r#"{"packageFile":"unused","deps":[{"currentValue":0}]}"#, + r#"{"packageFile":"unused","deps":[{"updates":[{"newValue":0}]}]}"#, + ] { + cases.push((Renovate, Append("renovate.log", bytes(&format!("{{\"msg\":\"packageFiles with updates\",\"config\":{{\"regex\":[{bad}]}}}}\n"))), "not a map of package file lists")); + } + // A zero-byte gzip file is written directly, without the test compressor. + for (verb, file) in [ + (Gvcf, "truth.vcf.gz"), + (Clashes, "raw.vcf.gz"), + (Clashes, "split.vcf.gz"), + (Clashes, "chrA.tsv.gz"), + ] { + cases.push(( + verb, + Replace(file, Vec::new()), + "wrong compression (expected gzip)", + )); + } + // Plain inputs reject gzip, and compressed inputs reject plain text. + cases.push(( + Gvcf, + Replace("ref.fasta", vec![0x1f, 0x8b, b'A', b'A']), + "wrong compression (expected plain)", + )); + cases.push(( + Gvcf, + Replace("truth.bed", vec![0x1f, 0x8b]), + "wrong compression (expected plain)", + )); + for verb in [Gvcf, Clashes, Trim, Renovate] { + cases.push((verb, LongName, "File name too long")); + } + for verb in [Gvcf, Clashes, Trim, Renovate, Comphet] { + cases.push((verb, ReadOnly, "read-only destination")); + cases.push((verb, ParentReadOnly, "parent directory is not writable")); + cases.push((verb, Directory, "destination is not a regular file")); + } + eprintln!( + "refusal contract: {} rows, {} CLI invocations", + cases.len(), + cases.len() * 2 + ); + for (index, (verb, fault, message)) in cases.into_iter().enumerate() { + for linked in [false, true] { + let d = planted(&format!("refusal-{index}-{linked}")); + let inputs = d.join("inputs"); + fs::create_dir(&inputs).unwrap(); + if matches!(verb, Gvcf | Clashes) { + let kind = if matches!(verb, Gvcf) { + "hg002-gvcf" + } else { + "clashes" + }; + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/inputs") + .join(format!("dev-evidence-{kind}")); + for entry in fs::read_dir(src).unwrap() { + let entry = entry.unwrap(); + fs::copy(entry.path(), inputs.join(entry.file_name())).unwrap(); + } + // Force the offset controls to read position 13. + if matches!(&fault, Replace("ref.fasta.fai", text) if text.starts_with(b"chrA\t50")) + { + fs::write(inputs.join("sites.tsv"), b"id\tchr\tpos\nx\tchrA\t13\n") + .unwrap(); + fs::write(inputs.join("truth.bed"), b"chrA\t0\t50\n").unwrap(); + } + } + fs::write(inputs.join("report.json"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + let input_root = if matches!(verb, Renovate) { + &d + } else { + &inputs + }; + match &fault { + Replace(name, data) | Append(name, data) => { + let path = input_root.join(name); + let mut content = if matches!(fault, Append(..)) { + if name.ends_with(".gz") { + use std::io::Read; + let mut old = Vec::new(); + flate2::read::MultiGzDecoder::new(fs::File::open(&path).unwrap()) + .read_to_end(&mut old) + .unwrap(); + old + } else { + fs::read(&path).unwrap() + } + } else { + Vec::new() + }; + content.extend_from_slice(data); + if name.ends_with(".gz") && !content.is_empty() { + let mut gz = flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + ); + gz.write_all(&content).unwrap(); + content = gz.finish().unwrap(); + } + fs::write(path, content).unwrap(); + } + Prefix(data) => { + let old = fs::read(d.join("renovate.log")).unwrap(); + fs::write(d.join("renovate.log"), [data.as_slice(), &old].concat()) + .unwrap(); + } + _ => {} + } + let outputs = d.join("outputs"); + let targets = d.join("targets"); + fs::create_dir(&outputs).unwrap(); + fs::create_dir(&targets).unwrap(); + let long = "x".repeat(255); + let names = if matches!(verb, Comphet) { + vec!["reads.sam", "vep.vcf", "comphet.vcf"] + } else { + vec![if matches!(fault, LongName) { + &long + } else { + "out" + }] + }; + let mut previous = Vec::new(); + for (n, name) in names.iter().enumerate() { + let output = outputs.join(name); + let target = if linked { + targets.join(name) + } else { + output.clone() + }; + let directory = matches!(fault, Directory) && n == 0; + if directory { + fs::create_dir(&target).unwrap(); + fs::write(target.join("sentinel"), b"PREVIOUS OUTPUT\n").unwrap(); + } else { + fs::write(&target, b"PREVIOUS OUTPUT\n").unwrap(); + } + let mode = if matches!(fault, ReadOnly) && n == 0 { + 0o444 + } else if directory { + 0o755 + } else { + 0o4600 + }; + fs::set_permissions(&target, fs::Permissions::from_mode(mode)).unwrap(); + if linked { + symlink(&target, &output).unwrap(); + } + previous.push((output, target, mode, directory)); + } + let parent = if linked { &targets } else { &outputs }; + if matches!(fault, ParentReadOnly) { + fs::set_permissions(parent, fs::Permissions::from_mode(0o555)).unwrap(); + } + let p = |n: &str| inputs.join(n).to_str().unwrap().to_owned(); + let dest = previous[0].0.to_str().unwrap().to_owned(); + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let code = match verb { + Gvcf => crate::evidence::cli( + &[ + "hg002-gvcf".into(), + p("truth.vcf.gz"), + p("truth.bed"), + p("ref.fasta"), + p("sites.tsv"), + dest, + ], + &mut stdout, + &mut stderr, + ), + Clashes => crate::evidence::cli( + &[ + "clashes".into(), + p("raw.vcf.gz"), + p("split.vcf.gz"), + inputs.to_str().unwrap().into(), + dest, + ], + &mut stdout, + &mut stderr, + ), + Trim => { + let mut args = vec!["trim-pharmcat".into(), p("report.json"), dest]; + if let Limit(limit) = &fault { + args.push((*limit).into()); + } + crate::fixture::cli(&args, &mut stdout, &mut stderr) + } + Renovate => cli_with_summary( + &[ + d.join("renovate.log").to_str().unwrap().into(), + "--repo".into(), + d.to_str().unwrap().into(), + ], + &mut stdout, + &mut stderr, + Some(Path::new(&dest)), + ), + Comphet => crate::fixture::cli( + &["comphet".into(), outputs.to_str().unwrap().into()], + &mut stdout, + &mut stderr, + ), + }; + fs::set_permissions(parent, fs::Permissions::from_mode(0o755)).unwrap(); + let diagnostic = String::from_utf8(stderr).unwrap(); + assert_ne!( + code, 0, + "row {index} {verb:?}, linked={linked}: false success" + ); + assert!( + diagnostic.contains(message), + "row {index} {verb:?}, linked={linked}: want {message:?}, got {diagnostic}" + ); + for (output, target, mode, directory) in previous { + let meta = fs::symlink_metadata(&output).unwrap(); + assert_eq!(meta.file_type().is_symlink(), linked, "row {index}"); + assert_eq!( + fs::read_link(&output).ok(), + linked.then_some(target.clone()), + "row {index}: link target changed" + ); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o7777, + mode, + "row {index}: mode changed" + ); + assert_eq!( + fs::read(if directory { + target.join("sentinel") + } else { + target + }) + .unwrap(), + b"PREVIOUS OUTPUT\n", + "row {index}: bytes changed" + ); + } + for parent in [&outputs, &targets] { + assert!( + !fs::read_dir(parent).unwrap().any(|e| e + .unwrap() + .file_name() + .to_string_lossy() + .contains(".tmp-")), + "row {index}: temporary sibling leaked" + ); + } + fs::remove_dir_all(d).unwrap(); + } + } + } + + #[test] + fn a_pin_renovate_misses_fails() { + let d = planted("pin"); + fs::write( + d.join(".github/workflows/more.yml"), + " - uses: 'nf-core/setup-nextflow@abc'\n\n with:\n version: '26.04.0'\n", + ) + .expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL pins: nextflow-io/nextflow in .github/workflows/more.yml: the text pins it 1 times, Renovate found 0"), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn an_image_renovate_misses_and_a_stale_twin_fail() { + let d = planted("image"); + let log = fs::read_to_string(d.join("renovate.log")).expect("read"); + let log = log.replacen( + r#"{"depName":"x/b","currentValue":"2","datasource":"docker"},"#, + "", + 1, + ); + let log = log.replacen( + r#""docs/versions.md","deps":[{"depName":"x/a","currentValue":"1""#, + r#""docs/versions.md","deps":[{"depName":"x/a","currentValue":"0""#, + 1, + ); + fs::write(d.join("renovate.log"), log).expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL count: x/b (versions.env) is not a detected dependency"), + "{out}" + ); + assert!( + out.contains( + "FAIL twins: docs/versions.md and versions.env differ on ('x/a', '0', None)" + ), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn a_glob_that_matches_nothing_is_refused() { + let d = planted("glob"); + fs::write(d.join(PINS), "aldy\tdocs/*.txt\taldy==\\d\n").expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 2, "{out}"); + assert!(out.contains("docs/*.txt matches no file"), "{out}"); + let _ = fs::remove_dir_all(&d); + } + + #[test] + fn renovate_table_uses_checked_runner() { + let d = planted("status"); + let args = vec![ + d.join("renovate.log").to_str().unwrap().into(), + "--repo".into(), + d.to_str().unwrap().into(), + ]; + // No dependence on the process's GITHUB_STEP_SUMMARY environment. + fn verb(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + cli_with_summary(args, out, err, None) + } + crate::stream_tests::assert_status_errors(verb, &args, true); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn dependency_counts_refuse_overflow() { + let mut count = usize::MAX; + assert!( + add_count(&mut count, 1) + .unwrap_err() + .contains("dependency count overflows") + ); + assert_eq!(count, usize::MAX); + } + + #[test] + fn non_writable_summary_parent_preserves_previous_summary() { + use std::os::unix::fs::PermissionsExt; + let d = planted("summary-parent"); + let parent = d.join("summary-dir"); + fs::create_dir(&parent).unwrap(); + let summary = parent.join("result"); + fs::write(&summary, b"old").unwrap(); + fs::set_permissions(&parent, fs::Permissions::from_mode(0o555)).unwrap(); + let args = vec![ + d.join("renovate.log").to_str().unwrap().into(), + "--repo".into(), + d.to_str().unwrap().into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli_with_summary(&args, &mut out, &mut err, Some(&summary)); + fs::set_permissions(&parent, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(code, 1); + assert!( + String::from_utf8(err) + .unwrap() + .contains(": parent directory is not writable") + ); + assert_eq!(fs::read(&summary).unwrap(), b"old"); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn configured_summary_write_failure_fails_the_check() { + let d = planted("summary"); + let args = vec![ + d.join("renovate.log").to_str().unwrap().to_owned(), + "--repo".to_owned(), + d.to_str().unwrap().to_owned(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli_with_summary(&args, &mut out, &mut err, Some(&d)), 1); + assert!(!String::from_utf8_lossy(&out).contains("every check passed")); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn malformed_log_shapes_and_text_fail() { + for (label, bad) in [ + ("null-deps", r#"{"packageFile":"unused","deps":null}"#), + ( + "null-updates", + r#"{"packageFile":"unused","deps":[{"updates":null}]}"#, + ), + ( + "scalar-update", + r#"{"packageFile":"unused","deps":[{"updates":[42]}]}"#, + ), + ("scalar-dep", r#"{"packageFile":"unused","deps":[42]}"#), + ("scalar-file", r#"{"packageFile":[],"deps":[]}"#), + ( + "map-updates", + r#"{"packageFile":"unused","deps":[{"updates":{}}]}"#, + ), + ( + "number-update", + r#"{"packageFile":"unused","deps":[{"updates":[{"newValue":0}]}]}"#, + ), + ( + "number-name", + r#"{"packageFile":"unused","deps":[{"depName":0}]}"#, + ), + ( + "number-current", + r#"{"packageFile":"unused","deps":[{"currentValue":0}]}"#, + ), + ] { + let d = planted(label); + let log = fs::read_to_string(d.join("renovate.log")).unwrap(); + let log = log.replacen(r#""regex":["#, &format!(r#""regex":[{bad},"#), 1); + fs::write(d.join("renovate.log"), log).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 1, "{label}: {out}"); + assert!( + out.contains("not a map of package file lists"), + "{label}: {out}" + ); + fs::remove_dir_all(d).unwrap(); + } + for (label, prefix) in [ + ("utf8", b"\xff\n".as_slice()), + ("array", b"[]\n".as_slice()), + ("null", b"null\n".as_slice()), + ] { + let d = planted(label); + let log = [prefix, &fs::read(d.join("renovate.log")).unwrap()].concat(); + fs::write(d.join("renovate.log"), log).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 1, "{label}: {out}"); + fs::remove_dir_all(d).unwrap(); + } + } + + #[test] + fn repository_sources_use_universal_newlines() { + for name in [ENV, PINS, ".github/workflows/ci.yml"] { + for ending in ["\r", "\r\n"] { + let d = planted(&format!( + "newlines-{}-{}", + name.replace('/', "-"), + ending.len() + )); + let baseline = run(&d); + let text = fs::read_to_string(d.join(name)) + .unwrap() + .replace('\n', ending); + fs::write(d.join(name), text).unwrap(); + assert_eq!(run(&d), baseline, "{name}/{ending:?}"); + fs::remove_dir_all(d).unwrap(); + } + } + } + + #[test] + fn real_log_falsy_warnings_and_absent_empty_packages_are_accepted() { + for entry in [ + r#"{}"#, + r#"{"deps":[]}"#, + r#"{"packageFile":"unused","deps":[{"warnings":null}]}"#, + r#"{"packageFile":"unused","deps":[{"warnings":false}]}"#, + r#"{"packageFile":"unused","deps":[{"warnings":""}]}"#, + r#"{"packageFile":"unused","deps":[{"warnings":{}}]}"#, + ] { + let d = planted("falsy"); + let log = fs::read_to_string(d.join("renovate.log")).unwrap(); + fs::write( + d.join("renovate.log"), + log.replacen(r#""regex":["#, &format!(r#""regex":[{entry},"#), 1), + ) + .unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 0, "{entry}: {out}"); + fs::remove_dir_all(d).unwrap(); + } + } + + #[test] + fn repository_sources_reject_gzip_even_under_plain_names() { + for name in [ENV, PINS, ".github/workflows/ci.yml"] { + let d = planted(&format!("compressed-{}", name.replace('/', "-"))); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(&fs::read(d.join(name)).unwrap()).unwrap(); + fs::write(d.join(name), gz.finish().unwrap()).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 2, "{out}"); + assert!(out.contains("wrong compression"), "{out}"); + fs::remove_dir_all(d).unwrap(); + } + } + + #[test] + fn renovate_requires_plain_compression_and_pins_record_limit() { + let d = planted("compression-limit"); + let text = fs::read(d.join("renovate.log")).unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(&text).unwrap(); + fs::write(d.join("renovate.log"), gz.finish().unwrap()).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 1); + assert!(out.contains("compression"), "{out}"); + fs::write(d.join("renovate.log"), vec![b'x'; (16 << 20) + 1]).unwrap(); + let (code, out) = run(&d); + assert_eq!(code, 1); + assert!(out.contains("record exceeds 16777216 bytes"), "{out}"); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn a_log_without_the_record_fails() { + let d = planted("nolog"); + fs::write(d.join("renovate.log"), "{\"msg\":\"other\"}\n").expect("write"); + let (code, out) = run(&d); + assert_eq!(code, 1, "{out}"); + assert!( + out.contains("FAIL: no 'packageFiles with updates' record in"), + "{out}" + ); + let _ = fs::remove_dir_all(&d); + } +} diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index c6b6cd901..99ca25ab7 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -78,6 +78,11 @@ const USAGE: &str = "usage: og [args] og exomiser report [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] + og dev fixture comphet OUT [--swap] + og dev fixture trim-pharmcat REPORT OUT [MAX] + og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv + og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf + og dev renovate-check LOG og dev pharmcat diplotypes REPORT [--called|--called-lines] og dev nf-trace requests|attempts TRACE [options] og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args] diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index ba755ddcd..5400ae61d 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -78,7 +78,7 @@ cn-guard-1-mapq0 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/paras cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py +comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh diff --git a/scripts/ci/e2e-scope.sh b/scripts/ci/e2e-scope.sh index 4e254dc56..b3305168d 100755 --- a/scripts/ci/e2e-scope.sh +++ b/scripts/ci/e2e-scope.sh @@ -45,8 +45,9 @@ export LC_ALL=C # runs no case. The push paths of e2e.yml are the same list (e2e-plan checks). SCOPE_DIRS=(scripts modules workflows bin conf assets tests/e2e .github/actions) SCOPE_FILES=(tests/fixtures/VERSION main.nf nextflow.config nextflow_schema.json versions.env .github/workflows/e2e.yml + rust/crates/og-dev/src/fixture.rs # The committed fixtures the cases read (their rows name them too). - tests/fixtures/annotsv/class45.sh tests/fixtures/comphet/make_fixture.py tests/fixtures/consumer_export/sites.tsv.gz + tests/fixtures/annotsv/class45.sh tests/fixtures/consumer_export/sites.tsv.gz tests/fixtures/families/stub_families.tsv tests/fixtures/pharmcat/refblocks.g.vcf tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf tests/fixtures/vcf/grch37_no_contigs.vcf) diff --git a/scripts/ci/renovate-pins.tsv b/scripts/ci/renovate-pins.tsv new file mode 100644 index 000000000..ba15cba87 --- /dev/null +++ b/scripts/ci/renovate-pins.tsv @@ -0,0 +1,25 @@ +# The non-image pins `og dev renovate-check` counts (.github/workflows/renovate-dry-run.yml): +# each must be a dependency Renovate detects in every place it is pinned, as often as the text has it. +# dependencyfile or glob (wildcards in the last part only)regular expression for one pin, +# with ^ and $ at line ends. A missing file or a glob that matches nothing is an error. +# Only the files listed for a dependency are counted: python is also the docker image of +# versions.env, which the image checks cover. +# A setup-nextflow step is a pin when a version: line with a literal version follows its uses: +# line inside the same step (the lines in between are blank or start with anything but '-', +# which opens the next step); a computed version (an expression) is not a pin. +nextflow-io/nextflow versions.env ^NEXTFLOW_VERSION=" +nextflow-io/nextflow .github/workflows/*.yml uses: ["\x27]?nf-core/setup-nextflow@[^\n]*\n(?:[ \t]*(?:[^\s-][^\n]*)?\n)*?[ \t]+version: ["\x27]?\d +nextflow-io/nextflow docs/*.md NXF_VER=\d +rhysd/actionlint .github/workflows/*.yml ACTIONLINT_VERSION=\d +gitleaks/gitleaks .github/workflows/*.yml GITLEAKS_VERSION=\d +koalaman/shellcheck .github/workflows/lint.yml ^[ \t]+version: v\d +renovate .github/workflows/renovate-dry-run.yml RENOVATE_VERSION: \d +renovate docs/updating.md renovate@\d +python .github/workflows/docs.yml python-version: ['\"]?\d +ANHIG/IMGTHLA scripts/lib/common.sh ^HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-\d +PGScatalog/pgsc_calc versions.env ^PGSC_CALC_VERSION=" +PGScatalog/pgsc_calc conf/containers.config ext\.pipeline_version = ' +playwright tests/demo/retake-screenshots.sh playwright==\d +pillow tests/demo/retake-screenshots.sh pillow==\d +pyobjc-framework-Vision tests/demo/retake-screenshots.sh pyobjc-framework-Vision==\d +aldy docs/21-cyrius.md aldy==\d|\[Aldy\]\([^)]*\) v\d diff --git a/tests/e2e/comphet-phasing-1-reads.sh b/tests/e2e/comphet-phasing-1-reads.sh index 2dc4e2ccf..5107b6ca6 100755 --- a/tests/e2e/comphet-phasing-1-reads.sh +++ b/tests/e2e/comphet-phasing-1-reads.sh @@ -2,7 +2,7 @@ # Step 31 phases each compound-het pair from the reads, and the Nextflow # COMPHET_PHASE process writes the same pairs table. # -# The fixture is tests/fixtures/comphet/make_fixture.py: read pairs on a +# The fixture is og dev fixture comphet (tests/fixtures/comphet): read pairs on a # made-up contig (chrP) and a VEP VCF with three pairs, all rare and HIGH: # GENEC SNV at 1000 and a deletion at 1003 on one copy -> cis, 22 reads # GENET SNVs at 5000 and 5400, one on each copy; only a read and its mate @@ -28,7 +28,7 @@ cell() { make_sample() { local s=$1 f="${CASE_TMP}/fixture-$1" rm -rf "$f" "${GENOME_DIR:?}/${s}" - python3 "${REPO}/tests/fixtures/comphet/make_fixture.py" "$f" "${@:2}" + "${REPO}/bin/og" dev fixture comphet "$f" "${@:2}" mkdir -p "${GENOME_DIR}/${s}/vep" "${GENOME_DIR}/${s}/aligned" sed "s/CHPH/${s}/g" "${f}/vep.vcf" > "${GENOME_DIR}/${s}/vep/${s}_vep.vcf" sed "s/CHPH/${s}/g" "${f}/reads.sam" > "${GENOME_DIR}/${s}/aligned/${s}.sam" diff --git a/tests/e2e/vntyper-1.sh b/tests/e2e/vntyper-1.sh index 7cf376973..9e30e05dd 100755 --- a/tests/e2e/vntyper-1.sh +++ b/tests/e2e/vntyper-1.sh @@ -5,7 +5,7 @@ # over MUC1): VNtyper has nothing to read, and the step says "not # assessed" with that reason, never "negative"; # - two BAMs of VNtyper's own public test data (tests/smoke/ -# vntyper_testdata.py reads them out of its Zenodo archive, sha256 +# vntyper_testdata.sh reads them out of its Zenodo archive, sha256 # pinned): their reads over MUC1 carried onto the fixture's header and # added to case 20's BAM, as a whole-genome BAM would hold them. The # sample with a frameshift reads positive, with the variant, and the one @@ -38,7 +38,7 @@ check "the fixture: the message says the BAM has no read over MUC1" has 'no read # --- VNtyper's positive and negative samples on the fixture's header ------------------- check "VNtyper's two test BAMs download and match their sha256" \ - retry python3 -I "${REPO}/tests/smoke/vntyper_testdata.py" "$T" positive.bam negative.bam + retry sh "${REPO}/tests/smoke/vntyper_testdata.sh" "$T" positive.bam negative.bam H="${SAMPLE}muc" for s in positive negative; do dk "$SAMTOOLS_IMAGE" samtools index "${T}/${s}.bam" diff --git a/tests/fixtures/comphet/README.md b/tests/fixtures/comphet/README.md index b0a5816c6..fc17a306e 100644 --- a/tests/fixtures/comphet/README.md +++ b/tests/fixtures/comphet/README.md @@ -1,6 +1,7 @@ # Made-up reads for the compound-het phase (step 31) -`make_fixture.py OUT_DIR [--swap]` writes the fixture: read pairs of 100 bp +`bin/og dev fixture comphet OUT_DIR [--swap]` writes the fixture +(`rust/crates/og-dev/src/fixture.rs`): read pairs of 100 bp from 500 bp fragments on a made-up 80 kb contig (`chrP`, seeded random sequence), a VEP VCF of six heterozygous variants for step 31, and the three pairs as slivar names them. The reads and variants are made up; they describe diff --git a/tests/fixtures/comphet/make_fixture.py b/tests/fixtures/comphet/make_fixture.py deleted file mode 100755 index 38d842fc6..000000000 --- a/tests/fixtures/comphet/make_fixture.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -"""Write the compound-het phasing fixture: made-up reads on a made-up contig. - - make_fixture.py OUT_DIR [--swap] - -writes, for the sample CHPH on contig chrP (80 kb of seeded random sequence): - - reads.sam read pairs of 100 bp, 500 bp fragments (mates 400 bp apart) - vep.vcf the six heterozygous variants with a VEP CSQ, as step 13 writes - them, for step 31 (all rare, HIGH impact, three genes) - comphet.vcf the three pairs as slivar compound-hets names them - (INFO/slivar_comphet), for og comphet phase alone - -The three pairs and what the reads show: - - GENEC SNV at 1000, 2-base deletion at 1003: both on copy 1 (cis); single - reads cover both - GENET SNVs at 5000 and 5400, one on each copy (trans); no single read - covers both, only a read and its mate do - GENEF SNVs at 20000 and 70000, one on each copy; 50 kb apart, so no read - pair covers both and the phase cannot be read (unknown) - ---swap moves GENEC's deletion to copy 2: the same positions, now in trans. -The output is the same for the same arguments; nothing is random at run time. -""" -import os -import random -import sys - -SAMPLE = "CHPH" -CONTIG = "chrP" -LENGTH = 80000 -READ = 100 -FRAG = 500 -FIELDS = ("Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|" - "Existing_variation|MAX_AF|gnomADe_AF|gnomADg_AF|CLIN_SIG") - - -def reference(): - rng = random.Random(31) - return "".join(rng.choice("ACGT") for _ in range(LENGTH)) - - -def other(base): - return "ACGT"["ACGT".index(base) - 1] - - -def variants(seq, swap=False): - """[(gene, pos, ref, alt, consequence, copy)]: pos 1-based, copy 0 or 1.""" - snv = lambda pos: (seq[pos - 1], other(seq[pos - 1])) # noqa: E731 - return [ - ("GENEC", 1000, *snv(1000), "stop_gained", 0), - ("GENEC", 1003, seq[1002:1005], seq[1002], "frameshift_variant", 1 if swap else 0), - ("GENET", 5000, *snv(5000), "stop_gained", 0), - ("GENET", 5400, *snv(5400), "stop_gained", 1), - ("GENEF", 20000, *snv(20000), "stop_gained", 0), - ("GENEF", 70000, *snv(70000), "stop_gained", 1), - ] - - -def read_on(seq, vs, copy, start): - """(sequence, cigar) of a READ-base read from 0-based start on one copy.""" - snvs = {v[1] - 1: v[3] for v in vs if v[5] == copy and len(v[2]) == len(v[3])} - dels = {v[1] + len(v[3]) - 1: len(v[2]) - len(v[3]) for v in vs if v[5] == copy and len(v[2]) > len(v[3])} - out, ops, p = [], [], start - while len(out) < READ: - if p in dels: - ops.append(("D", dels[p])) - p += dels[p] - continue - out.append(snvs.get(p, seq[p])) - ops.append(("M", 1)) - p += 1 - cigar, last, n = "", None, 0 - for op, k in ops + [(None, 0)]: - if op == last: - n += k - continue - if last: - cigar += f"{n}{last}" - last, n = op, k - return "".join(out), cigar - - -def sam(seq, vs): - lines = ["@HD\tVN:1.6\tSO:unsorted", f"@SQ\tSN:{CONTIG}\tLN:{LENGTH}", f"@RG\tID:{SAMPLE}\tSM:{SAMPLE}"] - # Fragment starts (0-based) around each pair: read 1 covers the first - # position, its mate the second one when they are 400 bp apart. - starts = {"c": range(920, 991, 7), "t": range(4905, 4996, 10), "f": range(19905, 19996, 10), - "g": range(69905, 69996, 10)} - for region, rs in starts.items(): - for copy in (0, 1): - for s in rs: - name = f"{region}{copy}_{s}" - m = s + FRAG - READ - s1, c1 = read_on(seq, vs, copy, s) - s2, c2 = read_on(seq, vs, copy, m) - q = "I" * READ - lines.append(f"{name}\t99\t{CONTIG}\t{s + 1}\t60\t{c1}\t=\t{m + 1}\t{FRAG}\t{s1}\t{q}\tRG:Z:{SAMPLE}") - lines.append(f"{name}\t147\t{CONTIG}\t{m + 1}\t60\t{c2}\t=\t{s + 1}\t-{FRAG}\t{s2}\t{q}\tRG:Z:{SAMPLE}") - return "\n".join(lines) + "\n" - - -def vcf_head(info): - return ("##fileformat=VCFv4.2\n" - f"##contig=\n" - '##FILTER=\n' - + info - + '##FORMAT=\n' - '##FORMAT=\n' - '##FORMAT=\n' - '##FORMAT=\n' - f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n") - - -def vep_vcf(vs): - out = vcf_head('##INFO=\n') - for gene, pos, ref, alt, cons, _ in vs: - csq = (f"{alt}|{cons}|HIGH|{gene}|ENSG{gene}|Transcript|ENST{pos}|protein_coding|YES|NM_{pos}.1|" - f"rs{pos}|0.0001|0.0001|0.0001|") - out += f"{CONTIG}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tCSQ={csq}\tGT:GQ:DP:AD\t0/1:50:30:15,15\n" - return out - - -def comphet_vcf(vs): - out = vcf_head('##INFO=\n') - pid = {"GENEC": 1, "GENET": 2, "GENEF": 3} - for i, (gene, pos, ref, alt, _, _) in enumerate(vs): - mate = vs[i + 1] if i % 2 == 0 else vs[i - 1] - tag = f"{SAMPLE}/{gene}/{pid[gene]}/{CONTIG}/{mate[1]}/{mate[2]}/{mate[3]}" - out += f"{CONTIG}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tslivar_comphet={tag}\tGT:GQ:DP:AD\t0/1:50:30:15,15\n" - return out - - -def write(out_dir, swap=False): - seq = reference() - vs = variants(seq, swap) - os.makedirs(out_dir, exist_ok=True) - for name, text in (("reads.sam", sam(seq, vs)), ("vep.vcf", vep_vcf(vs)), ("comphet.vcf", comphet_vcf(vs))): - with open(os.path.join(out_dir, name), "w") as fh: - fh.write(text) - return vs - - -if __name__ == "__main__": - args = [a for a in sys.argv[1:] if a != "--swap"] - if len(args) != 1: - sys.exit(__doc__.split("\n\n")[1]) - write(args[0], swap="--swap" in sys.argv[1:]) diff --git a/tests/fixtures/pharmcat/README.md b/tests/fixtures/pharmcat/README.md index a4451df8d..9e15c7b00 100644 --- a/tests/fixtures/pharmcat/README.md +++ b/tests/fixtures/pharmcat/README.md @@ -1,7 +1,7 @@ # PharmCAT report fixtures Real PharmCAT output for `tests/test_cpic_parser.py` (one `report-.json` per release in its `REAL_REPORTS`), trimmed with -`trim_report.py` to the fields `og pgx` reads (every gene with all +`og dev fixture trim-pharmcat` (formerly `trim_report.py`, which wrote the files below) to the fields `og pgx` reads (every gene with all its listed diplotypes and `relatedDrugs`; up to 4 matched annotations per drug guideline). @@ -16,7 +16,7 @@ with `VERSION` replaced by the PharmCAT release (for example `3.4.0`), and add the release to `REAL_REPORTS` in `tests/test_cpic_parser.py`: ```bash -python3 tests/fixtures/pharmcat/trim_report.py report.json tests/fixtures/pharmcat/report-VERSION.json +bin/og dev fixture trim-pharmcat report.json tests/fixtures/pharmcat/report-VERSION.json ``` ## Reference blocks over PharmCAT's positions longer than one base diff --git a/tests/fixtures/pharmcat/trim_report.py b/tests/fixtures/pharmcat/trim_report.py deleted file mode 100755 index 33adcc5a2..000000000 --- a/tests/fixtures/pharmcat/trim_report.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""trim_report.py: cut a PharmCAT report.json down to a test fixture. - - python3 tests/fixtures/pharmcat/trim_report.py REPORT.json OUT.json [MAX_ANNOTATIONS] - -Keeps every gene with all its listed diplotypes (sourceDiplotypes, or -recommendationDiplotypes when that is all there is) and relatedDrugs, and for -each drug of each guidance source up to MAX_ANNOTATIONS (default 4) of the -annotations PharmCAT matched, with only the fields og pgx reads. -Writes compact JSON. See README.md for how each fixture was made. -""" -import json -import sys - -src, dst = sys.argv[1], sys.argv[2] -MAX_ANN = int(sys.argv[3]) if len(sys.argv) > 3 else 4 -d = json.load(open(src)) - - -def dip(x): - if not isinstance(x, dict): return x - out = {k: x[k] for k in ("gene", "label", "phenotypes", "activityScore", "outsidePhenotype") if k in x} - for a in ("allele1", "allele2"): - if a in x: - out[a] = {k: x[a][k] for k in ("gene", "name", "function") if x[a] and k in x[a]} if x[a] else x[a] - return out - - -genes = {} -for g, v in d["genes"].items(): - genes[g] = {k: v[k] for k in ("geneSymbol", "chr", "phased", "callSource") if k in v} - genes[g]["relatedDrugs"] = [{"name": r["name"], "id": r.get("id")} for r in v.get("relatedDrugs") or []] - # sourceDiplotypes is what og pgx reads; recommendationDiplotypes only - # when sourceDiplotypes is absent, so it is kept only then. - k = "sourceDiplotypes" if v.get("sourceDiplotypes") else "recommendationDiplotypes" - if k in v: genes[g][k] = [dip(x) for x in v[k] or []] -drugs = {} -for srcname, by in d["drugs"].items(): - for name, rep in by.items(): - gls = [] - for gl in rep.get("guidelines") or []: - anns = [] - for a in (gl.get("annotations") or [])[:MAX_ANN]: - anns.append({"phenotypes": a.get("phenotypes"), "classification": a.get("classification"), - "drugRecommendation": a.get("drugRecommendation"), - "genotypes": [{"diplotypes": [{"gene": x.get("gene"), "label": x.get("label")} - for x in (gt.get("diplotypes") or []) if isinstance(x, dict)]} - for gt in (a.get("genotypes") or [])]}) - if anns: gls.append({"name": gl.get("name"), "source": gl.get("source"), "annotations": anns}) - if gls: - drugs.setdefault(srcname, {})[name] = {"name": rep.get("name"), "id": rep.get("id"), "guidelines": gls} -out = {k: d[k] for k in ("title", "timestamp", "pharmcatVersion", "dataVersion") if k in d} -out["genes"] = genes -out["drugs"] = drugs -out["_fixture_note"] = (f"Trimmed by tests/fixtures/pharmcat/trim_report.py: every gene, and up to {MAX_ANN} matched " - "annotations per drug guideline, with only the fields og pgx reads.") -json.dump(out, open(dst, "w"), separators=(",", ":")) diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 025f2c9cc..198b93015 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -982,6 +982,20 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +dev-fixture-comphet plain - out out/reads.sam,out/vep.vcf,out/comphet.vcf bytes PYTHON_IMAGE +dev-fixture-comphet swap - out --swap out/reads.sam,out/vep.vcf,out/comphet.vcf bytes PYTHON_IMAGE +dev-fixture-trim-pharmcat report-3.2.0 gen:pgx-fixtures.sh report-3.2.0.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat report-3.4.0 gen:pgx-fixtures.sh report-3.4.0.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat pharmcat-docs-example gen:pgx-fixtures.sh pharmcat-docs-example.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat one-annotation gen:pgx-fixtures.sh report-3.4.0.json out.json 1 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat all-but-last gen:pgx-fixtures.sh pharmcat-docs-example.json out.json -1 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-evidence-clashes synthetic dev-evidence-clashes/raw.vcf.gz,dev-evidence-clashes/split.vcf.gz,dev-evidence-clashes/chrA.tsv.gz,dev-evidence-clashes/chrB.tsv.gz,dev-evidence-clashes/chrZ.tsv.gz,dev-evidence-clashes/notes.tsv.gz raw.vcf.gz split.vcf.gz . affected.tsv affected.tsv bytes PYTHON_IMAGE +dev-evidence-hg002-gvcf synthetic dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites.tsv truth.vcf.gz truth.bed ref.fasta sites.tsv out.vcf out.vcf bytes PYTHON_IMAGE +dev-evidence-hg002-gvcf past-the-end dev-evidence-hg002-gvcf/truth.vcf.gz,dev-evidence-hg002-gvcf/truth-past-end.bed,dev-evidence-hg002-gvcf/ref.fasta,dev-evidence-hg002-gvcf/ref.fasta.fai,dev-evidence-hg002-gvcf/sites-past-end.tsv truth.vcf.gz truth-past-end.bed ref.fasta sites-past-end.tsv out.vcf out.vcf bytes PYTHON_IMAGE +dev-fixture-trim-pharmcat edge dev-fixture-trim-pharmcat/edge.json edge.json out.json out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge2 dev-fixture-trim-pharmcat/edge.json edge.json out.json 2 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge0 dev-fixture-trim-pharmcat/edge.json edge.json out.json 0 out.json json:mask=_fixture_note PYTHON_IMAGE +dev-fixture-trim-pharmcat edge-all-but-last dev-fixture-trim-pharmcat/edge.json edge.json out.json -1 out.json json:mask=_fixture_note PYTHON_IMAGE dev-pharmcat flat ci-readers/oracle.sh,ci-readers/flat.json flat.json - bytes PYTHON_IMAGE dev-pharmcat-called flat ci-readers/oracle.sh,ci-readers/flat.json flat.json --called - bytes PYTHON_IMAGE dev-pharmcat-lines flat ci-readers/oracle.sh,ci-readers/flat.json flat.json --called-lines - bytes PYTHON_IMAGE diff --git a/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf new file mode 100644 index 000000000..1e494f38c --- /dev/null +++ b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf @@ -0,0 +1,11 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +##INFO= +##ALT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG002 +chrA 3 . G T 50 PASS . GT 0/1 +chrA 10 . CGT C 50 PASS . GT 0/1 +chrB 8 . C T 50 PASS . GT 0/1 +chrC 1 . A G 50 PASS . GT 0/1 +chrB 30 . A <*> . . END=30 GT 0/0 diff --git a/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout new file mode 100644 index 000000000..233285d28 --- /dev/null +++ b/tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/stdout @@ -0,0 +1 @@ +gVCF: 4 GIAB records, 1 reference blocks at panel sites diff --git a/tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz new file mode 100644 index 000000000..5dc766e8d Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/chrA.tsv.gz differ diff --git a/tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz new file mode 100644 index 000000000..2ff2824f7 Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/chrB.tsv.gz differ diff --git a/tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz new file mode 100644 index 000000000..987e91c2c Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/chrZ.tsv.gz differ diff --git a/tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz b/tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz new file mode 100644 index 000000000..035133bbf Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/notes.tsv.gz differ diff --git a/tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz b/tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz new file mode 100644 index 000000000..386cc0414 Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/raw.vcf.gz differ diff --git a/tests/parity/inputs/dev-evidence-clashes/split.vcf.gz b/tests/parity/inputs/dev-evidence-clashes/split.vcf.gz new file mode 100644 index 000000000..4a4e80c1e Binary files /dev/null and b/tests/parity/inputs/dev-evidence-clashes/split.vcf.gz differ diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta new file mode 100644 index 000000000..b37a59afb --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta @@ -0,0 +1,7 @@ +>chrA test +ACGTACGTACGTACGTACGT +NNACGTacgtACGTACGTAC +GTACGTACGT +>chrB test +ACNNNTGCAtgcaACGTACG +TACGTACGTA diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai new file mode 100644 index 000000000..9980f0dd1 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/ref.fasta.fai @@ -0,0 +1,2 @@ +chrA 50 11 20 21 +chrB 30 75 20 21 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv new file mode 100644 index 000000000..bb13404a5 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites-past-end.tsv @@ -0,0 +1,4 @@ +id chrom pos +s1 chrB 30 +s2 chrB 31 +s3 chrB 35 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv new file mode 100644 index 000000000..b6a517815 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/sites.tsv @@ -0,0 +1,19 @@ +id chrom pos ref +s0 chrA 3 X +s1 chrA 11 X +s2 chrA 12 X +s3 chrA 13 X +s4 chrA 15 X +s5 chrA 16 X +s6 chrA 19 X +s7 chrA 21 X +s8 chrA 27 X +s9 chrA 45 X +s10 chrA 46 X +s11 chrB 3 X +s12 chrB 8 X +s13 chrB 10 X +s14 chrB 30 X +s15 chrC 1 X +s16 chrD 4 X +s17 chrA 1 X diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed new file mode 100644 index 000000000..d6806d6ff --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth-past-end.bed @@ -0,0 +1 @@ +chrB 0 40 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed new file mode 100644 index 000000000..6a2ed2e11 --- /dev/null +++ b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.bed @@ -0,0 +1,4 @@ +chrA 18 45 +chrA 0 15 +chrB 0 30 +chrC 0 10 diff --git a/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz new file mode 100644 index 000000000..f13c00115 Binary files /dev/null and b/tests/parity/inputs/dev-evidence-hg002-gvcf/truth.vcf.gz differ diff --git a/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json b/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json new file mode 100644 index 000000000..dd46aeb6b --- /dev/null +++ b/tests/parity/inputs/dev-fixture-trim-pharmcat/edge.json @@ -0,0 +1,25 @@ +{"title":"edge é","timestamp":"t","pharmcatVersion":"3.4.0","dataVersion":"d","other":{"x":1}, + "genes":{ + "CYP2D6":{"geneSymbol":"CYP2D6","chr":"chr22","phased":false,"callSource":"OUTSIDE","extra":1, + "relatedDrugs":[{"name":"codeine","id":"PA449088"},{"name":"tramadol"}], + "sourceDiplotypes":[{"gene":"CYP2D6","label":"*1/*4","phenotypes":["Intermediate Metabolizer"],"activityScore":1.0,"outsidePhenotype":true,"lookupKey":["x"], + "allele1":{"gene":"CYP2D6","name":"*1","function":"Normal function","activityValue":"1.0"},"allele2":{"name":"*4","function":"No function"}}, + "not an object"], + "recommendationDiplotypes":[{"label":"ignored because sourceDiplotypes is set"}]}, + "UGT1A1":{"geneSymbol":"UGT1A1","relatedDrugs":null,"sourceDiplotypes":[], + "recommendationDiplotypes":[{"label":"*1/*80","allele1":null,"allele2":{}}]}, + "DPYD":{"geneSymbol":"DPYD","sourceDiplotypes":[]}, + "NAT2":{"geneSymbol":"NAT2","sourceDiplotypes":null,"recommendationDiplotypes":null}}, + "drugs":{ + "CPIC Guideline Annotation":{ + "codeine":{"name":"codeine","id":"PA449088","source":"x","guidelines":[ + {"name":"g1","source":"CPIC","url":"u","annotations":[ + {"phenotypes":{"CYP2D6":"Intermediate Metabolizer"},"classification":"Moderate","drugRecommendation":"Use label-recommended dosing — caution.","implications":["dropped"], + "genotypes":[{"diplotypes":[{"gene":"CYP2D6","label":"*1/*4","x":1},"skip me",{"label":"no gene"}]},{}]}, + {"classification":"Strong"}, + {"classification":"Optional"}]}, + {"name":"g2","annotations":[]}, + {"name":"g3"}]}, + "tramadol":{"name":"tramadol","guidelines":[]}}, + "DPWG Guideline Annotation":{ + "empty":{"name":"empty","guidelines":null}}}} diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 809f44861..372370311 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -63,6 +63,10 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +dev-fixture-comphet fe6b67e09cef81c396c2277472df10407c56d7ac file:tests/fixtures/comphet/make_fixture.py python3 dev fixture comphet +dev-fixture-trim-pharmcat fe6b67e09cef81c396c2277472df10407c56d7ac file:tests/fixtures/pharmcat/trim_report.py python3 dev fixture trim-pharmcat +dev-evidence-clashes fe6b67e09cef81c396c2277472df10407c56d7ac file:.github/workflows/public-evidence.yml sh -c 'd=$(mktemp -d) && awk -v n=1 "/<<.PY.\$/ {k++; if (k == n) {on = 1; next}} on && /^ *PY\$/ {exit} on {sub(/^ /, \"\"); print}" "$0" > "$d/p.py" && exec python3 -I "$d/p.py" "$@"' dev evidence clashes +dev-evidence-hg002-gvcf fe6b67e09cef81c396c2277472df10407c56d7ac file:.github/workflows/public-evidence.yml sh -c 'd=$(mktemp -d) && awk -v n=2 "/<<.PY.\$/ {k++; if (k == n) {on = 1; next}} on && /^ *PY\$/ {exit} on {sub(/^ /, \"\"); print}" "$0" > "$d/p.py" && exec python3 -I "$d/p.py" "$@"' dev evidence hg002-gvcf dev-pharmcat 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:scripts/ci/parity-diff.sh sh -c 'exec bash ./oracle.sh diplotypes "$0" "$@"' dev pharmcat diplotypes dev-pharmcat-called 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:scripts/ci/image-smoke.sh sh -c 'exec bash ./oracle.sh called "$0" "$@"' dev pharmcat diplotypes dev-pharmcat-lines 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:tests/e2e/lib.sh sh -c 'exec bash ./oracle.sh called-lines "$0" "$@"' dev pharmcat diplotypes diff --git a/tests/smoke/commands.tsv b/tests/smoke/commands.tsv index e7387040c..039419d5d 100644 --- a/tests/smoke/commands.tsv +++ b/tests/smoke/commands.tsv @@ -52,7 +52,7 @@ PARASCOPY_IMAGE net,also=PARASCOPY_DATA_VERSION fullref,slice sh /smoke/parascop PARAPHASE_IMAGE net fullref v=${PARAPHASE_IMAGE##*:} && u=https://raw.githubusercontent.com/PacificBiosciences/paraphase/v${v%%--*}/tests/data/bams/HG01175_smn1_extracted.bam && wget -q -O hg.bam "$u" && wget -q -O hg.bam.bai "${u}.bai" && { printf '%s hg.bam\n%s hg.bam.bai\n' c9659364ad9e60e41ee4771cf92dc017313b6b75a1e08df3e8c0e95ef38eadca 8b5dd238fdf1cc508e85105ff2a157a2c3d73aa3f441d08afdba9458beecca51 | sha256sum -c || { echo 'the demo BAM or its index is not the file Paraphase v4.1.0 ships'; exit 1; }; } && paraphase -b hg.bam -r /in/ref.fa -o pp -p HG01175 -g smn1 -t 4 && cp pp/HG01175.paraphase.json smn1.json && rm -rf pp hg.bam hg.bam.bai jqcheck "SMN1, SMN2 and SMN exon 7-8 deletion copy numbers of HG01175 (2, 1, 1)" smn1.json '.smn1.region_specific_info | [.smn1_cn, .smn2_cn, .smn_del78_cn] | all(.[]; type == "number") and . == [2, 1, 1]' # MEGAnE's call needs a whole chromosome: on mini.fa's 500 kb it stops in its own code (IndexError). The e2e case mei-1 runs it on chr20. MEGANE_IMAGE - ref build_kmerset -fa /in/mini.fa -prefix mini -outdir /out/mk && { call_genotype_38 -h > help.txt 2>&1 || true; } at_least "repeated 32-mers in the k-mer set of mini.fa" "$(cat mk/mini.mi)" 1000; same "the .mk holds the 8-byte k-mers the .mi counts" "$(( $(wc -c < mk/mini.mk) / 8 ))" "$(cat mk/mini.mi)"; contains "call_genotype_38 takes the -mainchr step 50 passes" help.txt "-mainchr"; contains "and -no_sex_chr" help.txt "-no_sex_chr"; contains "and -only_ins" help.txt "-only_ins" -# Step 52 on VNtyper's own test data, two BAMs read out of its Zenodo archive by range requests (tests/smoke/vntyper_testdata.py, +# Step 52 on VNtyper's own test data, two BAMs read out of its Zenodo archive by range requests (tests/smoke/vntyper_testdata.sh, # sha256 pinned): a sample with a MUC1 VNTR frameshift must be called, and one without must be negative (the red control); # a BAM with no read over MUC1 must stop VNtyper with the message step 52 turns into "no read over MUC1". VNTYPER_IMAGE net - sh /smoke/vntyper.sh has "the positive sample: Kestrel calls a frameshift with high precision" "$(awk -F'\t' '$1 == "positive" {print $2}' calls.tsv)" "^High_Precision"; same "the negative sample: no call" "$(awk -F'\t' '$1 == "negative" {print $2}' calls.tsv)" Negative; at_least "the positive sample: mean depth over the VNTR" "$(awk -F'\t' '$1 == "positive" {print int($4)}' calls.tsv)" 100; at_least "the negative sample: mean depth over the VNTR" "$(awk -F'\t' '$1 == "negative" {print int($4)}' calls.tsv)" 100; same "a BAM with no read over MUC1: VNtyper stops with exit 1" "$(cat empty_rc.txt)" 1; contains "and its log names the empty read layout step 52 reads" empty.log "FASTQ layout .empty."; has "vntyper --version is the release of the image tag" "$(cat version.txt)" "${VNTYPER_IMAGE##*:}" diff --git a/tests/smoke/vntyper.sh b/tests/smoke/vntyper.sh index 0c3d7c7bb..e726c3182 100644 --- a/tests/smoke/vntyper.sh +++ b/tests/smoke/vntyper.sh @@ -1,12 +1,18 @@ # shellcheck shell=sh # VNTYPER_IMAGE row: what step 52 runs, on VNtyper's own public test data -# (tests/smoke/vntyper_testdata.py fetches two BAMs out of its Zenodo archive): +# (tests/smoke/vntyper_testdata.sh fetches two BAMs out of its Zenodo archive): # a sample with a MUC1 VNTR frameshift and one without. The image's samtools # indexes them, then VNtyper runs with the step's flags (and no IGV view in its HTML) # from /opt/vntyper, # the folder its config.json paths are relative to. set -e -python3 /smoke/vntyper_testdata.py /out/data positive.bam negative.bam +sh /smoke/vntyper_testdata.sh /out/data positive.bam negative.bam +# Hashing stdin must also work when the destination contains a newline. +newline_out='/out/new +line' +sh /smoke/vntyper_testdata.sh "$newline_out" negative.bam +cmp /out/data/negative.bam "$newline_out/negative.bam" +rm -rf "$newline_out" for s in positive negative; do samtools index "/out/data/${s}.bam" (cd /opt/vntyper && vntyper pipeline --bam "/out/data/${s}.bam" -o "/out/${s}" --sample-name "$s" \ diff --git a/tests/smoke/vntyper_testdata.py b/tests/smoke/vntyper_testdata.py deleted file mode 100644 index b33731d94..000000000 --- a/tests/smoke/vntyper_testdata.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -"""Fetch members of VNtyper's public test data without the whole archive. - -VNtyper publishes its test BAMs as one 1.2 GB zip on Zenodo (record 19181821, -"VNtyper 2.0 test data", the archive_file of tests/test_data_config.json in -VNtyper v2.0.47). A zip keeps its file list at the end, so this script reads -that list with an HTTP range request, then fetches and inflates only the -members it is asked for, about 10 MB instead of 1.2 GB (the .bai files are -not fetched: samtools index writes them). Each member is checked -against the CRC-32 the archive stores and against the sha256 pinned below. - -Usage: vntyper_testdata.py OUTDIR NAME... (NAME: a key of MEMBERS) -Standard library only: it runs in VNTYPER_IMAGE's Python and on the host. -""" -import hashlib -import os -import struct -import sys -import time -import urllib.request -import zlib - -URL = "https://zenodo.org/records/19181821/files/data.zip?download=1" -SIZE = 1174975271 # bytes; Zenodo lists md5 fdd5958d58c733cdca452f3287898964 for the whole file - -# name -> (path in the archive, sha256 of the inflated file) -MEMBERS = { - # Positive: a MUC1 VNTR frameshift VNtyper calls High_Precision* (its - # integration test example_dfc3_GRCh38_fast runs the same reads on NCBI - # contig names). Reads realigned by VNtyper to GRCh38 chr1, UCSC names. - "positive.bam": ("data/remapped/bwa/hg38/example_dfc3_hg38_bwa.bam", - "5934330a8ac2144ac0cc6ae8c84d7d9262e90b28127417e4068f7d662b7c64bb"), - # Negative: no variant called (its integration test - # example_40cf_hg38_subset_fast_gdp_guard), mean VNTR depth about 148x. - "negative.bam": ("data/example_40cf_hg38_subset.bam", - "44aff8e3b2096b7ea3529989660052c61b29c8a26afd7f48de4988f7047d0652"), -} - - -def get(start, end): - """Bytes start..end (inclusive) of the archive, with retries.""" - for attempt in range(4): - try: - req = urllib.request.Request(URL, headers={"Range": f"bytes={start}-{end}"}) - with urllib.request.urlopen(req, timeout=120) as r: - if r.status != 206: - raise OSError(f"HTTP {r.status}, want 206 (a range)") - data = r.read() - if len(data) != end - start + 1: - raise OSError(f"got {len(data)} bytes, want {end - start + 1}") - return data - except OSError as e: - if attempt == 3: - raise - print(f"range {start}-{end}: {e}; again in {15 * (attempt + 1)} s", file=sys.stderr) - time.sleep(15 * (attempt + 1)) - raise AssertionError("unreachable") - - -def directory(): - """{path: (method, crc, compressed size, size, local header offset)}""" - tail = get(SIZE - 65536, SIZE - 1) - i = tail.rfind(b"PK\x05\x06") - if i < 0: - raise SystemExit("no end-of-central-directory record in the last 64 kB") - _, _, _, _, n, cd_size, cd_off, _ = struct.unpack("> 8 & 255)) $(($1 >> 16 & 255)) $(($1 >> 24 & 255)))" +} + +[ $# -ge 2 ] || { echo "usage: vntyper_testdata.sh OUTDIR NAME... (NAME of positive.bam, negative.bam)" >&2; exit 2; } +out=$1 +shift +for name in "$@"; do + member "$name" > /dev/null || { echo "vntyper_testdata.sh: unknown member ${name} (positive.bam, negative.bam)" >&2; exit 2; } +done +# GNU mv's -T rejects a directory even if it appears during publication. +# Homebrew calls it gmv on macOS; the pinned Linux image calls it mv. +if command -v gmv > /dev/null 2>&1; then move="gmv"; else move="mv"; fi + +# Resolve file symlinks, including relative/dangling targets, before staging. +destination() { + dest="${out}/${1}" + links=0 + while [ -L "$dest" ]; do + links=$((links + 1)) + [ "$links" -le 40 ] || { echo "destination: too many symlinks: $dest" >&2; return 1; } + target=$(readlink "$dest"; printf x) + target=${target%x} + target=${target%' +'} + case $target in + /*) dest=$target ;; + *) dest="${dest%/*}/$target" ;; + esac + done + [ ! -d "$dest" ] || { echo "destination is a directory: $dest" >&2; return 1; } + if [ -e "$dest" ]; then + [ -f "$dest" ] && [ -w "$dest" ] || { echo "destination is not a writable regular file: $dest" >&2; return 1; } + fi +} + +work= +cleanup() { [ -z "$work" ] || rm -rf "$work"; } +trap cleanup EXIT +trap 'exit 1' HUP INT TERM +mkdir -p "$out" +# A bad destination must fail before any network request. +for name in "$@"; do destination "$name"; done +for name in "$@"; do + read -r path off csize size crc want <&2; exit 1; } + # gzip member: magic, deflate, no flags, no time, no extra flags, unknown OS; + # then the raw deflate stream; then CRC-32 and size, both little-endian. + { printf '\037\213\010\000\000\000\000\000\000\377'; cat "${part}.raw"; le32 "$((0x$crc))"; le32 "$size"; } > "${part}.gz" + gzip -dc "${part}.gz" > "$part" || { echo "${path}: size or CRC-32 differs from the archive's directory" >&2; exit 1; } + got=$(sha256sum < "$part" | cut -d' ' -f1) + [ "$got" = "$want" ] || { echo "${path}: sha256 ${got}, pinned ${want}" >&2; exit 1; } + rm -f "${part}.raw" "${part}.gz" + # Preserve the Python output mode. New files inherit the caller's umask. + if [ -e "$dest" ]; then + case $(uname -s) in + Darwin) mode=$(stat -f '%Lp' "$dest") ;; + *) mode=$(stat -c '%a' "$dest") ;; + esac + chmod "$mode" "$part" + fi + "$move" -fT "$part" "$dest" + cleanup + work= + printf '%s\t%s\t%s\t%s\n' "$name" "$path" "$size" "$got" +done