refactor(ci): move workflow inline Python into og dev verbs

.github/workflows/e2e.yml

@@ -65,9 +65,9 @@ on:
       - 'nextflow_schema.json'
       - 'versions.env'
       - '.github/workflows/e2e.yml'
+      - 'rust/crates/og-dev/src/fixture.rs'
       - '.github/actions/**'
       - 'tests/fixtures/annotsv/class45.sh'
-      - 'tests/fixtures/comphet/make_fixture.py'
       - 'tests/fixtures/consumer_export/sites.tsv.gz'
       - 'tests/fixtures/families/stub_families.tsv'
       - 'tests/fixtures/pharmcat/refblocks.g.vcf'

.github/workflows/guard.yml

@@ -315,6 +315,9 @@ jobs:
       - name: cargo test
         run: cargo test --workspace --locked --no-fail-fast
 
+      - name: Evidence offset regression in release mode
+        run: cargo test --release --locked -p og-dev --no-fail-fast fai_offsets_reject_overflow_and_out_of_file_before_publication
+
       - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation)
         if: ${{ !cancelled() }}   # runs even when a step above failed, so each result shows
         run: |

.github/workflows/public-evidence.yml

@@ -166,6 +166,11 @@ jobs:
           name: archaic-1kg-sites
           path: sites
 
+      - name: Build og as scripts/setup.sh --og does (og dev evidence)
+        run: |
+          scripts/setup.sh --og
+          rust/og.sh check --required
+
       - name: setup.sh --archaic and step 43 on HG002 (chromosome ${{ matrix.part }})
         id: run
         continue-on-error: ${{ matrix.part == 'empty' }}
@@ -230,47 +235,7 @@ jobs:
           # same-position rule of archaic_report.py decides).
           docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \
             bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz
-          python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY'
-          import glob, gzip, os, sys
-          raw, split, data, out = sys.argv[1:]
-          tables = sorted(glob.glob(os.path.join(data, "chr*.tsv.gz")))
-          chroms = {os.path.basename(t)[:-7] for t in tables}  # this part's chromosomes only
-          def clashes(path):
-              """Positions of this part with two records or more, one an SNV (the VCF is sorted)."""
-              hit, key, n, snvs = set(), None, 0, 0
-              for line in gzip.open(path, "rt"):
-                  if line.startswith("#"):
-                      continue
-                  c = line.split("\t", 6)
-                  if c[0] not in chroms:
-                      continue
-                  k = (c[0], int(c[1]))
-                  if k != key:
-                      key, n, snvs = k, 0, 0
-                  n += 1
-                  snvs += len(c[3]) == 1 and len(c[4]) == 1
-                  if n >= 2 and snvs >= 1:
-                      hit.add(key)
-              return hit
-          sets = {"raw": clashes(raw), "split": clashes(split)}
-          on_table = {k: 0 for k in sets}
-          for t in tables:
-              chrom = os.path.basename(t)[:-7]
-              want = {k: {p for c, p in v if c == chrom} for k, v in sets.items()}
-              if not any(want.values()):
-                  continue
-              for line in gzip.open(t, "rt"):
-                  c = line.split("\t", 2)
-                  if c[0] != chrom:
-                      continue
-                  p = int(c[1])
-                  for k in want:
-                      if p in want[k]:
-                          on_table[k] += 1
-          with open(out, "w") as f:
-              for k in sets:
-                  f.write(f"{k}_positions\t{len(sets[k])}\n{k}_on_table\t{on_table[k]}\n")
-          PY
+          bin/og dev evidence clashes "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv"
           cat "$OUT/affected.tsv"
 
       - name: The red control failed, as it must
@@ -597,6 +562,11 @@ jobs:
           gzip -dc "${R}/${f}.gz" > "${R}/GRCh38_no_alt_analysis_set.fasta"
           rm -f "${R}/${f}.gz"
 
+      - name: Build og as scripts/setup.sh --og does (og dev evidence)
+        run: |
+          scripts/setup.sh --og
+          rust/og.sh check --required
+
       - name: setup.sh --ancient-panel and step 44 on HG002, measured
         env:
           THREADS: '4'
@@ -672,57 +642,9 @@ jobs:
           mkdir -p "${G}/HG002/vcf"
           curl -fsSL --retry 5 -o "${EV}/truth.vcf.gz" "${GIAB_HG002}.vcf.gz"
           curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed"
-          python3 -I - "${EV}/truth.vcf.gz" "${EV}/truth.bed" "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \
-              "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" <<'PY'
-          import bisect, gzip, sys
-          truth, bed, ref, sites, out = sys.argv[1:]
-          fai = {l.split("\t")[0]: [int(x) for x in l.split("\t")[1:5]] for l in open(ref + ".fai")}
-          fa = open(ref, "rb")
-          def base(c, p):
-              n, off, lb, lw = fai[c]
-              fa.seek(off + ((p - 1) // lb) * lw + (p - 1) % lb)
-              return fa.read(1).decode().upper()
-          conf = {}
-          for l in open(bed):
-              c, s, e = l.split()[:3]
-              conf.setdefault(c, []).append((int(s), int(e)))
-          starts = {c: [s for s, _ in sorted(v)] for c, v in conf.items()}
-          ends = {c: [e for _, e in sorted(v)] for c, v in conf.items()}
-          def confident(c, p):
-              i = bisect.bisect_right(starts.get(c, []), p - 1) - 1
-              return i >= 0 and ends[c][i] >= p
-          covered, head, body = {}, [], []
-          for l in gzip.open(truth, "rt"):
-              if l.startswith("##"):
-                  head.append(l)
-                  continue
-              if l.startswith("#"):
-                  head.append('##INFO=<ID=END,Number=1,Type=Integer,Description="End of the reference block">\n')
-                  head.append('##ALT=<ID=*,Description="Any other allele">\n')
-                  head.append(l)
-                  continue
-              f = l.split("\t", 5)
-              p = int(f[1])
-              covered.setdefault(f[0], set()).update(range(p, p + len(f[3])))
-              body.append(l)
-          blocks = 0
-          with open(sites) as s:
-              next(s)
-              for l in s:
-                  _, c, p = l.split("\t")[:3]
-                  p = int(p)
-                  if c not in fai or p in covered.get(c, ()) or not confident(c, p):
-                      continue
-                  b = base(c, p)
-                  if b not in "ACGT":
-                      continue
-                  body.append(f"{c}\t{p}\t.\t{b}\t<*>\t.\t.\tEND={p}\tGT\t0/0\n")
-                  blocks += 1
-          with open(out, "w") as o:
-              o.writelines(head)
-              o.writelines(body)
-          print(f"gVCF: {len(body) - blocks} GIAB records, {blocks} reference blocks at panel sites")
-          PY
+          /usr/bin/time -v -o "${M}/gvcf.time" bin/og dev evidence hg002-gvcf "${EV}/truth.vcf.gz" "${EV}/truth.bed" \
+            "${G}/reference/GRCh38_no_alt_analysis_set.fasta" "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf"
+          grep -E 'Maximum resident|Elapsed' "${M}/gvcf.time"
           docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/ev" -v "${G}:/g" "$BCFTOOLS_IMAGE" bash -euo pipefail -c '
             bcftools sort -m 2G -T /ev/sort -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /ev/hg002.unsorted.vcf
             bcftools index -t /g/HG002/vcf/HG002.g.vcf.gz'