fix(ci): fail a personal-data scan that scanned nothing

.github/workflows/e2e.yml

@@ -65,9 +65,9 @@ on:
       - 'nextflow_schema.json'
       - 'versions.env'
       - '.github/workflows/e2e.yml'
+      - 'rust/crates/og-dev/src/fixture.rs'
       - '.github/actions/**'
       - 'tests/fixtures/annotsv/class45.sh'
-      - 'tests/fixtures/comphet/make_fixture.py'
       - 'tests/fixtures/consumer_export/sites.tsv.gz'
       - 'tests/fixtures/families/stub_families.tsv'
       - 'tests/fixtures/pharmcat/refblocks.g.vcf'

.github/workflows/freshness.yml

@@ -129,10 +129,16 @@ jobs:
       - name: og dev freshness self-test (a bogus image, a missing tag and a bogus URL must exit non-zero)
         run: rust/target/release/og dev freshness --self-test
 
-      - name: check-links.sh self-test (a 404, no answer and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage)
+      # check-links.sh and check-pgs-labels.sh read with og dev versions
+      # expand and og dev pgs (rust/crates/og-dev), the og built above.
+      - name: check-links.sh self-test (a 404, no answer, an unreadable versions.env line and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage)
+        env:
+          OG: ${{ github.workspace }}/rust/target/release/og
         run: scripts/ci/check-links.sh --self-test
 
-      - name: check-pgs-labels.sh self-test (a wrong label, an empty answer and no answer must fail)
+      - name: check-pgs-labels.sh self-test (a wrong label, an empty answer, no answer and an effect of true must fail)
+        env:
+          OG: ${{ github.workspace }}/rust/target/release/og
         run: scripts/ci/check-pgs-labels.sh --self-test
 
   pull-request:
@@ -158,37 +164,38 @@ jobs:
             echo "report=$(has 'rust/crates/og-fresh/.*|versions\.env|\.github/workflows/freshness\.yml')"
           } >> "$GITHUB_OUTPUT"
 
-      - name: Every download URL this pull request adds answers
-        env:
-          BASE: ${{ github.event.pull_request.base.sha }}
-        run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY"
-
-      - name: Every PGS ID's label matches the PGS Catalog
-        if: steps.changed.outputs.prs == 'true'
-        run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY"
-
-      # og dev freshness (rust/crates/og-fresh) writes the report: the
-      # toolchain rust/rust-toolchain.toml pins, og built without the BAM
-      # reader, which the report never needs.
+      # og dev freshness (rust/crates/og-fresh) writes the report, and the
+      # link and label checks read with og dev versions expand and og dev
+      # pgs: the toolchain rust/rust-toolchain.toml pins, og built without
+      # the BAM reader, which none of them needs.
       - name: Install the toolchain rust/rust-toolchain.toml pins
-        if: steps.changed.outputs.report == 'true'
         working-directory: rust
         run: |
           channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml)
           rustup toolchain install "$channel" --profile minimal
           rustc --version
 
       - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
-        if: steps.changed.outputs.report == 'true'
         with:
           workspaces: rust
           key: freshness
 
       - name: Build og
-        if: steps.changed.outputs.report == 'true'
         working-directory: rust
         run: cargo build --release --locked -p og --no-default-features
 
+      - name: Every download URL this pull request adds answers
+        env:
+          BASE: ${{ github.event.pull_request.base.sha }}
+          OG: ${{ github.workspace }}/rust/target/release/og
+        run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Every PGS ID's label matches the PGS Catalog
+        if: steps.changed.outputs.prs == 'true'
+        env:
+          OG: ${{ github.workspace }}/rust/target/release/og
+        run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY"
+
       - name: Report dry run (no issue is written)
         if: steps.changed.outputs.report == 'true'
         env:
@@ -236,6 +243,7 @@ jobs:
         id: checks
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          OG: ${{ github.workspace }}/rust/target/release/og
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           out="${RUNNER_TEMP}"

.github/workflows/nextflow.yml

@@ -119,8 +119,8 @@ jobs:
       - name: Validate nextflow_schema.json and the samplesheet schema
         if: steps.scope.outputs.run == 'true'
         run: |
-          python3 -m json.tool nextflow_schema.json > /dev/null
-          python3 -m json.tool assets/schema_input.json > /dev/null
+          jq -s -e 'length == 1' nextflow_schema.json > /dev/null
+          jq -s -e 'length == 1' assets/schema_input.json > /dev/null
 
       # The stub run below pulls every image of the default tools: the pinned
       # leg ran out of disk in it twice on 2026-11-05 (#300, job 127077180393;
@@ -363,6 +363,8 @@ jobs:
       # the image versions.env gives it.
       - name: Every process ran and states the tag of its image from versions.env
         if: steps.scope.outputs.run == 'true'
+        env:
+          OG: ${{ github.workspace }}/rust/target/release/og   # og dev versions check-run, built above
         run: |
           cat "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml"
           scripts/ci/gen-containers-config.sh --check-versions "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml"
@@ -430,83 +432,10 @@ jobs:
           nextflow run main.nf -profile test_all -stub -ansi-log false -c "${RUNNER_TEMP}/resources.config" \
             --tools vep,manta,cnvpytor,prs --max_cpus 16 --max_memory 64.GB --max_time 48.h \
             --outdir "${RUNNER_TEMP}/resources" -work-dir "${RUNNER_TEMP}/resources-work"
-          python3 - "${RUNNER_TEMP}"/resources/pipeline_info/trace_*.txt "$exec_cpus" "$exec_mem_gb" <<'PY'
-          import csv, sys
-          GB, H = 1024 ** 3, 3600 * 1000
-          exec_cpus, exec_mem = int(sys.argv[2]), int(sys.argv[3]) * GB
-          # process: (cpus, memory, time) of the first attempt
-          want = {
-              'DEEPVARIANT':   (8, 32 * GB, 48 * H),
-              'FASTP':         (8, 8 * GB, 8 * H),
-              'ALIGN_MINIMAP2': (8, 32 * GB, 24 * H),
-              'MANTA':         (8, 16 * GB, 24 * H),
-              'CNVPYTOR':      (4, 8 * GB, 24 * H),
-              'VEP':           (8, 16 * GB, 24 * H),
-              'PRS_PREPARE':   (1, 2 * GB, 2 * H),
-              'PRS_SCORE_SITES': (2, 4 * GB, 4 * H),
-              'PRS':           (4, 24 * GB, 8 * H),
-          }
-          seen, bad = {p: {} for p in want}, 0
-          attempts = all_cpus = all_mem = 0
-          rows = {}
-          with open(sys.argv[1]) as fh:
-              for row in csv.DictReader(fh, delimiter='\t'):
-                  first_row = rows.setdefault(row['task_id'], row)
-                  if first_row is not row:
-                      # The late copy can be built before the exit status was
-                      # set, so it may read '-' there; nothing else may differ.
-                      exits = {first_row['exit'], row['exit']} - {'-'}
-                      if len(exits) > 1 or any(first_row[k] != row[k] for k in row if k != 'exit'):
-                          print(f"FAIL: task {row['task_id']} has two different rows in the trace: {first_row} and {row}")
-                          bad += 1
-                      else:
-                          print(f"{row['name']}: attempt {row['attempt']} is written twice in the trace; counted once")
-                      continue
-                  attempts += 1
-                  all_cpus += int(row['cpus'])
-                  all_mem += int(row['memory'])
-                  proc = row['name'].split(' (')[0].split(':')[-1]
-                  if proc not in want:
-                      continue
-                  got = (int(row['cpus']), int(row['memory']), int(row['time']))
-                  print(f"{row['name']}: attempt {row['attempt']}, exit {row['exit']}, "
-                        f"cpus {got[0]}, memory {got[1] / GB:g} GB, time {got[2] / H:g} h")
-                  seen[proc].setdefault(int(row['attempt']), {}).setdefault(row['name'], []).append(got)
-          for proc, (cpus, mem, time) in want.items():
-              first, second = seen[proc].get(1, {}), seen[proc].get(2, {})
-              unexpected = sorted(set(seen[proc]) - {1, 2})
-              if unexpected:
-                  print(f"FAIL {proc}: attempt(s) {unexpected} in the trace, want only a first attempt and one retry")
-                  bad += 1
-              if not first:
-                  print(f"FAIL {proc}: no first attempt in the trace")
-                  bad += 1
-                  continue
-              for name in sorted(set(first) | set(second)):
-                  n1, n2 = len(first.get(name, [])), len(second.get(name, []))
-                  if n1 != 1 or n2 != 1:
-                      print(f"FAIL {name}: {n1} first and {n2} second attempts in the trace, want one of each")
-                      bad += 1
-              for got in (g for gs in first.values() for g in gs):
-                  if got != (cpus, mem, time):
-                      print(f"FAIL {proc}: first attempt asks for cpus {got[0]}, memory {got[1] / GB:g} GB, "
-                            f"time {got[2] / H:g} h; want {cpus}, {mem / GB:g} GB, {time / H:g} h")
-                      bad += 1
-              for got in (g for gs in second.values() for g in gs):
-                  if got[1] != 2 * mem:
-                      print(f"FAIL {proc}: the retry asks for {got[1] / GB:g} GB, want {2 * mem / GB:g} GB")
-                      bad += 1
-          # Every attempt takes its request once and gives it back once. If
-          # the executor has at least the sum, losing every release still
-          # leaves room for the next task.
-          print(f"{attempts} task attempts ask for {all_cpus} CPUs and {all_mem / GB:g} GB added together; "
-                f"the executor is told it has {exec_cpus} CPUs and {exec_mem / GB:g} GB")
-          if all_cpus > exec_cpus or all_mem > exec_mem:
-              print("FAIL: executor.cpus or executor.memory of this step is below that sum; raise exec_cpus / exec_mem_gb")
-              bad += 1
-          print(f"{bad} failure(s)")
-          sys.exit(1 if bad else 0)
-          PY
+          traces=("${RUNNER_TEMP}"/resources/pipeline_info/trace_*.txt)
+          [ "${#traces[@]}" -eq 1 ] && [ -f "${traces[0]}" ] || { echo "FAIL: expected exactly one resource trace"; exit 2; }
+          rust/target/release/og dev nf-trace requests "${traces[0]}" --want scripts/ci/nf-trace-requests.tsv \
+            --executor-cpus "$exec_cpus" --executor-mem-gb "$exec_mem_gb"
 
       # The step above fakes the kill with beforeScript, so it cannot see how
       # ALIGN_MINIMAP2's own pipe reports one. This runs the module's real
@@ -574,28 +503,8 @@ jobs:
             --max_cpus 16 --max_memory 64.GB --outdir "$t/out" -work-dir "$t/work" || rc=$?
           echo "nextflow exited ${rc}"
           cat "$t/trace.txt"
-          python3 - "$t/trace.txt" "$rc" <<'PY'
-          import csv, sys
-          GB = 1024 ** 3
-          # One row per task_id: the trace can list a finished task twice
-          # (the retry step above says why), and the first row is the one
-          # written when the task completed.
-          # A repeated row must match the first; only its exit may read '-'.
-          rows = {}
-          for r in csv.DictReader(open(sys.argv[1]), delimiter='\t'):
-              first = rows.setdefault(r['task_id'], r)
-              exits = {first['exit'], r['exit']} - {'-'}
-              if len(exits) > 1 or any(first[k] != r[k] for k in r if k != 'exit'):
-                  print(f"FAIL: task {r['task_id']} has two different rows in the trace: {first} and {r}")
-                  sys.exit(1)
-          rows = [r for r in rows.values() if 'ALIGN_MINIMAP2' in r['name']]
-          got = [(int(r['attempt']), r['exit'], int(r['memory']) // GB) for r in rows]
-          want = [(1, '137', 32), (2, '0', 64)]
-          print(f"ALIGN_MINIMAP2 attempts (attempt, exit, GB): {got}; want {want}")
-          ok = sorted(got) == want and sys.argv[2] == '0'
-          print('OK' if ok else 'FAIL: a kill inside the pipe did not reach Nextflow as exit 137 and a retry')
-          sys.exit(0 if ok else 1)
-          PY
+          rust/target/release/og dev nf-trace attempts "$t/trace.txt" --name-glob '*ALIGN_MINIMAP2*' \
+            --want '1:137:32,2:0:64' --nextflow-exit "$rc"
 
       # PRS runs pgsc_calc, a Nextflow pipeline of its own, and gives it the
       # task's memory as --max_memory, the most one pgsc_calc task may have. A
@@ -696,26 +605,10 @@ jobs:
             --max_cpus 16 --max_memory 64.GB --outdir "$t/out" -work-dir "$t/work" || rc=$?
           echo "nextflow exited ${rc}"
           cat "$t/trace.txt"
-          python3 - "$t/trace.txt" "$rc" <<'PY'
-          import csv, sys
-          GB = 1024 ** 3
-          # One row per task_id, as in the ALIGN_MINIMAP2 step above.
-          rows = {}
-          for r in csv.DictReader(open(sys.argv[1]), delimiter='\t'):
-              first = rows.setdefault(r['task_id'], r)
-              exits = {first['exit'], r['exit']} - {'-'}
-              if len(exits) > 1 or any(first[k] != r[k] for k in r if k != 'exit'):
-                  print(f"FAIL: task {r['task_id']} has two different rows in the trace: {first} and {r}")
-                  sys.exit(1)
-          rows = list(rows.values())
-          want, ok = [(1, '137', 24), (2, '0', 48)], sys.argv[2] == '0'
-          for sample in ('sigkill', 'pool'):
-              got = sorted((int(r['attempt']), r['exit'], int(r['memory']) // GB) for r in rows if r['name'] == f'PRS ({sample})')
-              print(f"PRS ({sample}) attempts (attempt, exit, GB): {got}; want {want}")
-              ok = ok and got == want
-          print('OK' if ok else 'FAIL: a memory kill inside pgsc_calc did not reach Nextflow as exit 137 and a retry that passes')
-          sys.exit(0 if ok else 1)
-          PY
+          for sample in sigkill pool; do
+            rust/target/release/og dev nf-trace attempts "$t/trace.txt" --name-glob "PRS ($sample)" \
+              --want '1:137:24,2:0:48' --nextflow-exit "$rc"
+          done
 
       # A Nextflow run that fails, or that its step's timeout cuts, leaves its
       # log in the checkout as .nextflow.log (older runs of the job as

.github/workflows/personal-data.yml

@@ -19,11 +19,32 @@ concurrency:
 jobs:
   personal-data:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 20
+    env:
+      OG: ${{ github.workspace }}/rust/target/release/og
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count must be caught)
+      # The scan is og dev scan personal-data (rust/crates/og-dev): the
+      # toolchain rust/rust-toolchain.toml pins, og built without the BAM
+      # reader, which the scan never needs.
+      - name: Install the toolchain rust/rust-toolchain.toml pins
+        working-directory: rust
+        run: |
+          channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml)
+          rustup toolchain install "$channel" --profile minimal
+          rustc --version
+
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          workspaces: rust
+          key: personal-data
+
+      - name: Build og
+        working-directory: rust
+        run: cargo build --release --locked -p og --no-default-features
+
+      - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count, and a scan of no text file, must be caught)
         run: scripts/ci/check-personal-data.sh --self-test
 
       - name: No tracked file holds a home path, a private address or a genotype