diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 05cf8d80e..11fade087 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,9 +65,9 @@ on: - 'nextflow_schema.json' - 'versions.env' - '.github/workflows/e2e.yml' + - 'rust/crates/og-dev/src/fixture.rs' - '.github/actions/**' - 'tests/fixtures/annotsv/class45.sh' - - 'tests/fixtures/comphet/make_fixture.py' - 'tests/fixtures/consumer_export/sites.tsv.gz' - 'tests/fixtures/families/stub_families.tsv' - 'tests/fixtures/pharmcat/refblocks.g.vcf' diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml index 491342c72..2df8e31e8 100644 --- a/.github/workflows/freshness.yml +++ b/.github/workflows/freshness.yml @@ -129,10 +129,16 @@ jobs: - name: og dev freshness self-test (a bogus image, a missing tag and a bogus URL must exit non-zero) run: rust/target/release/og dev freshness --self-test - - name: check-links.sh self-test (a 404, no answer and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage) + # check-links.sh and check-pgs-labels.sh read with og dev versions + # expand and og dev pgs (rust/crates/og-dev), the og built above. + - name: check-links.sh self-test (a 404, no answer, an unreadable versions.env line and an empty tree must fail; a github.com 5xx is asked again, and one that lasts fails naming the outage) + env: + OG: ${{ github.workspace }}/rust/target/release/og run: scripts/ci/check-links.sh --self-test - - name: check-pgs-labels.sh self-test (a wrong label, an empty answer and no answer must fail) + - name: check-pgs-labels.sh self-test (a wrong label, an empty answer, no answer and an effect of true must fail) + env: + OG: ${{ github.workspace }}/rust/target/release/og run: scripts/ci/check-pgs-labels.sh --self-test pull-request: @@ -158,20 +164,11 @@ jobs: echo "report=$(has 'rust/crates/og-fresh/.*|versions\.env|\.github/workflows/freshness\.yml')" } >> "$GITHUB_OUTPUT" - - name: Every download URL this pull request adds answers - env: - BASE: ${{ github.event.pull_request.base.sha }} - run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Every PGS ID's label matches the PGS Catalog - if: steps.changed.outputs.prs == 'true' - run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY" - - # og dev freshness (rust/crates/og-fresh) writes the report: the - # toolchain rust/rust-toolchain.toml pins, og built without the BAM - # reader, which the report never needs. + # og dev freshness (rust/crates/og-fresh) writes the report, and the + # link and label checks read with og dev versions expand and og dev + # pgs: the toolchain rust/rust-toolchain.toml pins, og built without + # the BAM reader, which none of them needs. - name: Install the toolchain rust/rust-toolchain.toml pins - if: steps.changed.outputs.report == 'true' working-directory: rust run: | channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) @@ -179,16 +176,26 @@ jobs: rustc --version - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - if: steps.changed.outputs.report == 'true' with: workspaces: rust key: freshness - name: Build og - if: steps.changed.outputs.report == 'true' working-directory: rust run: cargo build --release --locked -p og --no-default-features + - name: Every download URL this pull request adds answers + env: + BASE: ${{ github.event.pull_request.base.sha }} + OG: ${{ github.workspace }}/rust/target/release/og + run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Every PGS ID's label matches the PGS Catalog + if: steps.changed.outputs.prs == 'true' + env: + OG: ${{ github.workspace }}/rust/target/release/og + run: scripts/ci/check-pgs-labels.sh | tee -a "$GITHUB_STEP_SUMMARY" + - name: Report dry run (no issue is written) if: steps.changed.outputs.report == 'true' env: @@ -236,6 +243,7 @@ jobs: id: checks env: GITHUB_TOKEN: ${{ github.token }} + OG: ${{ github.workspace }}/rust/target/release/og RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | out="${RUNNER_TEMP}" diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 304360fbd..d1ca63a11 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -33,12 +33,25 @@ jobs: # two files generated from it (conf/containers.config, docs/versions.md) # are up to date. Each script's self-test plants its faults first. images: + needs: og runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Self-test (a literal image in a script, module, workflow or doc, a process with no selector and a stale generated file must be caught) + # The scans and the run-versions check are og dev verbs + # (rust/crates/og-dev): the og the og job built. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: og-unit-tests + path: ${{ runner.temp }}/og + + - name: og for the checks (OG) + run: | + chmod 755 "$RUNNER_TEMP/og/og" + echo "OG=${RUNNER_TEMP}/og/og" >> "$GITHUB_ENV" + + - name: Self-test (a literal image in a script, module, workflow or doc, a stale exemption, a process with no selector and a stale generated file must be caught) run: scripts/ci/check-images.sh --self-test - name: Self-test of the containers.config generator (a stale tag, a missing or stray selector, a module's own container and a wrong version must be caught) @@ -138,7 +151,8 @@ jobs: # og for the unit tests (tests/og_binary.py): the tests of a ported tool # and the demo sample builder run og as a subprocess. Built once here, so - # the three shards do not each spend minutes of their 10 on it. + # the three shards do not each spend minutes of their 10 on it. The images + # job runs its og dev scans with it too. og: runs-on: ubuntu-latest timeout-minutes: 20 @@ -251,6 +265,30 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Keep Python 3.9's regex cases and the newer Unicode tables available. The + # scanner fails closed on version-dependent scalars; span iteration + # intentionally uses Python 3.14+ semantics for empty-text \B. + # Standalone builds also provide Python 3.9 on newer ubuntu-latest + # images where actions/setup-python has no matching distribution. + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: '0.12.18' + enable-cache: false + + - name: Install all three Unicode oracle interpreters + run: | + uv python install --no-bin 3.9 3.12 3.15 + interpreters="" + for version in 3.9 3.12 3.15; do + python=$(uv python find --managed-python "$version") + "$python" -I -c 'import sys,unicodedata; print(sys.version.split()[0],unicodedata.unidata_version)' + interpreters="${interpreters:+$interpreters:}$python" + done + echo "PERSONAL_DATA_PYTHONS=$interpreters" >> "$GITHUB_ENV" + echo "PERSONAL_DATA_REQUIRE_PYTHONS=3.9:3.12:3.15" >> "$GITHUB_ENV" + # Span iteration intentionally follows the newest interpreter. + dirname "$python" >> "$GITHUB_PATH" + - name: Install the toolchain rust/rust-toolchain.toml pins, with rustfmt and clippy run: | channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) @@ -279,9 +317,226 @@ jobs: grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." + - name: Control (an unreadable PharmCAT report reported as success must fail) + run: | + log="${RUNNER_TEMP}/ci-reader-control.log" + rc=0 + cargo test --locked --no-fail-fast -p og-dev --features planted-ci-reader --test ci_readers \ + unreadable_pharmcat_is_unassessed > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] || { echo "ERROR: the planted unreadable-report success passed"; exit 1; } + grep -E 'unreadable_pharmcat_is_unassessed.*FAILED' "$log" + echo "Control: the unreadable-report success was reported as a failed test." + - name: Traits controls (diplotype order, REF check and missing phase rows) + env: + CARGO_BUILD_JOBS: 3 + run: | + src=crates/og-steps/src/traits.rs + saved="${RUNNER_TEMP}/traits.rs" + log="${RUNNER_TEMP}/traits-controls.log" + cp -f "$src" "$saved" + trap 'cp -f "$saved" "$src"' EXIT + perl -0pi -e 's/(fn allowed\b.*?)(\n out\n)/$1\n out.reverse();$2/s; s/if c\[2\]\.chars\(\)\.next\(\) != s\.row\["ref"\]\.chars\(\)\.next\(\)/if c[2].is_empty()/; s/(fn phase_vcf\b.*?)( let mut out = AtomicFile::create\(path\))/$1 sites.clear();\n$2/s' "$src" + cmp -s "$saved" "$src" && { echo "ERROR: no traits control was planted"; exit 1; } + if cargo test --locked -p og-steps traits::tests --no-fail-fast > "$log" 2>&1; then + cat "$log" + echo "ERROR: traits accepted the planted faults" + exit 1 + fi + cat "$log" + for test in apoe_grid_keeps_haplotype_order_and_missing_calls stream_prefers_first_snv_but_checks_every_ref_and_bounds_memory phase_requires_one_block_and_only_multiple_hets_are_exported; do + grep -E "traits::tests::${test} .*FAILED" "$log" || { echo "ERROR: $test did not catch its fault"; exit 1; } + done + cp -f "$saved" "$src" + trap - EXIT + - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # Each pinned test-only commit still contains the scanner it attacks. + - name: Scanner controls (both rounds must fail against their pre-fix code) + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/personal-data-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/personal-data-fixed.rs" "$source"; cp "${RUNNER_TEMP}/matcher-fixed.rs" "$matcher"' EXIT + for round in 1 2; do + if [ "$round" = 1 ]; then + before=80a675eadf73640c2a0f2d63fcfe30e6fdd0deef + filter=privacy_ + tests="report_never_contains_values_or_patterns python_boundaries_differential python_casefold_differential staged_newline_differential fixed_seed_differential_fuzz" + else + before=fcb7ac4ec4dc547074f5b4e12699ae9ecb1c24ef + filter=round2_ + tests="boundary_backtracking custom_unicode cr_patterns empty_alternative exception_lines" + fi + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch --quiet --depth=1 origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-red-controls-${round}.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast "$filter" > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] || { echo "ERROR: the pre-fix scanner passed"; exit 1; } + for test in $tests; do + grep -E "${filter}${test} .*FAILED" "$log" || { echo "ERROR: missing red control $test"; exit 1; } + done + echo "Control: all five scanner regressions in round ${round} failed, as they must." + done + + - name: Scanner round-three controls (nullable operands and raw-byte paths must fail before the fix) + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/round3-personal-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/round3-matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/round3-personal-fixed.rs" "$source"; cp "${RUNNER_TEMP}/round3-matcher-fixed.rs" "$matcher"' EXIT + before=3c2c2f288c5ac516c22ed85b6aa8b04e43e734c6 + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch --quiet --depth=1 origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-round3-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round3_ > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + for test in nullable_atoms_are_refused nullable_repetitions_fail_closed raw_byte_paths_fail_closed; do + grep -E "round3_${test} .*FAILED" "$log" || { echo "ERROR: missing round-three red control"; exit 1; } + done + echo "Control: nullable operands and raw-byte paths failed before the fix." + + - name: Unicode guard regenerates from pinned official data + run: bash ../tests/parity/gen/personal-unicode-generate.sh "$RUNNER_TEMP/personal-unicode" + + - name: Scanner round-four controls (Unicode versions and symlink targets fail before the fix) + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/round4-personal-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/round4-matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/round4-personal-fixed.rs" "$source"; cp "${RUNNER_TEMP}/round4-matcher-fixed.rs" "$matcher"' EXIT + before=ac030e631afa5550f2c128a0998d2cb95dee14db + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch --quiet --depth=1 origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-round4-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round4_ > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + for test in unicode_versions_fail_closed symlink_targets_fail_closed; do + grep -E "round4_${test} .*FAILED" "$log" || { echo "ERROR: missing round-four red control"; exit 1; } + done + echo "Control: all 24 Unicode and four symlink scenarios failed before the fix." + + - name: Scanner round-five controls (all five findings must fail before the fix) + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/round5-personal-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/round5-matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/round5-personal-fixed.rs" "$source"; cp "${RUNNER_TEMP}/round5-matcher-fixed.rs" "$matcher"' EXIT + before=f155efd343e82ebf8b53a563ee4b2c5b6f4dbf85 + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch --quiet --depth=1 origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-round5-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round5_ > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + for test in regex_case_relations stable_case_scalars_pass gitlinks_fail_closed guard_filenames every_symlink_hop; do + grep -E "round5_${test} .*FAILED" "$log" || { echo "ERROR: missing round-five red control"; exit 1; } + done + echo "Control: all five round-five regressions failed before the fix." + + - name: Scanner round-six controls (astral cases, range interiors and symlink names fail before the fix) + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/round6-personal-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/round6-matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/round6-personal-fixed.rs" "$source"; cp "${RUNNER_TEMP}/round6-matcher-fixed.rs" "$matcher"' EXIT + before=152bf7860e95951dc585259dcf9f4c66869e36db + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch --quiet --depth=1 origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-round6-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round6_ > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + for test in astral_composite_cases guarded_range_interiors symlink_unicode_names; do + grep -E "round6_${test} .*FAILED" "$log" || { echo "ERROR: missing round-six red control"; exit 1; } + done + grep -E 'round6_stable_ranges_and_uncased_astral .*ok' "$log" + echo "Control: all three regressions failed; stable ranges and uncased astral text passed." + + - name: Scanner round-seven controls against the pre-fix scanner + run: | + source=crates/og-dev/src/personal_data.rs + matcher=crates/og-dev/src/lookaround.rs + cp "$source" "${RUNNER_TEMP}/round7-personal-fixed.rs" + cp "$matcher" "${RUNNER_TEMP}/round7-matcher-fixed.rs" + trap 'cp "${RUNNER_TEMP}/round7-personal-fixed.rs" "$source"; cp "${RUNNER_TEMP}/round7-matcher-fixed.rs" "$matcher"' EXIT + before=0c881620fe1bee894d6a868ab5d7f06f055bbad3 + git cat-file -e "$before^{commit}" 2>/dev/null || git fetch origin "$before" + git show "$before:rust/$source" > "$source" + git show "$before:rust/$matcher" > "$matcher" + log="${RUNNER_TEMP}/scanner-round7-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round7_ > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + for test in ordinary_cyrillic pattern_case_scope cached_target_before_skip required_interpreters_fail_closed; do + grep -E "round7_${test} .*FAILED" "$log" || { echo "ERROR: missing round-seven red control"; exit 1; } + done + echo "Control: Cyrillic text, pattern scope, indexed targets and required oracles all failed before the fix." + + - name: Regex relation verifier rejects a missing table + run: | + source=crates/og-dev/src/personal_data.rs + cp "$source" "${RUNNER_TEMP}/regex-case-fixed.rs" + trap 'cp "${RUNNER_TEMP}/regex-case-fixed.rs" "$source"' EXIT + awk '/^const REGEX_CASE_RELATIONS:/ { print "const REGEX_CASE_RELATIONS: &[(u32, u32)] = &[];"; drop=1 } + drop && /^];/ { drop=0; next } + !drop { print }' "${RUNNER_TEMP}/regex-case-fixed.rs" > "$source" + log="${RUNNER_TEMP}/regex-table-red.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round5_regex_case_table_matches_interpreters > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + grep 'checked-in regex table misses a differing pair' "$log" + echo "Control: real regex engines rejected the incomplete table." + + - name: Scanner matcher controls (word classes and swallowed errors must be caught) + run: | + source=crates/og-dev/src/lookaround.rs + saved="${RUNNER_TEMP}/matcher-controls-fixed.rs" + cp "$source" "$saved" + trap 'cp "$saved" "$source"' EXIT + sed 's/^const WORD: .*$/const WORD: \&str = r"[\\w]";/' "$saved" > "$source" + log="${RUNNER_TEMP}/matcher-word-control.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast --lib > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + grep -E 'round2_generated_patterns_and_tables .*FAILED' "$log" + grep -E 'finditer_spans_match_real_python .*FAILED' "$log" + grep -E 'exhaustive_small_model_spans .*FAILED' "$log" + sed 's/Some(Err(()))/None/' "$saved" > "$source" + log="${RUNNER_TEMP}/matcher-error-control.log" + rc=0 + cargo test -p og-dev --locked --no-fail-fast round2_unsupported > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] + grep -E 'round2_unsupported_and_exhausted_patterns_fail_closed .*FAILED' "$log" + echo "Control: incorrect word classes and a swallowed matcher error both failed." + + - name: Evidence offset regression in release mode + run: cargo test --release --locked -p og-dev --no-fail-fast fai_offsets_reject_overflow_and_out_of_file_before_publication + - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation) if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows run: | @@ -351,7 +606,7 @@ jobs: tests/test_results_index.py \ tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_callability.py; do + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_genotype_rules_indels.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in @@ -363,6 +618,43 @@ jobs: done exit "$fail" + - name: Traits independent expectations reject changed reference labels and marker metadata + working-directory: . + env: + OG: ${{ github.workspace }}/bin/og + OG_REQUIRED: 1 + run: | + python3 - <<'PY' + import contextlib, io, pathlib, runpy, tempfile + test = runpy.run_path("tests/test_traits_report.py")["main"].__globals__ + original = pathlib.Path(test["RULES"]).read_text() + with tempfile.TemporaryDirectory() as work: + for rid, key, label in [("APOL1", "G0/G0", "G0/G0"), ("CAD9p21", "nonrisk/nonrisk", "no risk copy")]: + path = pathlib.Path(work) / "rules.tsv" + old = f"combination\t{rid}\t{key}\t{label}\t" + assert old in original + path.write_text(original.replace(old, f"combination\t{rid}\t{key}\tWRONG_REFERENCE\t")) + test["RULES"], test["FAILS"] = str(path), 0 + log = io.StringIO() + with contextlib.redirect_stdout(log): + rc = test["main"]() + print(log.getvalue()) + assert rc == 1 and test["FAILS"] == 1 + assert f"[FAIL] {rid}, reference blocks only: {label}" in log.getvalue() + print(f"Control: {rid} WRONG_REFERENCE failed its independent expectation") + marker = test["tr"].MARKERS["SMN1_27134"] + rs, chrom, pos, ref, alt = marker["dbsnp"] + marker["dbsnp"] = rs, chrom, pos + 1, ref, alt + test["FAILS"] = 0 + log = io.StringIO() + with contextlib.redirect_stdout(log): + test["marker_checks"]() + print(log.getvalue()) + assert test["FAILS"] == 1 + assert "[FAIL] marker SMN1_27134: the pinned dbSNP record agrees" in log.getvalue() + print("Control: retained SMN1 off-copy metadata failed its independent expectation") + PY + # The fake-docker job builds no og, so its cn-engine-calls case says it # is skipped there; here bin/og is built and the case must run. - name: The fake-docker case cn-engine-calls (steps 35, 41 and 46 through og cn) with og diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index 8f5426c6c..d7f8985cd 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -119,8 +119,8 @@ jobs: - name: Validate nextflow_schema.json and the samplesheet schema if: steps.scope.outputs.run == 'true' run: | - python3 -m json.tool nextflow_schema.json > /dev/null - python3 -m json.tool assets/schema_input.json > /dev/null + jq -s -e 'length == 1' nextflow_schema.json > /dev/null + jq -s -e 'length == 1' assets/schema_input.json > /dev/null # The stub run below pulls every image of the default tools: the pinned # leg ran out of disk in it twice on 2026-11-05 (#300, job 127077180393; @@ -363,6 +363,8 @@ jobs: # the image versions.env gives it. - name: Every process ran and states the tag of its image from versions.env if: steps.scope.outputs.run == 'true' + env: + OG: ${{ github.workspace }}/rust/target/release/og # og dev versions check-run, built above run: | cat "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml" scripts/ci/gen-containers-config.sh --check-versions "${RUNNER_TEMP}/all-tools/pipeline_info/software_versions.yml" @@ -430,83 +432,10 @@ jobs: nextflow run main.nf -profile test_all -stub -ansi-log false -c "${RUNNER_TEMP}/resources.config" \ --tools vep,manta,cnvpytor,prs --max_cpus 16 --max_memory 64.GB --max_time 48.h \ --outdir "${RUNNER_TEMP}/resources" -work-dir "${RUNNER_TEMP}/resources-work" - python3 - "${RUNNER_TEMP}"/resources/pipeline_info/trace_*.txt "$exec_cpus" "$exec_mem_gb" <<'PY' - import csv, sys - GB, H = 1024 ** 3, 3600 * 1000 - exec_cpus, exec_mem = int(sys.argv[2]), int(sys.argv[3]) * GB - # process: (cpus, memory, time) of the first attempt - want = { - 'DEEPVARIANT': (8, 32 * GB, 48 * H), - 'FASTP': (8, 8 * GB, 8 * H), - 'ALIGN_MINIMAP2': (8, 32 * GB, 24 * H), - 'MANTA': (8, 16 * GB, 24 * H), - 'CNVPYTOR': (4, 8 * GB, 24 * H), - 'VEP': (8, 16 * GB, 24 * H), - 'PRS_PREPARE': (1, 2 * GB, 2 * H), - 'PRS_SCORE_SITES': (2, 4 * GB, 4 * H), - 'PRS': (4, 24 * GB, 8 * H), - } - seen, bad = {p: {} for p in want}, 0 - attempts = all_cpus = all_mem = 0 - rows = {} - with open(sys.argv[1]) as fh: - for row in csv.DictReader(fh, delimiter='\t'): - first_row = rows.setdefault(row['task_id'], row) - if first_row is not row: - # The late copy can be built before the exit status was - # set, so it may read '-' there; nothing else may differ. - exits = {first_row['exit'], row['exit']} - {'-'} - if len(exits) > 1 or any(first_row[k] != row[k] for k in row if k != 'exit'): - print(f"FAIL: task {row['task_id']} has two different rows in the trace: {first_row} and {row}") - bad += 1 - else: - print(f"{row['name']}: attempt {row['attempt']} is written twice in the trace; counted once") - continue - attempts += 1 - all_cpus += int(row['cpus']) - all_mem += int(row['memory']) - proc = row['name'].split(' (')[0].split(':')[-1] - if proc not in want: - continue - got = (int(row['cpus']), int(row['memory']), int(row['time'])) - print(f"{row['name']}: attempt {row['attempt']}, exit {row['exit']}, " - f"cpus {got[0]}, memory {got[1] / GB:g} GB, time {got[2] / H:g} h") - seen[proc].setdefault(int(row['attempt']), {}).setdefault(row['name'], []).append(got) - for proc, (cpus, mem, time) in want.items(): - first, second = seen[proc].get(1, {}), seen[proc].get(2, {}) - unexpected = sorted(set(seen[proc]) - {1, 2}) - if unexpected: - print(f"FAIL {proc}: attempt(s) {unexpected} in the trace, want only a first attempt and one retry") - bad += 1 - if not first: - print(f"FAIL {proc}: no first attempt in the trace") - bad += 1 - continue - for name in sorted(set(first) | set(second)): - n1, n2 = len(first.get(name, [])), len(second.get(name, [])) - if n1 != 1 or n2 != 1: - print(f"FAIL {name}: {n1} first and {n2} second attempts in the trace, want one of each") - bad += 1 - for got in (g for gs in first.values() for g in gs): - if got != (cpus, mem, time): - print(f"FAIL {proc}: first attempt asks for cpus {got[0]}, memory {got[1] / GB:g} GB, " - f"time {got[2] / H:g} h; want {cpus}, {mem / GB:g} GB, {time / H:g} h") - bad += 1 - for got in (g for gs in second.values() for g in gs): - if got[1] != 2 * mem: - print(f"FAIL {proc}: the retry asks for {got[1] / GB:g} GB, want {2 * mem / GB:g} GB") - bad += 1 - # Every attempt takes its request once and gives it back once. If - # the executor has at least the sum, losing every release still - # leaves room for the next task. - print(f"{attempts} task attempts ask for {all_cpus} CPUs and {all_mem / GB:g} GB added together; " - f"the executor is told it has {exec_cpus} CPUs and {exec_mem / GB:g} GB") - if all_cpus > exec_cpus or all_mem > exec_mem: - print("FAIL: executor.cpus or executor.memory of this step is below that sum; raise exec_cpus / exec_mem_gb") - bad += 1 - print(f"{bad} failure(s)") - sys.exit(1 if bad else 0) - PY + traces=("${RUNNER_TEMP}"/resources/pipeline_info/trace_*.txt) + [ "${#traces[@]}" -eq 1 ] && [ -f "${traces[0]}" ] || { echo "FAIL: expected exactly one resource trace"; exit 2; } + rust/target/release/og dev nf-trace requests "${traces[0]}" --want scripts/ci/nf-trace-requests.tsv \ + --executor-cpus "$exec_cpus" --executor-mem-gb "$exec_mem_gb" # The step above fakes the kill with beforeScript, so it cannot see how # ALIGN_MINIMAP2's own pipe reports one. This runs the module's real @@ -574,28 +503,8 @@ jobs: --max_cpus 16 --max_memory 64.GB --outdir "$t/out" -work-dir "$t/work" || rc=$? echo "nextflow exited ${rc}" cat "$t/trace.txt" - python3 - "$t/trace.txt" "$rc" <<'PY' - import csv, sys - GB = 1024 ** 3 - # One row per task_id: the trace can list a finished task twice - # (the retry step above says why), and the first row is the one - # written when the task completed. - # A repeated row must match the first; only its exit may read '-'. - rows = {} - for r in csv.DictReader(open(sys.argv[1]), delimiter='\t'): - first = rows.setdefault(r['task_id'], r) - exits = {first['exit'], r['exit']} - {'-'} - if len(exits) > 1 or any(first[k] != r[k] for k in r if k != 'exit'): - print(f"FAIL: task {r['task_id']} has two different rows in the trace: {first} and {r}") - sys.exit(1) - rows = [r for r in rows.values() if 'ALIGN_MINIMAP2' in r['name']] - got = [(int(r['attempt']), r['exit'], int(r['memory']) // GB) for r in rows] - want = [(1, '137', 32), (2, '0', 64)] - print(f"ALIGN_MINIMAP2 attempts (attempt, exit, GB): {got}; want {want}") - ok = sorted(got) == want and sys.argv[2] == '0' - print('OK' if ok else 'FAIL: a kill inside the pipe did not reach Nextflow as exit 137 and a retry') - sys.exit(0 if ok else 1) - PY + rust/target/release/og dev nf-trace attempts "$t/trace.txt" --name-glob '*ALIGN_MINIMAP2*' \ + --want '1:137:32,2:0:64' --nextflow-exit "$rc" # PRS runs pgsc_calc, a Nextflow pipeline of its own, and gives it the # task's memory as --max_memory, the most one pgsc_calc task may have. A @@ -696,26 +605,10 @@ jobs: --max_cpus 16 --max_memory 64.GB --outdir "$t/out" -work-dir "$t/work" || rc=$? echo "nextflow exited ${rc}" cat "$t/trace.txt" - python3 - "$t/trace.txt" "$rc" <<'PY' - import csv, sys - GB = 1024 ** 3 - # One row per task_id, as in the ALIGN_MINIMAP2 step above. - rows = {} - for r in csv.DictReader(open(sys.argv[1]), delimiter='\t'): - first = rows.setdefault(r['task_id'], r) - exits = {first['exit'], r['exit']} - {'-'} - if len(exits) > 1 or any(first[k] != r[k] for k in r if k != 'exit'): - print(f"FAIL: task {r['task_id']} has two different rows in the trace: {first} and {r}") - sys.exit(1) - rows = list(rows.values()) - want, ok = [(1, '137', 24), (2, '0', 48)], sys.argv[2] == '0' - for sample in ('sigkill', 'pool'): - got = sorted((int(r['attempt']), r['exit'], int(r['memory']) // GB) for r in rows if r['name'] == f'PRS ({sample})') - print(f"PRS ({sample}) attempts (attempt, exit, GB): {got}; want {want}") - ok = ok and got == want - print('OK' if ok else 'FAIL: a memory kill inside pgsc_calc did not reach Nextflow as exit 137 and a retry that passes') - sys.exit(0 if ok else 1) - PY + for sample in sigkill pool; do + rust/target/release/og dev nf-trace attempts "$t/trace.txt" --name-glob "PRS ($sample)" \ + --want '1:137:24,2:0:48' --nextflow-exit "$rc" + done # A Nextflow run that fails, or that its step's timeout cuts, leaves its # log in the checkout as .nextflow.log (older runs of the job as diff --git a/.github/workflows/personal-data.yml b/.github/workflows/personal-data.yml index 1cc1f5315..81ace8050 100644 --- a/.github/workflows/personal-data.yml +++ b/.github/workflows/personal-data.yml @@ -19,11 +19,32 @@ concurrency: jobs: personal-data: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 20 + env: + OG: ${{ github.workspace }}/rust/target/release/og steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count must be caught) + # The scan is og dev scan personal-data (rust/crates/og-dev): the + # toolchain rust/rust-toolchain.toml pins, og built without the BAM + # reader, which the scan never needs. + - name: Install the toolchain rust/rust-toolchain.toml pins + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + key: personal-data + + - name: Build og + working-directory: rust + run: cargo build --release --locked -p og --no-default-features + + - name: Self-test (a planted home path, address, diplotype, rsID genotype and repeat count, and a scan of no text file, must be caught) run: scripts/ci/check-personal-data.sh --self-test - name: No tracked file holds a home path, a private address or a genotype diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..aa6e7dcc7 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -166,6 +166,11 @@ jobs: name: archaic-1kg-sites path: sites + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --archaic and step 43 on HG002 (chromosome ${{ matrix.part }}) id: run continue-on-error: ${{ matrix.part == 'empty' }} @@ -230,47 +235,7 @@ jobs: # same-position rule of archaic_report.py decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz - python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' - import glob, gzip, os, sys - raw, split, data, out = sys.argv[1:] - tables = sorted(glob.glob(os.path.join(data, "chr*.tsv.gz"))) - chroms = {os.path.basename(t)[:-7] for t in tables} # this part's chromosomes only - def clashes(path): - """Positions of this part with two records or more, one an SNV (the VCF is sorted).""" - hit, key, n, snvs = set(), None, 0, 0 - for line in gzip.open(path, "rt"): - if line.startswith("#"): - continue - c = line.split("\t", 6) - if c[0] not in chroms: - continue - k = (c[0], int(c[1])) - if k != key: - key, n, snvs = k, 0, 0 - n += 1 - snvs += len(c[3]) == 1 and len(c[4]) == 1 - if n >= 2 and snvs >= 1: - hit.add(key) - return hit - sets = {"raw": clashes(raw), "split": clashes(split)} - on_table = {k: 0 for k in sets} - for t in tables: - chrom = os.path.basename(t)[:-7] - want = {k: {p for c, p in v if c == chrom} for k, v in sets.items()} - if not any(want.values()): - continue - for line in gzip.open(t, "rt"): - c = line.split("\t", 2) - if c[0] != chrom: - continue - p = int(c[1]) - for k in want: - if p in want[k]: - on_table[k] += 1 - with open(out, "w") as f: - for k in sets: - f.write(f"{k}_positions\t{len(sets[k])}\n{k}_on_table\t{on_table[k]}\n") - PY + bin/og dev evidence clashes "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" cat "$OUT/affected.tsv" - name: The red control failed, as it must @@ -597,6 +562,11 @@ jobs: gzip -dc "${R}/${f}.gz" > "${R}/GRCh38_no_alt_analysis_set.fasta" rm -f "${R}/${f}.gz" + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + - name: setup.sh --ancient-panel and step 44 on HG002, measured env: THREADS: '4' @@ -672,57 +642,9 @@ jobs: mkdir -p "${G}/HG002/vcf" curl -fsSL --retry 5 -o "${EV}/truth.vcf.gz" "${GIAB_HG002}.vcf.gz" curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed" - python3 -I - "${EV}/truth.vcf.gz" "${EV}/truth.bed" "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \ - "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" <<'PY' - import bisect, gzip, sys - truth, bed, ref, sites, out = sys.argv[1:] - fai = {l.split("\t")[0]: [int(x) for x in l.split("\t")[1:5]] for l in open(ref + ".fai")} - fa = open(ref, "rb") - def base(c, p): - n, off, lb, lw = fai[c] - fa.seek(off + ((p - 1) // lb) * lw + (p - 1) % lb) - return fa.read(1).decode().upper() - conf = {} - for l in open(bed): - c, s, e = l.split()[:3] - conf.setdefault(c, []).append((int(s), int(e))) - starts = {c: [s for s, _ in sorted(v)] for c, v in conf.items()} - ends = {c: [e for _, e in sorted(v)] for c, v in conf.items()} - def confident(c, p): - i = bisect.bisect_right(starts.get(c, []), p - 1) - 1 - return i >= 0 and ends[c][i] >= p - covered, head, body = {}, [], [] - for l in gzip.open(truth, "rt"): - if l.startswith("##"): - head.append(l) - continue - if l.startswith("#"): - head.append('##INFO=\n') - head.append('##ALT=\n') - head.append(l) - continue - f = l.split("\t", 5) - p = int(f[1]) - covered.setdefault(f[0], set()).update(range(p, p + len(f[3]))) - body.append(l) - blocks = 0 - with open(sites) as s: - next(s) - for l in s: - _, c, p = l.split("\t")[:3] - p = int(p) - if c not in fai or p in covered.get(c, ()) or not confident(c, p): - continue - b = base(c, p) - if b not in "ACGT": - continue - body.append(f"{c}\t{p}\t.\t{b}\t<*>\t.\t.\tEND={p}\tGT\t0/0\n") - blocks += 1 - with open(out, "w") as o: - o.writelines(head) - o.writelines(body) - print(f"gVCF: {len(body) - blocks} GIAB records, {blocks} reference blocks at panel sites") - PY + /usr/bin/time -v -o "${M}/gvcf.time" bin/og dev evidence hg002-gvcf "${EV}/truth.vcf.gz" "${EV}/truth.bed" \ + "${G}/reference/GRCh38_no_alt_analysis_set.fasta" "${PANEL}/sites_grch38.tsv" "${EV}/hg002.unsorted.vcf" + grep -E 'Maximum resident|Elapsed' "${M}/gvcf.time" docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/ev" -v "${G}:/g" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' bcftools sort -m 2G -T /ev/sort -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /ev/hg002.unsorted.vcf bcftools index -t /g/HG002/vcf/HG002.g.vcf.gz' diff --git a/.github/workflows/renovate-dry-run.yml b/.github/workflows/renovate-dry-run.yml index 5722cb025..155568c19 100644 --- a/.github/workflows/renovate-dry-run.yml +++ b/.github/workflows/renovate-dry-run.yml @@ -6,7 +6,8 @@ name: Renovate dry run # matcher that misses an image, or a versioning that cannot read a tag, shows # up here first. The full debug log is kept as an artifact for 14 days. # -# The checks (each one has a planted fault in the self-test step): +# The checks (og dev renovate-check; each one has a planted fault in its +# self-test, which runs after the checks): # count every *_IMAGE line of versions.env is a detected docker dependency # names no package name carries a ':' or '@' (a digest line once became # a package named '...@sha256') @@ -16,9 +17,10 @@ name: Renovate dry run # one-pr a tool gets the same proposal in every file it is in # pins every non-image pin (Nextflow, actionlint, gitleaks, ShellCheck, # the Renovate CLI, the HLA database, pgsc_calc, the demo's Python -# packages, Aldy, the docs workflow's Python) is detected in every -# place it is pinned, as often as the text has them. A setup-nextflow -# step counts as a pin only when its version: line is a literal +# packages, Aldy, the docs workflow's Python; one row each in +# scripts/ci/renovate-pins.tsv) is detected in every place it is +# pinned, as often as the text has them. A setup-nextflow step +# counts as a pin only when its version: line is a literal # version, quoted or not; a computed one (an expression) is not a pin # lookups no lookup failed and no tag was unreadable; the only skipped # dependencies are the ones a rule in renovate.json disables @@ -31,7 +33,8 @@ on: # Every file a manager in renovate.json reads (the workflows hold the # Nextflow, actionlint, gitleaks, ShellCheck, Renovate and docs Python pins, # the docs the NXF_VER lines and the Aldy pin, scripts/lib/common.sh the HLA - # database, the demo script its Python packages), plus the doc generator. + # database, the demo script its Python packages), plus the doc generator, + # the pins table and the checker. # The lookups take about a minute. pull_request: branches: [main] @@ -42,6 +45,8 @@ on: - 'docs/*.md' - '.github/workflows/**' - 'scripts/ci/gen-versions-doc.sh' + - 'scripts/ci/renovate-pins.tsv' + - 'rust/crates/og-dev/src/renovate.rs' - 'scripts/lib/common.sh' - 'tests/demo/retake-screenshots.sh' push: @@ -91,7 +96,7 @@ jobs: lookup: # Public repo: GitHub-hosted runners only. runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 env: RENOVATE_VERSION: 44.199.2 steps: @@ -129,222 +134,14 @@ jobs: path: ${{ runner.temp }}/renovate.log retention-days: 14 - - name: Write the checker + # The checks are og dev renovate-check (rust/crates/og-dev/src/renovate.rs), + # with the non-image pins in scripts/ci/renovate-pins.tsv. + - name: Build og as scripts/setup.sh --og does if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} run: | - cat > "${RUNNER_TEMP}/check.py" <<'PY' - import collections, copy, glob, json, os, re, sys - - ENV = "versions.env" - TWINS = ("docs/versions.md", "conf/containers.config") - RULE_SKIPS = {"disabled", "package-rules"} - # The non-image pins: (dependency, files, how a pin of it looks in that text). - # A setup-nextflow step is a pin when a version: line with a literal - # version follows its uses: line inside the same step (the lines in - # between are blank or start with anything but '-', which opens the - # next step). - SETUP_NEXTFLOW = r'uses: ["\x27]?nf-core/setup-nextflow@[^\n]*\n(?:[ \t]*(?:[^\s-][^\n]*)?\n)*?[ \t]+version: ["\x27]?\d' - DEMO = "tests/demo/retake-screenshots.sh" - PINS = ( - ("nextflow-io/nextflow", [ENV], r'^NEXTFLOW_VERSION="'), - ("nextflow-io/nextflow", sorted(glob.glob(".github/workflows/*.yml")), SETUP_NEXTFLOW), - ("nextflow-io/nextflow", sorted(glob.glob("docs/*.md")), r"NXF_VER=\d"), - ("rhysd/actionlint", sorted(glob.glob(".github/workflows/*.yml")), r"ACTIONLINT_VERSION=\d"), - ("gitleaks/gitleaks", sorted(glob.glob(".github/workflows/*.yml")), r"GITLEAKS_VERSION=\d"), - ("koalaman/shellcheck", [".github/workflows/lint.yml"], r"^[ \t]+version: v\d"), - ("renovate", [".github/workflows/renovate-dry-run.yml"], r"RENOVATE_VERSION: \d"), - ("renovate", ["docs/updating.md"], r"renovate@\d"), - ("python", [".github/workflows/docs.yml"], r"python-version: ['\"]?\d"), - ("ANHIG/IMGTHLA", ["scripts/lib/common.sh"], r"^HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-\d"), - ("PGScatalog/pgsc_calc", [ENV], r'^PGSC_CALC_VERSION="'), - ("PGScatalog/pgsc_calc", ["conf/containers.config"], r"ext\.pipeline_version = '"), - ("playwright", [DEMO], r"playwright==\d"), - ("pillow", [DEMO], r"pillow==\d"), - ("pyobjc-framework-Vision", [DEMO], r"pyobjc-framework-Vision==\d"), - ("aldy", ["docs/21-cyrius.md"], r"aldy==\d|\[Aldy\]\([^)]*\) v\d"), - ) - # Only the files a pin is listed for are counted: python is also the - # docker image of versions.env, which the image checks cover. - PIN_FILES = collections.defaultdict(set) - for _name, _files, _ in PINS: - PIN_FILES[_name].update(_files) - - def pin_counts(): - want = collections.Counter() - for name, files, pattern in PINS: - for f in files: - n = len(re.findall(pattern, open(f).read(), re.M)) - if n: - want[(name, f)] += n - return want - - def load(log): - files = None - for line in open(log): - try: - rec = json.loads(line) - except ValueError: - continue - if rec.get("msg") == "packageFiles with updates": - files = rec["config"] - if files is None: - sys.exit("FAIL: no 'packageFiles with updates' record in %s" % log) - return [(pf["packageFile"], dep) for entries in files.values() for pf in entries for dep in pf.get("deps", [])] - - def env_images(text): - out = [] - for m in re.finditer(r'^[A-Z0-9_]+_IMAGE="([^"]*)"', text, re.M): - # name:tag@sha256:..., name@sha256:... or name:tag -> name - out.append(re.sub(r":[^:/]*$", "", m.group(1).split("@", 1)[0])) - return out - - def key(dep): - return (dep.get("depName"), dep.get("currentValue"), dep.get("currentDigest")) - - def proposals(dep): - return sorted((u.get("newValue") or "", u.get("newDigest") or "") for u in dep.get("updates", [])) - - def check(deps, env_text): - fails = [] - docker = [(f, d) for f, d in deps if d.get("datasource") == "docker"] - by_file = collections.defaultdict(list) - for f, d in docker: - by_file[f].append(d) - want = env_images(env_text) - got = [d.get("depName") for d in by_file[ENV]] - if len(got) != len(want): - fails.append("count: versions.env has %d *_IMAGE lines, Renovate found %d docker dependencies there" % (len(want), len(got))) - for name in sorted(set(want) - set(got)): - fails.append("count: %s (versions.env) is not a detected dependency" % name) - for f, d in docker: - if re.search(r"[:@]", d.get("depName") or ":"): - fails.append("names: %s: package name %r" % (f, d.get("depName"))) - env_keys = collections.Counter(key(d) for d in by_file[ENV]) - for f in TWINS: - keys = collections.Counter(key(d) for d in by_file[f]) - if f == "docs/versions.md" and keys != env_keys: - for k in sorted((keys - env_keys) | (env_keys - keys), key=str): - fails.append("twins: docs/versions.md and versions.env differ on %s" % (k,)) - else: - for k in sorted(set(keys) - set(env_keys), key=str): - fails.append("twins: %s has %s, which versions.env does not" % (f, k)) - seen = {} - for f, d in deps: - name = (d.get("datasource"), d.get("depName"), d.get("currentValue"), d.get("currentDigest")) - p = proposals(d) - if name in seen and seen[name][1] != p: - fails.append("one-pr: %s gets %s in %s but %s in %s" % (d.get("depName"), seen[name][1], seen[name][0], p, f)) - seen.setdefault(name, (f, p)) - for f, d in deps: - if d.get("warnings"): - fails.append("lookups: %s (%s): %s" % (d.get("depName"), f, "; ".join(w.get("message", str(w)) for w in d["warnings"]))) - if d.get("skipReason") and d["skipReason"] not in RULE_SKIPS: - fails.append("lookups: %s (%s) skipped: %s" % (d.get("depName"), f, d["skipReason"])) - got = collections.Counter((d.get("depName"), f) for f, d in deps - if f in PIN_FILES.get(d.get("depName"), ())) - for k in sorted(set(want_pins) | set(got)): - if want_pins[k] != got[k]: - fails.append("pins: %s in %s: the text pins it %d times, Renovate found %d" % (k[0], k[1], want_pins[k], got[k])) - return fails - - def self_test(deps, env_text): - ok = True - base = set(check(deps, env_text)) - def plant(label, fn, pattern): - nonlocal ok - d = copy.deepcopy(deps) - try: - fn(d) - except StopIteration: - print("self-test %-8s NOT PLANTED: no dependency to plant it on" % label) - ok = False - return - # Only a failure the plant added counts as caught. - hits = [x for x in check(d, env_text) if x.startswith(pattern) and x not in base] - print("self-test %-8s %s" % (label, "caught: " + hits[0] if hits else "NOT CAUGHT")) - ok = ok and bool(hits) - def first(d, f): - return next(x for g, x in d if g == f and x.get("datasource") == "docker") - def drop(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("datasource") == "docker")) - def digest_name(d): - first(d, ENV)["depName"] += "@sha256" - def stale_twin(d): - first(d, "docs/versions.md")["currentValue"] = "0.0.0-planted" - def split_pr(d): - first(d, "conf/containers.config")["updates"] = [{"newValue": "9.9.9-planted"}] - def drop_pin(d): - d.remove(next((g, x) for g, x in d if g == ENV and x.get("depName") == "nextflow-io/nextflow")) - def broken_lookup(d): - first(d, ENV)["skipReason"] = "invalid-value" - plant("count", drop, "count:") - plant("names", digest_name, "names:") - plant("twins", stale_twin, "twins:") - plant("one-pr", split_pr, "one-pr:") - plant("pins", drop_pin, "pins:") - plant("lookups", broken_lookup, "lookups:") - # The setup-nextflow pattern itself, on planted workflow text. - step = (' - uses: %snf-core/setup-nextflow@%s%s # v3.0.1\n' - '%s' - ' if: steps.scope.outputs.run == \'true\'\n' - ' with:\n' - ' version: %s\n' - '\n' - ' - uses: actions/setup-java@%s # v6\n' - ' with:\n' - ' java-version: "17"\n') - for label, quote, gap, version, want in ( - ("computed", "", "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed, uses quoted", '"', "", "$" + "{{ steps.nf.outputs.version }}", 0), - ("computed after a blank line", "", "\n", "$" + "{{ steps.nf.outputs.version }}", 0), - ("literal", "", "", '"25.10.8"', 1), - ("literal after a blank line", "", "\n", '"25.10.8"', 1), - ("single-quoted literal", "'", "", "'25.10.8'", 1), - ("bare literal, uses quoted", '"', "", "25.10.8", 1)): - text = step % (quote, "a" * 40, quote, gap, version, "b" * 40) - n = len(re.findall(SETUP_NEXTFLOW, text, re.M)) - print("self-test pin-text %-27s counted %d, want %d" % (label, n, want)) - ok = ok and n == want - clean = check(deps, env_text) - print("self-test clean %s" % ("passes" if not clean else "fails: %s" % clean[0])) - return ok - - def summary(deps): - rows, seen = [], set() - files = collections.defaultdict(set) - for f, d in deps: - files[(d.get("datasource"), d.get("depName"))].add(f) - for f, d in deps: - k = (d.get("datasource"), d.get("depName")) - if k in seen: - continue - seen.add(k) - cur = d.get("currentValue") or d.get("currentDigest") or "" - ups = ", ".join("%s (%s)" % (u.get("newValue") or u.get("newDigest"), u.get("updateType")) - for u in d.get("updates", [])) or "none" - note = d.get("skipReason") or "" - rows.append("| `%s` | `%s` | %s | %s | %d |" % (d.get("depName"), cur, ups, note, len(files[k]))) - return "\n".join(["| Dependency | Current | Proposed update | Skipped | Files |", "|---|---|---|---|---|"] + rows) - - deps = load(sys.argv[1]) - env_text = open(ENV).read() - want_pins = pin_counts() - table = summary(deps) - print(table) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: - out.write("## Renovate dry run\n\n" + table + "\n") - fails = check(deps, env_text) - for x in fails: - print("FAIL " + x) - if not self_test(deps, env_text): - fails.append("the self-test did not catch a planted fault") - print("FAIL the self-test did not catch a planted fault") - if fails: - sys.exit(1) - print("OK: %d dependencies; every check passed" % len(deps)) - PY + scripts/setup.sh --og + rust/og.sh check --required - name: Check what Renovate found if: ${{ !cancelled() && steps.lookup.outcome == 'success' }} - run: python3 "${RUNNER_TEMP}/check.py" "${RUNNER_TEMP}/renovate.log" + run: bin/og dev renovate-check "${RUNNER_TEMP}/renovate.log" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3175932c1..8f9f7ea7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,49 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Ordinary Russian, Ukrainian and Serbian text scans again. BMP regex case differences now refuse patterns that name a differing fold class under active case-insensitive flags. This includes negated classes and range interiors. Stable case-sensitive Cyrillic names work. The Unicode property guard and the astral casing guard still apply to text, filenames and exception rows. Indexed symlink target names receive the full filename guard before SKIP, including directory components. Both Unicode verifiers fail when an explicitly requested Python cannot run. CI requires Python 3.9, 3.12 and 3.15. A custom pattern refused by the case-difference rule now says so, with the file and line, and says what to change: turn the `i` flag off for that part and list the exact spellings. + +- The scanner's Unicode checks install Python 3.9, 3.12 and 3.15 through pinned uv tooling, so the older oracle remains available on newer `ubuntu-latest` images. + +- The personal-data scan refuses every astral cased character from UCD 13/15/16/17, including case-mapping targets. Text in Deseret, Osage, Old Hungarian, Warang Citi, Medefaidrin, Adlam and other astral cased scripts needs an ASCII SKIP glob. Uncased astral text still scans. Custom class ranges with a guarded interior return redacted exit 2. BMP verification now covers composite classes, ranges and alternatives on every available Python. Every working-copy symlink target gets the same redacted Unicode filename guard before following it. + +- The personal-data guard now includes effective Python regex case relations, including Python 3.9 Cyrillic differences. Six stable older scalars pass again. Selected filenames get the same Unicode guard before exclusions, with a generic redacted error. Cached gitlinks and unreadable indexed objects fail with exit 2 unless skipped. Working-copy symlinks validate every hop, including directory components, with a 40-hop bound. The 248 newer stable scalars remain conservatively blocked. + +- The personal-data scan fails closed with redacted exit 2 on Unicode-version-dependent text, patterns and ALLOW/SKIP entries. Its generated guard covers UCD 13/15/16/17 and unassigned scalars. Newer emoji and plain symbols still scan. Files with newer script letters need an ASCII SKIP glob. Invalid UTF-8 symlink targets now fail before exclusions in working-copy and index scans. + +- The personal-data scan, the image scan and the version checks of CI now run in the Rust `og` binary: `og dev scan personal-data` (`scripts/ci/check-personal-data.sh`), `og dev scan images` and `og dev versions coupled` (`scripts/ci/check-images.sh`), `og dev versions check-run` (`scripts/ci/gen-containers-config.sh --check-versions`), `og dev versions expand` (`scripts/ci/check-links.sh`) and `og dev pgs` (`scripts/ci/check-pgs-labels.sh`). The scripts keep their options and lists; the Python inside them is gone. Fixes: + - A personal-data scan that read no text file (every file skipped or binary) fails with `FAIL: no text file was scanned`. It printed `scanned 0 text files` and passed. + - A personal-data scan whose root is reached through a symbolic link places each file under that root, so the lists of skipped files and documented examples still apply to it. + - An image exemption in `check-images.sh` whose file no longer names that image fails with `FAIL: no longer names ; drop its line from EXEMPT_PAIRS`. A stale exemption was never reported. + - `check-links.sh` and `check-images.sh` read `versions.env` strictly: a line that is not a plain `NAME="value"` (with no `"`, `$` or backquote in the value), or a name set twice, fails the check and names the line. Before, such a line was skipped, so the URLs built from it went unchecked. + - The PGS evaluation check never reads a JSON `true` as the number 1, so `"estimate": true` no longer matches an effect of `OR 1`. + - Personal-data findings print only the kind, escaped filename and line number. Matched values and custom-pattern contents never appear in diagnostics. The Python scanner printed the detected line. + - Built-in word boundaries and sample-label case folding follow Python, including combining marks, dotted and dotless I, long s and the Kelvin sign. + - The staged scan uses NUL-delimited Git requests and responses, so a filename containing a newline cannot hide a finding. + - Built-in and custom personal-data patterns share a Python-subset compiler and a backtracking matcher. Word boundaries stay inside the search, so a chip row ending in `A-` can backtrack to its shorter match. Unicode word, space and case rules also apply to custom patterns. + - Empty custom matches keep Python's iteration order, including a nonempty alternative at the same position. Capture groups keep the full match when checking allowed examples. + - Pattern files use universal newlines. ALLOW and SKIP rows use Python's line separators, so a CR or Unicode separator cannot extend an exemption into another file. + - Custom patterns cannot quantify an operand that can match empty text. Such patterns stop with redacted exit 2 because their match spans can differ from Python and hide a finding under a partial exemption. + - A filename with invalid UTF-8 stops the personal-data scan with redacted exit 2 in working-copy and index modes. It cannot be changed into another name or skipped. + - Unsupported custom constructs or matcher resource limits stop the scan with a redacted error and exit 2. The supported subset is documented in `scripts/ci/check-personal-data.sh`. + +- CI evidence rejects FASTA offset overflow and offsets outside the file before reading or publishing. Guard tests this in both debug and release builds. Every new dev verb fails on stdout or stderr write or flush errors. A truth VCF header after the first data record fails without replacing the destination. Additional deliberate refusals are documented and tested: zero-byte gzip inputs, zero-width unused FAI rows, non-ASCII or out-of-range annotation limits, non-writable staging parents, and gzip FASTA files with long FNAME headers. Publication replaces a file and leaves other hard links on the old inode. + +- CI evidence and Renovate inputs now enforce their original explicit gzip or plain format above strict UTF-8 and universal-newline handling. Renovate normalizes repository sources too and accepts falsy warnings and packages with no dependencies. Output publication follows destination symlinks, preserves existing permissions, rejects read-only files and directories, and uses the umask for new files. VNtyper uses private temporary directories with failure and signal cleanup, and GNU `mv -T` prevents publication inside a directory. Comphet stages and validates all destinations before publishing. Three renames are still not one transaction. Stricter parsing is deliberate and tested: BED refuses unsupported Unicode/control whitespace, Renovate refuses wrong-typed present values, PharmCAT refuses truthy non-list containers and non-object alleles, evidence positions must be ASCII u64 values, truth VCF rows need five columns, blank FAI rows fail, text records have a 16 MiB limit, and clash rows need a trailing VCF column after ALT. Truth VCF records get a final LF. `docs/rust.md` lists the error classes. + +- `og dev evidence` now rejects invalid UTF-8, truncated selected records, blank BED rows and missing panel headers. It reads LF, CRLF and CR line endings alike, with a 16 MiB limit per record. Single-file evidence and fixture outputs are published by rename after validation. Comphet stages and validates all three outputs before its first rename. Three renames are still not one transaction. A crash or a filesystem failure between renames can publish only part of the new set. `og dev renovate-check` rejects malformed log records and present fields of the wrong type, and fails if its configured step summary cannot be written. The PharmCAT trimmer consumes source genes as it trims them. The VNtyper downloader also accepts output paths that contain a newline. + + +- The Python that CI and the fixtures ran outside the pipeline now runs in the Rust `og` binary. The public evidence workflow runs `og dev evidence clashes` and `og dev evidence hg002-gvcf` instead of two inline Python programs; the Renovate dry run checks what Renovate found with `og dev renovate-check`, whose list of non-image pins is now data in `scripts/ci/renovate-pins.tsv`; `og dev fixture comphet` writes the compound-het fixture of the e2e case `comphet-phasing-1-reads`, byte for byte; and `og dev fixture trim-pharmcat` trims a PharmCAT report for `tests/fixtures/pharmcat`. `tests/fixtures/comphet/make_fixture.py`, `tests/fixtures/pharmcat/trim_report.py` and `tests/smoke/vntyper_testdata.py` are gone: VNtyper's two test BAMs now come from `tests/smoke/vntyper_testdata.sh`, one HTTP range request per BAM with curl, checked by gzip against the archive's CRC-32 and size and then against the pinned sha256. On GIAB's HG002 benchmark VCF (4,048,342 records) with the chr20-22 reference, `og dev evidence hg002-gvcf` writes the same gVCF in 3.5 s and 97 MB instead of 11.8 s and 3.8 GB, because it keeps the spans the records cover instead of every covered position. No pipeline output changes. What changes on purpose: + - `og dev evidence hg002-gvcf` skips a panel site past the end of its contig. The Python read the byte after the contig, and at the end of the file wrote a reference block with an empty REF. + - `og dev renovate-check` stops with exit 2 when a pin's file is missing or its glob matches no file. The Python counted such a pin in no file, so it could never fail. + - `og dev fixture trim-pharmcat` names itself in the trimmed report's `_fixture_note`. + - Renovate skips only malformed JSON lines. JSON reader limits (signed 128-bit integers, Unicode scalar strings and 1000 nesting levels) now fail before an earlier config can pass the check. Evidence, fixture and Renovate refusals preserve existing output bytes, modes and symlink targets. [The refusal rule](docs/rust.md#input-and-destination-refusal-rule) states the numeric and size limits, with a shared table of negative CLI controls. +- The shared `og-io` line reader bounds memory by the longest record for CR-only files as well as LF and CRLF files. Smoke coordinates require signed integers, so decimal and nonfinite coordinates fail. Trace integer cells accept whitespace, signs and underscores but deliberately refuse Unicode digits and negative resources. Integers outside the target range fail. Duplicate task rows retain concrete exits. Smoke JSON checks require one document, numeric counts and copy numbers, array catalogs and string genotypes. Numeric depth strings stay supported; booleans are refused. The array and genotype rules deliberately reject malformed values Python accepted. Smoke fixtures build a missing default `og` on first use. Setup output goes to stderr so captured PharmCAT counts and CSV values stay clean. +- CI readers now use `og dev pharmcat diplotypes`, `nf-trace` and `smoke`. Unreadable PharmCAT reports fail with exit 2 and are marked not assessed. Resource checks refuse trace globs and malformed numeric cells. The smoke mini-reference uses a fixed Rust seed for the planted bases. Gzip smoke reads have deterministic headers. Smoke table checks report concise property results. +- Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. +- The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. + - The Rust summary's CHIP reader marks a depth table with NaN unreadable instead of sorting it into a usable sensitivity estimate. Its unindexed APOL1 G2 reader keeps only records overlapping the deletion span and stops past that span on sorted input. - The Rust `og` binary now reads the rest of the summary's own sections: `og summary collect --only c`. It reads 32 sections of summary.json (couple, parascopy, paralogs, hla, prs, ancestry_fine, manta, delly, cnvpytor, sv_consensus, annotsv, expansions and their readings, telomere, roh, haplogroup, y_haplogroup, y_tree, y_str, consumer_export, bloodtype, lpa, vntyper, locityper, mito, traits with the APOL1 G2 call, archaic, ancient, viral, exomiser, chip, mei and mosaic). Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 15 new parity cases, compared as JSON with key order. The readers run on a thread pool, one per section. On a synthetic sample folder with every section and a 1,000,000-record VCF it takes 0.33 s and 34 MB, where the Python readers took 0.33 s and 41 to 59 MB. What differs from the Python: diff --git a/bin/copy_number.py b/bin/copy_number.py deleted file mode 100755 index 9412104b9..000000000 --- a/bin/copy_number.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""copy_number.py: the catalog reader and the depth-ratio engine of -bin/cn_loci.tsv, as the catalog tests (tests/test_copy_number.py, -test_hba_copy_number.py, test_lpa_kiv2.py) import them. No step runs it. - -The commands are og cn's (rust/crates/og-cn, docs/cn-loci.md): `og cn bed` -prints a locus's segments, `og cn call` turns `samtools bedcov` over them -into the step's table, `og cn intervals` reads the interval rows and `og cn -check-bam` refuses a BAM aligned with ALT contigs, and `og bloodtype -report` reads step 41's RHD row with og cn's engine. This file keeps -load_catalog, assess, cut_copies, BrokenControl and CatalogError for those -tests until they move to og too. - -One method, one catalog row per locus: the depth over the locus's target -segments against the depth over single-copy control segments, as -`samtools bedcov` sums it. With T the target depth and C the control depth -(each combined as the row's target_depth and control_depth say), - estimate = scale x T / C - offset -then guarded by a minimum control depth, flank agreement and the share of the -target depth from reads of mapping quality 0, and called as the row's `call` -column says. docs/cn-loci.md describes the columns and how to add a locus. -Standard library only. -""" -import csv -import math -import os - -CATALOG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cn_loci.tsv") -COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", - "scale", "offset", "bedcov", "mapq0_min", "mapq0_max", "min_control_depth", "max_control_ratio", - "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] -KINDS = ("gene", "interval", "organelle") -# GRCh38 names of the primary chromosomes (the ClinGen BED spells chrX 'chrx': -# og dev build cn-intervals normalises it, and a row with any other -# spelling is refused here). -CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} -INPUTS = ("bedcov", "cnvpytor-level") -Z95 = 1.96 -# The findings a psv row needs a reading for. -PSV_FINDINGS = ("none", "deletion", "conversion", "pseudogene_deletion") -BASES = "ACGT" - - -class CatalogError(ValueError): - pass - - -class BrokenControl(ValueError): - """A control segment with no read while the other controls are covered.""" - - -def _num(row, col, default=None): - v = row[col] - if v == ".": - return default - try: - x = float(v) - except ValueError: - x = math.nan - if not math.isfinite(x): - raise CatalogError(f"{row['locus']}: {col} is {v!r}, not a number or '.'") - return x - - -def _segments(row, col): - """[(name, start, end, label)] of a targets, controls or details cell.""" - if row[col] == ".": - return [] - out = [] - for part in row[col].split(";"): - bits = part.split(":", 2) - try: - name, (s, e) = bits[0], bits[1].split("-") - start, end = int(s), int(e) - except (IndexError, ValueError): - raise CatalogError(f"{row['locus']}: {col} segment {part!r} is not NAME:start-end[:label]") - if not name or end <= start or start < 0: - raise CatalogError(f"{row['locus']}: {col} segment {part!r} has no name or an empty span") - out.append((name, start, end, bits[2] if len(bits) > 2 else "")) - return out - - -def _readings(row): - out = {} - if row["readings"] == ".": - return out - for part in row["readings"].split("|"): - k, sep, text = part.partition("=") - if not sep or not k or not text: - raise CatalogError(f"{row['locus']}: reading {part!r} is not KEY=text") - out[k] = text - return out - - -def _call(row): - mode, sep, args = row["call"].partition(":") - try: - vals = [float(x) for x in args.split(",")] - except ValueError: - vals = [] - if not all(math.isfinite(v) for v in vals): - vals = [] - if not sep or mode not in ("round", "cut", "class", "psv") or not vals \ - or (mode in ("round", "psv") and len(vals) != 1) or (mode == "class" and len(vals) != 2) \ - or vals != sorted(vals): - raise CatalogError(f"{row['locus']}: call {row['call']!r} is not round:D, cut:A,B..., class:LOW,HIGH or psv:D") - return mode, vals - - -def _psv_sites(row, details): - """[(name, gene 0-based position, pseudogene 0-based position, gene - bases, pseudogene bases, text)] of a psv row: every details segment is one - base, labelled 'PSEUDO_POS GENE>PSEUDO text' (bases among ACGT; In2G's - gene base is A or C).""" - out = [] - for name, s, e, label in details: - bits = label.split(" ", 2) - try: - pos = int(bits[0]) - gene, _, pseudo = bits[1].partition(">") - except (IndexError, ValueError): - raise CatalogError(f"{row['locus']}: PSV {name} label {label!r} is not 'PSEUDO_POS GENE>PSEUDO text'") - if e - s != 1 or pos < 0 or not gene or not pseudo or set(gene + pseudo) - set(BASES) \ - or set(gene) & set(pseudo) or len(bits) < 3: - raise CatalogError(f"{row['locus']}: PSV {name} must be one base with distinct gene and pseudogene " - f"bases and a text (got {s}-{e}, {label!r})") - out.append((name, s, pos, set(gene), set(pseudo), bits[2])) - return out - - -def parse_row(row): - """The catalog row checked, with its parsed fields under '_' keys.""" - loc = row["locus"] - if row["kind"] not in KINDS: - raise CatalogError(f"{loc}: kind {row['kind']!r} (one of {', '.join(KINDS)})") - if row["input"] not in INPUTS: - raise CatalogError(f"{loc}: input {row['input']!r} (one of {', '.join(INPUTS)})") - if row["chrom"] not in CHROMS: - raise CatalogError(f"{loc}: chrom {row['chrom']!r} (GRCh38 names, chr1 to chrM)") - p = dict(row) - p["_targets"], p["_controls"], p["_details"] = (_segments(row, c) for c in ("targets", "controls", "details")) - p["_call"] = _call(row) - p["_readings"] = _readings(row) - p["_psv"] = _psv_sites(row, p["_details"]) if p["_call"][0] == "psv" else [] - if p["_call"][0] == "psv" and (not p["_psv"] or row["target_depth"] != "sum" - or not set(PSV_FINDINGS) <= set(p["_readings"])): - raise CatalogError(f"{loc}: call psv needs PSV sites in details, target_depth sum (gene and pseudogene) " - f"and a reading for each of {', '.join(PSV_FINDINGS)}") - if not p["_targets"]: - raise CatalogError(f"{loc}: no target segment") - if row["input"] == "bedcov": - if not p["_controls"]: - raise CatalogError(f"{loc}: a bedcov row needs control segments") - if row["target_depth"] not in ("sum", "pooled") or row["control_depth"] not in ("flanks", "pooled"): - raise CatalogError(f"{loc}: target_depth sum|pooled and control_depth flanks|pooled") - p["_scale"], p["_offset"] = _num(row, "scale"), _num(row, "offset", 0.0) - if not p["_scale"] or p["_scale"] <= 0: - raise CatalogError(f"{loc}: scale must be a positive number") - p["_min_control"] = _num(row, "min_control_depth", 0.0) - p["_max_ratio"] = _num(row, "max_control_ratio") - p["_mapq0_min"], p["_mapq0_max"] = _num(row, "mapq0_min"), _num(row, "mapq0_max") - p["_passes"] = row["bedcov"].split(",") - if (p["_mapq0_min"] is not None or p["_mapq0_max"] is not None) and len(p["_passes"]) != 2: - raise CatalogError(f"{loc}: a MAPQ 0 guard needs two bedcov passes (all reads, MAPQ >= 1)") - if p["_call"][0] == "class" and row["interval"] != "cv": - raise CatalogError(f"{loc}: call class needs interval cv") - if row["interval"] not in ("cv", "."): - raise CatalogError(f"{loc}: interval {row['interval']!r} (cv or '.')") - names = [n for n, *_ in p["_targets"] + p["_controls"] + p["_details"]] - if len(names) != len(set(names)): - raise CatalogError(f"{loc}: a segment name is used twice") - return p - - -def load_catalog(path=None): - """{locus: parsed row} of the catalog, in file order.""" - path = path or CATALOG - with open(path) as f: - lines = [ln for ln in f if ln.strip() and not ln.startswith("#")] - rows = list(csv.DictReader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)) - if not rows or list(rows[0].keys()) != COLUMNS: - raise CatalogError(f"{path}: the header must be {' '.join(COLUMNS)}") - out, seen = {}, set() - for r in rows: - if None in r or None in r.values(): - raise CatalogError(f"{path}: row {r.get('locus')!r} has the wrong number of columns") - if r["locus"] in out: - raise CatalogError(f"{path}: locus {r['locus']} is listed twice") - p = parse_row(r) - for n, *_ in p["_targets"] + p["_controls"] + p["_details"]: - if n in seen: - raise CatalogError(f"{path}: segment name {n} is used by two loci") - seen.add(n) - out[r["locus"]] = p - return out - - -def regions(row): - """[(chrom, start, end, name)] of a row's segments, sorted by position.""" - segs = row["_controls"] + row["_targets"] + row["_details"] - return sorted(((row["chrom"], s, e, n) for n, s, e, _ in segs), key=lambda r: (r[1], r[2])) - - -def _combine(mode, segs, m): - if mode == "sum": - return sum(m[n] for n, *_ in segs) - if mode == "flanks": - return sum(m[n] for n, *_ in segs) / len(segs) - return sum(m[n] * (e - s) for n, s, e, _ in segs) / sum(e - s for _, s, e, _ in segs) - - -def _cv(values): - """Coefficient of variation (sample standard deviation over the mean).""" - mu = sum(values) / len(values) - if mu <= 0 or len(values) < 2: - return 0.0 - return math.sqrt(sum((v - mu) ** 2 for v in values) / (len(values) - 1)) / mu - - -def cut_copies(value, cuts): - """The number of cut points a value reaches (cut:0.5,1.5 gives 0, 1 or 2).""" - return sum(value >= c for c in cuts) - - -def reading(row, key): - r = row["_readings"] - if str(key) in r: - return r[str(key)] - if isinstance(key, int) and "more" in r and key > max(int(k) for k in r if k.isdigit()): - return r["more"].format(n=key) - return "" - - -def ploidy(row, sex): - """The copies of the row's chromosome a sample has: 2, or 1 on a male's - chrX or chrY; None where it cannot be said (a chrY row on a female or on - a sample of no known sex, a chrX row without a sex).""" - if row["chrom"] not in ("chrX", "chrY"): - return 2 - if sex == "male": - return 1 - if sex == "female" and row["chrom"] == "chrX": - return 2 - return None - - -def assess(row, m, q1=None, sex=None): - """The locus's block for mean depths m (every read) and q1 (reads of - MAPQ >= 1, or None for a row without a MAPQ 0 guard; mapq0_share is then - None). - - status is 'called' or 'not assessed'; reason names the guard that - stopped it: no_data, low_control_depth, controls_disagree, - low_mapq0_share, high_mapq0_share, below_floor, between_counts, - sex_unknown; status 'not applicable' (reason no_y) for a chrY row on a - female. sex matters only for a row on chrX or chrY: there the controls - sit on the same chromosome, so the scale follows the sample's count - there (ploidy). expected is the count with no change.""" - guarded = q1 is not None - if not guarded and (row["_mapq0_min"] is not None or row["_mapq0_max"] is not None): - raise ValueError(f"{row['locus']} has a MAPQ 0 guard: the MAPQ >= 1 depths are needed") - q1 = q1 if guarded else {} - tg, ct = row["_targets"], row["_controls"] - b = {"locus": row["locus"], "status": "not assessed", "reason": None, "estimate": None, "total": None, - "copies": None, "ci95": None, "class": None, "target_depth": None, "control_depth": None, - "depths": dict(m), "mapq0_share": None, "details": {}, "reading": "", "expected": None} - # expected: the copies with no change, the ploidy times the reference - # units (each target segment is one with target_depth sum: AZFc's 4 DAZ) - units = len(row["_targets"]) if row.get("target_depth") == "sum" else 1 - pl = ploidy(row, sex) - b["expected"] = None if pl is None else pl * units - if row["chrom"] == "chrY" and sex == "female": - b.update(status="not applicable", reason="no_y") - return b - if pl is None: - b["reason"] = "sex_unknown" - return b - if not m or any(n not in m for n, *_ in tg + ct): - b["reason"] = "no_data" - return b - for n, *_ in ct: - rest = [c for c in ct if c[0] != n] - if m[n] == 0 and rest and _combine(row["control_depth"], rest, m) >= row["_min_control"] > 0: - raise BrokenControl(f"BROKEN_CONTROL: control segment {n} of {row['locus']} has no read while the " - f"other controls have {_combine(row['control_depth'], rest, m):.1f}x; check its " - "coordinates in bin/cn_loci.tsv (or the sample lacks that stretch)") - T, C = _combine(row["target_depth"], tg, m), _combine(row["control_depth"], ct, m) - tq = _combine(row["target_depth"], tg, {n: q1.get(n, 0.0) for n, *_ in tg}) - # The MAPQ 0 share is known only when the MAPQ >= 1 pass was given. - share = (1 - tq / T if T > 0 else 0.0) if guarded else None - b.update(target_depth=T, control_depth=C, mapq0_share=share) - low = min(m[n] for n, *_ in ct) if row["control_depth"] == "flanks" else C - if low < row["_min_control"] or C <= 0: - b["reason"] = "low_control_depth" - return b - if row["control_depth"] == "flanks" and row["_max_ratio"] is not None and len(ct) > 1: - vals = [m[n] for n, *_ in ct] - if min(vals) <= 0 or max(vals) / min(vals) > row["_max_ratio"]: - b["reason"] = "controls_disagree" - return b - total = row["_scale"] * pl / 2 * T / C - est = total - row["_offset"] - b.update(total=total, estimate=est, - details={n: row["_scale"] * pl / 2 * m[n] / C for n, *_ in row["_details"] if n in m}) - # With no copy left there is no read to judge the aligner by (a row with - # an offset has a floor of its own below, after this guard). - no_copy = row["_offset"] == 0 and round(est) == 0 - if row["_mapq0_min"] is not None and not no_copy and share < row["_mapq0_min"]: - b["reason"] = "low_mapq0_share" - return b - if row["_mapq0_max"] is not None and share > row["_mapq0_max"]: - b["reason"] = "high_mapq0_share" - return b - if row["_offset"] > 0 and est <= 0: - b["reason"] = "below_floor" - return b - mode, vals = row["_call"] - if row["interval"] == "cv": - rel = math.sqrt(_cv([m[n] for n, *_ in tg]) ** 2 / len(tg) + _cv([m[n] for n, *_ in ct]) ** 2 / len(ct)) - b["ci95"] = (max(0.0, total * math.exp(-Z95 * rel) - row["_offset"]), - total * math.exp(Z95 * rel) - row["_offset"]) - if mode == "psv": - # the module total (gene plus pseudogene copies); og cn reads the sites - n = round(est) - if abs(est - n) > vals[0]: - b["reason"] = "between_counts" - return b - b["module_total"] = n - elif mode == "round": - n = round(est) - if abs(est - n) > vals[0]: - b["reason"] = "between_counts" - return b - b["copies"] = n - b["reading"] = reading(row, n) - elif mode == "cut": - b["copies"] = cut_copies(est, vals) - b["reading"] = reading(row, b["copies"]) - else: - lo, hi = b["ci95"] - b["class"] = "low" if hi < vals[0] else "high" if lo > vals[1] else "typical" - b["reading"] = reading(row, b["class"]) - b["status"] = "called" - return b diff --git a/bin/traits_report.py b/bin/traits_report.py index 6b984f1cb..0d8e67310 100755 --- a/bin/traits_report.py +++ b/bin/traits_report.py @@ -1,90 +1,17 @@ -#!/usr/bin/env python3 -"""traits_report.py: the traits table and the genotype rules of step 42, read from the gVCF genotypes. +# remnant: kept for collect_summary, demo and catalog tests; deleted in lane R3. +"""Step 42 metadata still read by the Python report collectors. - traits_report.py --table assets/traits.tsv --rules assets/genotype_rules.tsv --genotypes G.tsv \\ - [--phased P.vcf] --sample S --out S_traits.tsv --rules-out S_genotype_rules.tsv - traits_report.py --rules assets/genotype_rules.tsv --genotypes G.tsv --sample S --phase-sites IN.vcf - traits_report.py --check-table assets/traits.tsv [--rules assets/genotype_rules.tsv] - -G.tsv is what bcftools writes in step 42 (scripts/42-traits.sh and the -Nextflow TRAITS_SITES process run the same commands): one line per gVCF record -at a position of the traits table or of a rule site, after the reference -blocks are expanded to one record per base, as CHROM POS REF ALT GENOTYPE GQ, -the genotype as bases (C/T, or C|T when phased). A position with no line has -no genotype: it is "not_covered", never read as the reference. bcftools drops a -record with a missing allele (a no-call or a half call such as ./T) before -this script, so such a position has no line and reads as not_covered too; a -half call that still reaches here is other_allele. - -Each traits table row becomes one output row with the genotype (the reference -allele first), a status (called, not_covered, other_allele) and the reading -for that genotype. A gVCF whose REF differs from the table's reference base at -a position is not GRCh38 (or the table is wrong): the run stops. - -Each rule of the rules table (assets/genotype_rules.tsv) reads its sites -together and becomes one line of --rules-out: the two haplotypes the genotypes -make, named by the rule's haplotype lines, and the combination line of that -diplotype. A rule with needs_phase no takes the combination line of every -diplotype the unphased genotypes allow (APOE's double heterozygote is e2/e4 -or e1/e3). A rule with needs_phase yes and two or more heterozygous sites -needs their phase: --phase-sites writes those sites as a VCF for WhatsHap, -and --phased reads WhatsHap's answer; sites not phased in one block leave the -rule "unphased", not read. The APOE rule's line also goes into the traits -table, right after its last site row, where the reports read it. - ---check-table checks the schema of the table (and of the rules, with --rules) -and exits 1 on the first list of problems; tests/test_traits_table.py and -tests/test_traits_report.py run the same checks. Standard library only (it -runs in the plain python image and in the WhatsHap image). - -Copy-aware markers (MARKERS): two well-sourced markers of the rule catalog's -list sit in sequence the genome holds more than once, so a gVCF genotype at -one position cannot read them and they are not rule lines. Each is read from -the BAM instead, by the step that counts its copies (og cn marker-regions, -marker-read and call --marker, in rust/crates/og-cn): the reads at every -GRCh38 copy of the site are added up, and the share that carries the marker's -base is a fraction of copies, not a diploid genotype. MARKERS stays here for -the reports (bin/collect_summary.py reads the marker tables with it). - LPA_KIV2_4925 the LPA KIV-2 4925G>A splice variant (Coassin et al. 2017, - PMID 28444229), read by step 46 beside the KIV-2 count - SMN1_27134 SMN1 g.27134T>G (rs143838139; Luo et al. 2014, PMID - 23788250), read by step 35 beside the SMN1 psCN +The executable implementation is og traits table, phase-sites and check. +Copy-aware marker readers are og cn marker-read. """ -import argparse -import itertools -import os -import re -import sys - COLUMNS = ["chrom", "pos", "rsid", "gene", "ref", "alt", "trait", "hom_ref", "het", "hom_alt", "ancestry", "note", "pmids"] OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids"] RULE_OUT_COLUMNS = ["rule", "gene", "trait", "rsids", "chrom", "pos", "genotypes", "gq", "phase", "result", "status", "reading", "ancestry", "note", "pmids"] -# The columns of each kind of line of the rules table, after the kind and the rule id. -RULE_KINDS = {"rule": ["gene", "trait", "needs_phase", "ancestry", "note", "pmids"], - "site": ["rsid", "chrom", "pos", "ref", "alt"], - "haplotype": ["name", "alleles"], - "combination": ["diplotypes", "label", "reading"]} -CHROMS = {f"chr{n}" for n in range(1, 23)} -BASES = set("ACGT") -# APOE: rs429358 C (Arg112) and rs7412 C (Arg158) make e4; T and T make e2. -# The traits table must keep these two rows: the reports show them, and the -# APOE rule's line goes after them. -APOE_E4_SITE, APOE_E2_SITE = "rs429358", "rs7412" -APOE_ALLELES = {APOE_E4_SITE: ("T", "C"), APOE_E2_SITE: ("C", "T")} -# Rules whose line also goes into the traits table, where the reports read it -# (they fold the APOE lines away). Every other rule's line is in --rules-out. TRAITS_TABLE_RULES = ("APOE",) -# The rules table beside the traits table in the repository, for report()'s callers -# that give none; the command line always takes --rules. -DEFAULT_RULES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "assets", "genotype_rules.tsv") -MAX_RULE_SITES = 8 # the completeness check walks 3^sites genotypes -NOT_COVERED = "No genotype at this position in the gVCF (no reads, or a no-call): not read." -OTHER_ALLELE = "An allele this table does not describe at this position: not read." - class TableError(Exception): pass @@ -115,425 +42,6 @@ def read_table(path): return rows -def text_errors(where, text): - errs = [] - if "snpedia" in text.lower(): - errs.append(f"{where}: mentions SNPedia; the table takes no text from it") - if '"' in text or not text.isascii(): - errs.append(f"{where}: a double quote or a non-ASCII character (the reports read the table as plain TSV)") - return errs - - -def site_errors(where, r): - """Schema problems of one site: chrom, pos, rsid, ref and alt.""" - errs = [] - if r["chrom"] not in CHROMS: - errs.append(f"{where}: chrom '{r['chrom']}' is not chr1 to chr22") - if not re.fullmatch(r"[1-9][0-9]*", r["pos"]): - errs.append(f"{where}: pos '{r['pos']}' is not a positive integer") - if not re.fullmatch(r"rs[1-9][0-9]*", r["rsid"]): - errs.append(f"{where}: rsid is not rs") - for k in ("ref", "alt"): - if r[k] not in BASES: - errs.append(f"{where}: {k} '{r[k]}' is not one of A, C, G, T") - if r["ref"] == r["alt"]: - errs.append(f"{where}: ref and alt are the same base") - return errs - - -def check_rows(rows): - """Every schema problem of the table rows, as messages (empty when it is valid).""" - errs, ids, sites = [], set(), set() - for r in rows: - where = f"line {r['line']} ({r['rsid']})" - errs += site_errors(where, r) - if r["rsid"] in ids: - errs.append(f"{where}: rsid appears twice") - ids.add(r["rsid"]) - if (r["chrom"], r["pos"]) in sites: - errs.append(f"{where}: position {r['chrom']}:{r['pos']} appears twice") - sites.add((r["chrom"], r["pos"])) - for k in ("gene", "trait", "hom_ref", "het", "hom_alt", "note"): - if not r[k].strip(): - errs.append(f"{where}: {k} is empty") - if r["ancestry"] not in ("yes", "no"): - errs.append(f"{where}: ancestry '{r['ancestry']}' is not yes or no") - if not re.fullmatch(r"[1-9][0-9]*(,[1-9][0-9]*)*", r["pmids"]): - errs.append(f"{where}: pmids '{r['pmids']}' is not a comma-separated list of PubMed ids") - errs += text_errors(where, " ".join(r[k] for k in COLUMNS)) - by_id = {r["rsid"]: r for r in rows} - for rs, (ref, alt) in APOE_ALLELES.items(): - r = by_id.get(rs) - if r is None: - errs.append(f"APOE row {rs} is missing") - elif (r["ref"], r["alt"]) != (ref, alt) or r["chrom"] != "chr19": - errs.append(f"APOE row {rs} must be chr19 {ref}>{alt}, is {r['chrom']} {r['ref']}>{r['alt']}") - return errs - - -# --- The rules table --------------------------------------------------------------- - -def read_rules(path): - """{rule id: {gene, trait, needs_phase, ancestry, note, pmids, line, sites, haplotypes, - combinations}} in file order. Raises TableError on a line of an unknown kind or with - the wrong column count, and on a site, haplotype or combination line before its rule line.""" - rules = {} - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line or line.startswith("#"): - continue - cells = line.split("\t") - kind = cells[0] - if kind not in RULE_KINDS: - raise TableError(f"{path}:{n}: kind '{kind}' is not one of {', '.join(RULE_KINDS)}") - want = 2 + len(RULE_KINDS[kind]) - if len(cells) != want: - raise TableError(f"{path}:{n}: a {kind} line has {len(cells)} columns, expected {want}") - rid = cells[1] - d = dict(zip(RULE_KINDS[kind], cells[2:]), line=n) - if kind == "rule": - if rid in rules: - raise TableError(f"{path}:{n}: rule {rid} has two rule lines") - rules[rid] = dict(d, id=rid, sites=[], haplotypes=[], combinations=[]) - continue - if rid not in rules: - raise TableError(f"{path}:{n}: a {kind} line of rule '{rid}' before its rule line") - rules[rid]["sites" if kind == "site" else kind + "s"].append(d) - if not rules: - raise TableError(f"{path}: no rule line") - return rules - - -def diplotype_key(rule, a, b): - """'a/b' with the two names in the order of the rule's haplotype lines.""" - order = [h["name"] for h in rule["haplotypes"]] - return "/".join(sorted((a, b), key=order.index)) - - -def allowed_diplotypes(rule, genotypes): - """The diplotype keys, in haplotype order, whose two haplotypes give these unphased - genotypes (one [allele, allele] per site).""" - haps = rule["haplotypes"] - out = [] - for i, j in itertools.combinations_with_replacement(range(len(haps)), 2): - a, b = haps[i]["alleles"].split(","), haps[j]["alleles"].split(",") - if all(sorted((a[k], b[k])) == sorted(g) for k, g in enumerate(genotypes)): - out.append(f"{haps[i]['name']}/{haps[j]['name']}") - return out - - -def check_rules(rules, table_rows=None): - """Every schema problem of the rules, as messages (empty when they are valid). With - table_rows, also: a rule site that is a traits table row has the row's chrom, pos, ref - and alt, and the rules of TRAITS_TABLE_RULES exist with every site in the table.""" - errs = [] - by_rs = {r["rsid"]: r for r in table_rows or []} - for rid, rule in rules.items(): - n0 = len(errs) - where = f"rule {rid} (line {rule['line']})" - if not re.fullmatch(r"[A-Za-z0-9_]+", rid): - errs.append(f"{where}: the rule id is not letters, digits and _") - for k in ("gene", "trait", "note"): - if not rule[k].strip(): - errs.append(f"{where}: {k} is empty") - for k in ("needs_phase", "ancestry"): - if rule[k] not in ("yes", "no"): - errs.append(f"{where}: {k} '{rule[k]}' is not yes or no") - if not re.fullmatch(r"[1-9][0-9]*(,[1-9][0-9]*)*", rule["pmids"]): - errs.append(f"{where}: pmids '{rule['pmids']}' is not a comma-separated list of PubMed ids") - sites = rule["sites"] - if not 2 <= len(sites) <= MAX_RULE_SITES: - errs.append(f"{where}: {len(sites)} sites, expected 2 to {MAX_RULE_SITES}") - seen = set() - for s in sites: - w = f"{where}, site line {s['line']} ({s['rsid']})" - errs += site_errors(w, s) - if s["rsid"] in seen or (s["chrom"], s["pos"]) in seen: - errs.append(f"{w}: the site appears twice in the rule") - seen |= {s["rsid"], (s["chrom"], s["pos"])} - t = by_rs.get(s["rsid"]) - if t and (t["chrom"], t["pos"], t["ref"], t["alt"]) != (s["chrom"], s["pos"], s["ref"], s["alt"]): - errs.append(f"{w}: the traits table has {t['chrom']}:{t['pos']} {t['ref']}>{t['alt']}, " - f"the rule {s['chrom']}:{s['pos']} {s['ref']}>{s['alt']}") - if len({s["chrom"] for s in sites}) > 1: - errs.append(f"{where}: the sites are on more than one chromosome") - names, tuples = [], set() - for h in rule["haplotypes"]: - w = f"{where}, haplotype line {h['line']} ({h['name']})" - if not re.fullmatch(r"[A-Za-z0-9*.]+", h["name"]): - errs.append(f"{w}: the name is not letters, digits, * and .") - if h["name"] in names: - errs.append(f"{w}: the name appears twice") - names.append(h["name"]) - al = h["alleles"].split(",") - if len(al) != len(sites): - errs.append(f"{w}: {len(al)} alleles for {len(sites)} sites") - continue - for a, s in zip(al, sites): - if a not in (s["ref"], s["alt"]): - errs.append(f"{w}: allele {a} at {s['rsid']} is neither its ref {s['ref']} nor its alt {s['alt']}") - if tuple(al) in tuples: - errs.append(f"{w}: the same alleles as another haplotype") - tuples.add(tuple(al)) - if len(names) < 2: - errs.append(f"{where}: fewer than two haplotypes") - keys = {} - for c in rule["combinations"]: - w = f"{where}, combination line {c['line']} ({c['diplotypes']})" - for k in ("label", "reading"): - if not c[k].strip(): - errs.append(f"{w}: {k} is empty") - errs += text_errors(w, " ".join((c["diplotypes"], c["label"], c["reading"]))) - if c["diplotypes"] in keys: - errs.append(f"{w}: the diplotypes appear twice") - keys[c["diplotypes"]] = c - for d in c["diplotypes"].split("|"): - pair = d.split("/") - if len(pair) != 2 or any(p not in names for p in pair): - errs.append(f"{w}: '{d}' is not two of the rule's haplotypes joined by /") - elif diplotype_key(rule, *pair) != d: - errs.append(f"{w}: '{d}' is not in the order of the haplotype lines " - f"(write {diplotype_key(rule, *pair)})") - errs += text_errors(where, " ".join(str(rule[k]) for k in ("gene", "trait", "note"))) - if len(errs) > n0: - continue - # Completeness: the combination lines are exactly what the genotypes can give. - if rule["needs_phase"] == "yes": - want = {f"{a}/{b}" for a, b in itertools.combinations_with_replacement(names, 2)} - else: - want = set() - choices = [[[s["ref"]] * 2, [s["ref"], s["alt"]], [s["alt"]] * 2] for s in sites] - for g in itertools.product(*choices): - key = "|".join(allowed_diplotypes(rule, g)) - if key: - want.add(key) - for k in sorted(want - set(keys)): - errs.append(f"{where}: no combination line for {k}") - for k in sorted(set(keys) - want): - errs.append(f"{where}: the combination line {k} can never be read " - + ("(a rule that needs phase reads one diplotype)" if "|" in k else "from these haplotypes")) - if table_rows is not None: - for rid in TRAITS_TABLE_RULES: - if rid not in rules: - errs.append(f"rule {rid} is missing: its line goes into the traits table, where the reports read it") - else: - for s in rules[rid]["sites"]: - if s["rsid"] not in by_rs: - errs.append(f"rule {rid}: site {s['rsid']} has no traits table row") - return errs - - -# --- Genotypes ---------------------------------------------------------------------- - -def read_genotypes(path): - """{(chrom, pos): [(REF, [allele, allele], GQ), ...]} from bcftools' CHROM POS REF ALT GT [GQ] lines.""" - recs = {} - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line: - continue - c = line.split("\t") - if len(c) < 5: - raise TableError(f"{path}:{n}: {len(c)} columns, expected CHROM POS REF ALT GENOTYPE [GQ]") - gq = c[5] if len(c) > 5 and c[5] else "." - recs.setdefault((c[0], c[1]), []).append((c[2], re.split(r"[/|]", c[4]), gq)) - return recs - - -def site_genotype(site, recs): - """(alleles sorted with the reference first, status, GQ) of one site from its gVCF records.""" - if not recs: - return None, "not_covered", "." - for r in recs: - if r[0][:1] != site["ref"]: - raise TableError(f"{site['rsid']} at {site['chrom']}:{site['pos']}: the gVCF has REF {r[0]}, the table " - f"{site['ref']} (the GRCh38 base). Is the gVCF on GRCh38?") - snv = [r for r in recs if all(len(a) == 1 for a in r[1])] - _, alleles, gq = (snv or recs)[0] - order = {site["ref"]: 0, site["alt"]: 1} - alleles = sorted(alleles, key=lambda a: (order.get(a, 2), a)) - if len(alleles) != 2 or any(a not in order for a in alleles): - return alleles, "other_allele", gq - return alleles, "called", gq - - -def call(row, recs): - """(genotype, status, reading) of one traits table row from its gVCF records.""" - alleles, status, _ = site_genotype(row, recs) - if status == "not_covered": - return ".", "not_covered", NOT_COVERED - gt = "/".join(alleles) - if status == "other_allele": - return gt, "other_allele", OTHER_ALLELE - return gt, "called", row[("hom_ref", "het", "hom_alt")[alleles.count(row["alt"])]] - - -def read_phased(path): - """{(chrom, pos): (allele of haplotype 1, allele of haplotype 2, phase block)} of the - phased records of WhatsHap's VCF; unphased records are left out.""" - out = {} - with open(path, encoding="utf-8") as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 10: - continue - fmt = dict(zip(c[8].split(":"), c[9].split(":"))) - gt, ps = fmt.get("GT", ""), fmt.get("PS", ".") - if "|" not in gt or ps in (".", ""): - continue - bases = [c[3]] + c[4].split(",") - try: - a, b = (bases[int(i)] for i in gt.split("|")) - except (ValueError, IndexError): - continue - out[(c[0], c[1])] = (a, b, ps) - return out - - -def phase_sites(rules, recs): - """The (chrom, pos, rsid, ref, alt) of the heterozygous sites of every rule that needs - phase and has two or more of them, each site once, in rule order.""" - out, seen = [], set() - for rule in rules.values(): - if rule["needs_phase"] != "yes": - continue - g = [(s, site_genotype(s, recs.get((s["chrom"], s["pos"]), []))) for s in rule["sites"]] - if any(st != "called" for _, (_, st, _) in g): - continue - het = [s for s, (al, _, _) in g if al[0] != al[1]] - if len(het) < 2: - continue - for s in het: - k = (s["chrom"], s["pos"]) - if k not in seen: - seen.add(k) - out.append((s["chrom"], s["pos"], s["rsid"], s["ref"], s["alt"])) - return out - - -def write_phase_vcf(path, sample, sites): - """The sites as a one-sample VCF of 0/1 genotypes, sorted, for whatshap phase.""" - contigs = sorted({c for c, *_ in sites}, key=lambda c: int(c[3:])) - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("##fileformat=VCFv4.2\n") - for c in contigs: - f.write(f"##contig=\n") - f.write('##FORMAT=\n') - f.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + sample + "\n") - for c, p, rs, ref, alt in sorted(sites, key=lambda s: (int(s[0][3:]), int(s[1]))): - f.write(f"{c}\t{p}\t{rs}\t{ref}\t{alt}\t.\tPASS\t.\tGT\t0/1\n") - os.replace(tmp, path) - - -def rule_line(rule, recs, phased): - """One output line of a rule. phased: read_phased's dict, or None when no phasing ran.""" - sites = rule["sites"] - g = [site_genotype(s, recs.get((s["chrom"], s["pos"]), [])) for s in sites] - rsids = [s["rsid"] for s in sites] - line = {"rule": rule["id"], "gene": rule["gene"], "trait": rule["trait"], "rsids": ",".join(rsids), - "chrom": sites[0]["chrom"], "pos": ",".join(s["pos"] for s in sites), - "genotypes": ",".join("/".join(al) if al else "." for al, _, _ in g), - "gq": ",".join(gq for _, _, gq in g), "ancestry": rule["ancestry"], "note": rule["note"], - "pmids": rule["pmids"]} - phase = "not_used" if rule["needs_phase"] == "no" else "not_needed" - statuses = [st for _, st, _ in g] - if any(st != "called" for st in statuses): - need = (f"both {rsids[0]} and {rsids[1]}" if len(rsids) == 2 - else "every one of " + ", ".join(rsids[:-1]) + " and " + rsids[-1]) - return dict(line, phase=".", result=".", - status="not_covered" if "not_covered" in statuses else "other_allele", - reading=f"Not read: needs a genotype at {need}.") - alleles = [al for al, _, _ in g] - combos = {c["diplotypes"]: c for c in rule["combinations"]} - if rule["needs_phase"] == "no": - key = "|".join(allowed_diplotypes(rule, alleles)) - else: - het = [i for i, al in enumerate(alleles) if al[0] != al[1]] - h1, h2 = [al[0] for al in alleles], [al[1] for al in alleles] - if len(het) >= 2: - blocks = {(phased or {}).get((sites[i]["chrom"], sites[i]["pos"]), (None, None, None))[2] for i in het} - if None in blocks or len(blocks) != 1: - why = ("no read or read pair covered them together" - if phased is not None else "no phasing ran (no BAM, or phasing failed)") - return dict(line, phase="unphased", result=".", status="unphased", - reading=f"Heterozygous at {len(het)} of the {len(sites)} sites, and {why}, so which " - "alleles share a copy is not known: not read.") - for i in het: - h1[i], h2[i] = phased[(sites[i]["chrom"], sites[i]["pos"])][:2] - phase = "read_backed" - by_alleles = {tuple(h["alleles"].split(",")): h["name"] for h in rule["haplotypes"]} - n1, n2 = by_alleles.get(tuple(h1)), by_alleles.get(tuple(h2)) - unknown = [",".join(h) for h, n in ((h1, n1), (h2, n2)) if n is None] - if unknown: - return dict(line, phase=phase, result=".", status="other_haplotype", - reading=f"A haplotype this rule does not describe ({' and '.join(unknown)} at " - f"{', '.join(rsids)}): not read.") - key = diplotype_key(rule, n1, n2) - if not key or key not in combos: - return dict(line, phase=phase, result=".", status="other_haplotype", - reading="A combination of haplotypes this rule does not describe: not read.") - c = combos[key] - return dict(line, phase=phase, result=c["label"], status="called", reading=c["reading"]) - - -def rule_lines(rules, recs, phased=None): - """One output line per rule, in the order of the rules table.""" - return [rule_line(r, recs, phased) for r in rules.values()] - - -def report(rows, recs, rules=None, phased=None, lines=None): - """The traits output rows: one per table row, with the line of each rule of - TRAITS_TABLE_RULES right after its last site row. rules defaults to - DEFAULT_RULES; lines, rule_lines' output, is computed when not given.""" - if rules is None: - rules = read_rules(DEFAULT_RULES) - if lines is None: - lines = rule_lines(rules, recs, phased) - after = {} - for rid in TRAITS_TABLE_RULES: - rule = rules.get(rid) - if rule: - members = {s["rsid"] for s in rule["sites"]} - last = max((i for i, r in enumerate(rows) if r["rsid"] in members), default=None) - if last is not None: - after[last] = next(ln for ln in lines if ln["rule"] == rid) - out = [] - for i, r in enumerate(rows): - gt, status, reading = call(r, recs.get((r["chrom"], r["pos"]), [])) - out.append({"rsid": r["rsid"], "gene": r["gene"], "chrom": r["chrom"], "pos": r["pos"], - "trait": r["trait"], "genotype": gt, "status": status, "reading": reading, - "ancestry": r["ancestry"], "note": r["note"], "pmids": r["pmids"]}) - if i in after: - ln = after[i] - out.append({"rsid": ln["rsids"].replace(",", "+"), "gene": ln["gene"], "chrom": ln["chrom"], - "pos": ln["pos"], "trait": ln["trait"], "genotype": ln["result"], "status": ln["status"], - "reading": ln["reading"], "ancestry": ln["ancestry"], "note": ln["note"], - "pmids": ln["pmids"]}) - return out - - -# --- Copy-aware markers -------------------------------------------------------------- - -# Each marker: the dbSNP record of one copy (rsID, chrom, POS, REF, ALT of -# dbSNP b157's GRCh38 VCF, read from NCBI's Variation API on 2026-11-07), every -# GRCh38 copy of the site (1-based), the plus-strand base pair read there, and -# the GRCh38 sequence of 10 bases on each side of every copy (UCSC hg38 and -# Ensembl GRCh38 agree, 2026-11-07): the same 21 bases at each copy, which is -# why the reads cannot be placed on one copy. tests/test_traits_report.py -# checks every copy against the context and the record against the marker, and -# that og cn reads the same copies; tests/e2e/genotype-rules-2-smn1-marker.sh -# compares the records with dbSNP. -# min_depth fewer reads over all copies together: too little depth -# min_alt reads with the marker's base for 'present' ... -# min_fraction ... and a share of the reads at least this -# max_absent this many reads with the marker's base or fewer: 'not seen' -# (one or two reads can be sequencing errors); between the two -# the marker is 'uncertain' MARKERS = { # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds @@ -560,108 +68,3 @@ def report(rows, recs, rules=None, phased=None, lines=None): }, } MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") - - -def marker_errors(name, m=None, dbsnp=None): - """Problems of one marker (empty when it is consistent): the dbSNP record - is one of its copies with its ref and alt, on the plus strand and not - swapped; every copy sits in the middle of the same context, whose middle - base is the ref.""" - m = MARKERS[name] if m is None else m - rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp - comp = {"A": "T", "C": "G", "G": "C", "T": "A"} - errs = [] - if chrom != m["chrom"] or pos not in m["copies"]: - errs.append(f"{name}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies") - if (m["ref"], m["alt"]) == (alt, ref): - errs.append(f"{name}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt})") - elif (m["ref"], m["alt"]) == (comp.get(ref), comp.get(alt)): - errs.append(f"{name}: the alleles are the minus-strand bases (marker {m['ref']}>{m['alt']}, " - f"dbSNP {ref}>{alt}): write the plus-strand bases") - elif (m["ref"], m["alt"]) != (ref, alt): - errs.append(f"{name}: marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt}") - ctx = m["context"] - if len(ctx) % 2 != 1 or ctx[len(ctx) // 2] != m["ref"]: - errs.append(f"{name}: the middle base of the context is not the ref {m['ref']}") - if len(set(m["copies"])) != len(m["copies"]): - errs.append(f"{name}: a copy is listed twice") - return errs - - -def write_tsv(path, out, columns=OUT_COLUMNS): - tmp = path + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - f.write("\t".join(columns) + "\n") - for o in out: - f.write("\t".join(str(o[k]) for k in columns) + "\n") - os.replace(tmp, path) - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--table", help="assets/traits.tsv") - ap.add_argument("--rules", help="assets/genotype_rules.tsv") - ap.add_argument("--genotypes", help="bcftools' CHROM POS REF ALT GENOTYPE GQ lines at the table and rule positions") - ap.add_argument("--phased", help="WhatsHap's VCF of the --phase-sites sites (absent: no phasing ran)") - ap.add_argument("--sample") - ap.add_argument("--out", help="the sample's traits table (TSV)") - ap.add_argument("--rules-out", help="the sample's genotype rules (TSV)") - ap.add_argument("--phase-sites", metavar="VCF", help="write the sites WhatsHap must phase and exit") - ap.add_argument("--check-table", metavar="TABLE", help="check the table's schema (and --rules') and exit") - a = ap.parse_args(argv) - try: - if a.check_table: - rows = read_table(a.check_table) - errs = check_rows(rows) - if a.rules: - errs += check_rules(read_rules(a.rules), rows) - for e in errs: - print(f"ERROR: {e}", file=sys.stderr) - if not errs: - print(f"{a.check_table}{' and ' + a.rules if a.rules else ''}: schema OK") - return 1 if errs else 0 - if a.phase_sites: - if not (a.rules and a.genotypes and a.sample): - ap.error("--phase-sites needs --rules, --genotypes and --sample") - rules = read_rules(a.rules) - errs = check_rules(rules) - if errs: - for e in errs: - print(f"ERROR: {a.rules}: {e}", file=sys.stderr) - return 1 - sites = phase_sites(rules, read_genotypes(a.genotypes)) - write_phase_vcf(a.phase_sites, a.sample, sites) - print(f"{a.sample}: {len(sites)} heterozygous rule sites to phase. Written: {a.phase_sites}") - return 0 - if not (a.table and a.rules and a.genotypes and a.sample and a.out and a.rules_out): - ap.error("give --table, --rules, --genotypes, --sample, --out and --rules-out " - "(or --phase-sites, or --check-table)") - rows = read_table(a.table) - rules = read_rules(a.rules) - errs = [f"{a.table}: {e}" for e in check_rows(rows)] + [f"{a.rules}: {e}" for e in check_rules(rules, rows)] - if errs: - for e in errs: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - phased = read_phased(a.phased) if a.phased else None - recs = read_genotypes(a.genotypes) - lines = rule_lines(rules, recs, phased) - out = report(rows, recs, rules, phased, lines) - except (TableError, OSError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - write_tsv(a.out, out) - write_tsv(a.rules_out, lines, RULE_OUT_COLUMNS) - counts = {} - for o in out: - counts[o["status"]] = counts.get(o["status"], 0) + 1 - print(f"{a.sample}: {len(rows)} table rows; " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) - + f" (APOE line included). Written: {a.out}") - print(f"{a.sample}: {len(lines)} genotype rules; " - + ", ".join(f"{ln['rule']} {ln['result'] if ln['status'] == 'called' else ln['status']}" for ln in lines) - + f". Written: {a.rules_out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/42-traits.md b/docs/42-traits.md index 7c6affb70..ad3aa25cf 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -12,7 +12,7 @@ These are the results most people find easiest to read, and the pipeline had no - **bcftools** (`bcftools convert --gvcf2vcf`, `view`, `query`) reads the genotypes at the table's positions from the gVCF. - **WhatsHap** (`whatshap phase`) phases the heterozygous sites of a rule that needs phase from the BAM's reads. -- **`bin/traits_report.py`** (Python standard library) reads each genotype against the table, and each rule against the rule table. +- **`og traits table`** (Rust) reads each genotype against the table, and each rule against the rule table. ## Docker Image @@ -122,12 +122,12 @@ Kim et al. (2003) found five TAS2R38 haplotypes worldwide. The rule reads the tw ### Checks on the rule table -- `bin/traits_report.py --check-table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `tests/test_traits_report.py`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. +- `bin/og traits check --table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `tests/test_traits_report.py`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. - `scripts/ci/check-rule-alleles.sh` checks every site against dbSNP: the rsID at that GRCh38 position, ref as dbSNP's REF and alt among its ALT alleles. A pair written the wrong way round, or as the minus-strand bases (TAS2R38 lies on the minus strand, where the papers name the codon bases), fails with the reason. The dbSNP records it compares with sit in the script, taken from dbSNP's GRCh38 VCF of the build `CONSUMER_DBSNP_BUILD` pins in `versions.env` (b157); `--online` reads them again over HTTPS and fails when they differ, and the e2e case `genotype-rules-1-phase` runs it. A C/G or A/T site is its own complement, so there a strand mistake keeps valid alleles: the check prints a note for each such site (rs713598 is one), and its haplotype alleles rest on the codon: Pro49 has C at the first base of codon 49 on the coding strand (NM_176817.5 c.145C, where Ala49 has G), so the PAV base on GRCh38 is its complement, G. The other such sites: rs17580 (SERPINA1 S, minus strand: Glu264 GAA, Val264 GTA, so the S base on GRCh38 is the complement of T, A), rs1799945 (HFE H63D, plus strand: His63 CAT, Asp63 GAT, so D is G) and rs1333049 (no gene; the risk allele C is the plus-strand base of the WTCCC 2007 record). ### Adding a rule -Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `python3 tests/test_traits_report.py`. +Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `OG=bin/og python3 tests/test_traits_report.py`. ## APOE diff --git a/docs/cn-loci.md b/docs/cn-loci.md index a9b918f5f..71f2ef3e0 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -1,6 +1,6 @@ # Copy Number from Depth (the locus catalog) -Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one table, `bin/cn_loci.tsv`, with one row per locus, and one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)). Step 41 uses the same engine: `og cn bed` gives it the RHD regions, and `og bloodtype report` makes the RhD call with og cn's rules on the RHD row. `bin/copy_number.py` stays only for the catalog tests. A new locus is a new row, not a new script. +Several steps count copies of a gene from short-read depth: HBA1 and HBA2 in step 35, the RHD deletion in step 41 and the LPA KIV-2 repeat in step 46. Step 35 also counts three gene-dosage loci that no pinned caller models (HP, LCE3B/LCE3C and CCL3L1), CYP21A2 from depth and its pseudogene variant sites, and the interval rows: the recurrent CNV regions ClinGen curates (22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12 and the rest), five carrier deletion alleles and the Y chromosome AZF regions. They all use one table, `bin/cn_loci.tsv`, with one row per locus, and one engine, `og cn` (the Rust crate `rust/crates/og-cn`, run through `bin/og`; [docs/rust.md](rust.md)). Step 41 uses the same engine: `og cn bed` gives it the RHD regions, and `og bloodtype report` makes the RhD call with og cn's rules on the RHD row. The catalog tests materialise the historical Python engine from its pinned Git commit as a differential oracle. A new locus is a new row, not a new script. The table sits in `bin/` because `bin/` is the one repository folder that every container mounts: the bash steps mount it as `/pgp-bin`, and Nextflow puts it on every task's PATH. diff --git a/docs/rust.md b/docs/rust.md index 17ed2cecf..94103db1c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. @@ -110,3 +110,142 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to - **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. [Testing](testing.md) has the rest of CI. + +## CI evidence and fixture verbs + +`og dev evidence clashes` reads two gzip VCFs and gzip archaic tables. +`og dev evidence hg002-gvcf` reads a gzip truth VCF, plain BED, plain FASTA +and FAI, and a plain sites table. `og dev renovate-check` reads a plain log +and plain repository sources. Text uses strict UTF-8 and universal newlines +(LF, CRLF and CR). Compression is explicit, even when a filename is misleading. + +Evidence, `og dev fixture comphet`, `og dev fixture trim-pharmcat` and the +Renovate summary stage output beside its resolved destination. A file symlink +stays a symlink, existing permission bits are preserved, and read-only files +and directory destinations fail. New files use the caller's umask. +The VNtyper downloader follows the same destination policy, uses a private +`mktemp -d` directory, and removes it on handled failures and signals. It needs +GNU `mv -T` (`gmv` on macOS) so a directory can never receive a temporary BAM. + +Comphet stages all three files and checks all three destinations before the +first rename. A bad later destination leaves previous outputs intact. +Three renames are still not one transaction. A crash or a filesystem failure +between renames can publish only part of the new set. Unhandled SIGKILL can +leave temporary files. Single-file outputs are replaced by one rename. Publication replaces the file +and does not update other hard links. Those links still refer to the old inode. + +The shared runner fails the command if stdout or stderr cannot be written or +flushed. This includes the evidence status and Renovate table. A configured +Renovate summary must also be written and flushed successfully. +Guard runs the FASTA offset regression in debug and release profiles. + +### Input and destination refusal rule + +These evidence, fixture and Renovate verbs may refuse any input that is +malformed, outside the numeric ranges or size limits below, or not representable +by the JSON reader. They may also refuse any destination beside which they +cannot stage a sibling temporary file. A refusal always exits nonzero with a +specific message. It leaves every existing output byte-identical, with its mode +and link target unchanged. The crash and multi-file publication limits above +still apply. + +- Evidence numeric fields use ASCII unsigned integers from 0 through + 18446744073709551615. A reference lookup needs a position of at least 1. + Spans must fit that range. Calculated FASTA offsets must fit 0 through + 9223372036854775807 and address a byte inside the file. +- Every FAI row needs at least five columns and nonzero bases per line. +- `MAX_ANNOTATIONS` is an ASCII signed integer from -9223372036854775808 + through 9223372036854775807. +- Evidence and Renovate text records have a 16 MiB (16777216-byte) limit. + The PharmCAT reader has no separate file-size limit and materializes JSON. +- The JSON reader accepts signed 128-bit integers (-2^127 through 2^127-1), + Unicode scalar strings and at most 1000 nested arrays or objects. + It cannot represent lone surrogates, including in unused fields. +- Destinations need a writable parent and a regular writable target when one + exists. Resolution follows at most 40 final symlinks. Temporary sibling names + must fit the filesystem's name limit. A 255-byte basename is refused cleanly + before any output write because its temporary sibling name is longer. + +Inputs produced by the real tools are never refused: bcftools/DeepVariant VCF, +samtools faidx, header-free BED intervals, PharmCAT report JSON and Renovate +debug log. A BED file with a UCSC column header, a `track` line or a `browser` +line is refused, as the Python refused it: remove those lines first. + +Valid JSON nested up to the reader's 1000 levels is accepted. The Python stopped +with a RecursionError at a few hundred levels when it copied or wrote such a +value. That was a limit of the interpreter, not a check on the input, so the +Rust commands do not copy it. + +`refusals_preserve_bytes_modes_and_link_targets` pins the rule through each CLI. +Each table row runs with a regular output and with a symlink. It checks nonzero +exit, the specific diagnostic, unchanged bytes, mode and link target, and no +leftover temporary file. Adding a refusal case needs one row. + +Examples of refusals follow. This is not an exhaustive input grammar. +The final-LF normalization has a positive byte test. + +- BED fields use spaces and tabs as separators. Other Python whitespace + (Unicode White_Space and U+001C through U+001F) fails with `unsupported whitespace`. +- Renovate present fields of the wrong type fail with `not a map of package file lists`. + This includes map-valued updates and numeric dependency names, current values, + or update values. Falsy warnings and packages with no dependencies remain valid. +- PharmCAT truthy strings or objects where lists belong fail with `is not a list`. +- PharmCAT truthy non-object alleles fail with `is not an object`. +- Evidence positions with underscores, non-ASCII digits, negative values or + values outside u64 fail with `not a number`. +- Four-field truth VCF rows and blank FAI rows fail with `fewer than 5 columns`. +- Text records over 16 MiB, including ignored headers or log lines, fail with + `record exceeds 16777216 bytes`. +- Clash VCF rows need a tab after ALT. A newline in the ALT field of a short + row fails with `missing trailing VCF columns`. +- A truth VCF record without its final LF gets one in the output. Valid VCF ends with LF. + +- Zero-byte gzip VCFs or selected gzip tables fail with `wrong compression (expected gzip)`. +- Every FAI row needs nonzero bases per line, including unused rows. + A zero width fails with `a row has 0 bases per line`. +- `MAX_ANNOTATIONS` must parse as a signed 64-bit ASCII integer. + Underscores, non-ASCII digits and out-of-range values fail with + `MAX_ANNOTATIONS ... is not a whole number` (exit 2). +- Staging needs a writable parent directory, even for an existing writable file. + A parent with no write bits fails with `parent directory is not writable`. + Other access failures report the operating system error. +- The FASTA reader accepts plain files only. A gzip magic prefix fails with + `wrong compression (expected plain)`, including gzip with a long FNAME. +- A VCF header after the first truth record fails with + `VCF header after first data record`. The previous destination stays intact. +- FASTA offsets use checked subtraction, division, multiplication, addition + and remainder. Overflow, a position beyond the file, or an offset outside + the signed file-offset range fails with `invalid FASTA offset`. + +- A nonblank unused FAI row with fewer than five columns fails with + `a row has fewer than 5 columns`. +- A truth position that fits u64 but whose REF span does not fails with + `position span overflows`. +- A destination whose temporary sibling name is too long fails with the + filesystem diagnostic `File name too long`. +- JSON integers outside signed 128-bit range and escaped lone surrogates fail + with `Integer beyond 128 bits` and `Lone surrogate`, even in unused fields. + Excess nesting fails with `nested deeper than 1000 levels`. + +Renovate skips genuine JSON syntax errors only. Reader limitations fail the +command before an earlier matching config or the summary can be used. +`TooDeep` has its own error variant. The reader currently puts integer and +surrogate limitations in `JsonError::Decode`, so the loader distinguishes those +static reasons inside that variant. Other decode errors remain skippable. + +## CI readers + +`og dev pharmcat diplotypes REPORT` prints sorted, unique `GENE\tA1/A2` rows from flat or source-nested reports. `--called` prints the called count and names on stderr. `--called-lines` prints one called gene per line for e2e checks. An unreadable report exits 2 without a count. A valid empty report succeeds with a zero count. + +`og dev nf-trace requests TRACE --want FILE --executor-cpus N --executor-mem-gb N` checks initial requests, doubled retry memory and the executor budget. The expectation table is [nf-trace-requests.tsv](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/nf-trace-requests.tsv). `og dev nf-trace attempts TRACE --name-glob PATTERN --want '1:137:32,2:0:64' --nextflow-exit RC` checks a retry sequence. Both accept one explicit path, refuse globs, parse quoted TSV records and reject conflicting duplicates. Identical duplicate task IDs count once. Exit 1 means the check found a mismatch. Exit 2 means input or arguments could not be assessed. + +`og dev smoke` supplies `mini-ref SRC ORIG MINI POS LEN`, `long-reads HAP1 HAP2 OUT.gz`, `eh-catalog REF OUT END`, `csv-cell FILE KEY=VALUE[,KEY=VALUE] COLUMN` and `check KIND FILE`. Check kinds are `hap-ibd`, `ibdmix`, `rfmix`, `eigen` and `parascopy`. Mini-reference bases use a fixed Rust seed. Long-read gzip headers are deterministic. These synthetic fixtures do not change pipeline output files. Smoke references are limited to 1,003,000 bases. CSV lookups and table checks read one record at a time. + +CI integer cells accept surrounding whitespace, signs and digit separators. Trace cells deliberately refuse Unicode digits and negative resource values. Smoke coordinates accept Unicode decimal digits and use signed 128-bit integers. Integers beyond the target range fail. Decimal and nonfinite coordinates fail. Duplicate trace rows retain the first concrete exit status and reject later conflicts. + +The shared `og-io` reader consumes LF, CRLF and lone CR records with storage bounded by the longest line. Smoke JSON checks require one document and typed fields. Catalogs must be arrays and genotypes must be nonempty strings. This deliberately rejects malformed values that Python truthiness or `len()` accepted. Numeric depth strings remain supported; booleans are refused. Smoke fixtures and assertions build `og` on first use if the default binary is missing. An explicit `OG` is used as supplied. + +Step 42 uses `og traits table`, `og traits phase-sites` and `og traits check`. +The readers retain only catalog sites. The trait and genotype-rule output +columns and readings stay the same. `bin/traits_report.py` keeps report +metadata until the Python summary and report collectors move. diff --git a/modules/local/traits/main.nf b/modules/local/traits/main.nf index b7be4b1c8..b6087ab75 100644 --- a/modules/local/traits/main.nf +++ b/modules/local/traits/main.nf @@ -13,7 +13,7 @@ an empty file when no rule has two such sites TRAITS _traits.tsv: the reading of each genotype from the table, and the APOE line; _genotype_rules.tsv: one - line per rule (bin/traits_report.py) + line per rule (og traits table) Runs for the samples with a gVCF only (DEEPVARIANT's, or one the samplesheet gives); main.nf logs the samples it skips. A sample without @@ -89,7 +89,7 @@ process TRAITS_PHASE { script: // The same commands as scripts/42-traits.sh """ - traits_report.py --rules ${rules} --genotypes ${genotypes} --sample ${meta.id} --phase-sites phase_sites.vcf + og traits phase-sites --rules ${rules} --genotypes ${genotypes} --sample ${meta.id} --out phase_sites.vcf if grep -qv '^#' phase_sites.vcf; then whatshap phase --reference ${reference} --ignore-read-groups -o ${meta.id}_traits_phased.vcf phase_sites.vcf ${bam} else @@ -129,7 +129,7 @@ process TRAITS { script: def phased_arg = phased ? "--phased ${phased}" : '' """ - traits_report.py --table ${table} --rules ${rules} --genotypes ${genotypes} ${phased_arg} --sample ${meta.id} \\ + og traits table --table ${table} --rules ${rules} --genotypes ${genotypes} ${phased_arg} --sample ${meta.id} \\ --out ${meta.id}_traits.tsv --rules-out ${meta.id}_genotype_rules.tsv printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c5d9ae54a..2c2bc7544 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -48,6 +48,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.2" @@ -177,6 +192,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fancy-regex" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -664,6 +690,7 @@ dependencies = [ name = "og-dev" version = "0.1.0" dependencies = [ + "fancy-regex", "flate2", "indexmap", "og-intervals", @@ -672,6 +699,7 @@ dependencies = [ "png", "pycompat", "regex", + "regex-syntax", ] [[package]] diff --git a/rust/README.md b/rust/README.md index 39a8457c2..38c77bfbc 100644 --- a/rust/README.md +++ b/rust/README.md @@ -34,3 +34,8 @@ cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # py ``` A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. + +CI readers also run through `og dev pharmcat diplotypes`, `og dev nf-trace requests|attempts` and `og dev smoke`. See [the Rust command notes](../docs/rust.md#ci-readers). +`og traits table`, `og traits phase-sites` and `og traits check` read step 42's +curated catalogs and synthetic or bcftools genotype input. Their tests live +in `og-steps::traits` and the `traits` parity cases. diff --git a/rust/crates/og-dev/Cargo.toml b/rust/crates/og-dev/Cargo.toml index 91e36986b..7cf3c816c 100644 --- a/rust/crates/og-dev/Cargo.toml +++ b/rust/crates/og-dev/Cargo.toml @@ -1,11 +1,15 @@ [package] name = "og-dev" -description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only, build" +description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only, build, scans and version checks" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +# Only the CI red control enables this fault. Normal builds keep it off. +[features] +planted-ci-reader = [] + [dependencies] pycompat.workspace = true og-io.workspace = true @@ -14,6 +18,8 @@ og-intervals.workspace = true flate2.workspace = true indexmap.workspace = true png = "0.18" +fancy-regex = "=0.16.2" +regex-syntax = "=0.8.11" regex = { version = "1", default-features = false, features = ["std", "perf", "unicode"] } [lints] diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs new file mode 100644 index 000000000..a3e2f12db --- /dev/null +++ b/rust/crates/og-dev/src/evidence.rs @@ -0,0 +1,1140 @@ +//! `og dev evidence`: the measurements of `.github/workflows/public-evidence.yml` +//! on public data (GIAB's HG002), which were Python inside the workflow. +//! +//! - `og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv`: the +//! positions with two records or more, one of them an SNV, in the VCF as +//! GIAB writes it and split with `bcftools norm -m-`, on the chromosomes +//! of the archaic tables in DATA_DIR (`chr*.tsv.gz`), and how many rows of +//! those tables sit at such a position. OUT holds `raw_positions`, +//! `raw_on_table`, `split_positions` and `split_on_table`. +//! - `og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf`: +//! a gVCF of GIAB's genotypes plus a one-base reference block at every +//! panel site inside the benchmark regions that no GIAB record covers, +//! with the reference base from the FASTA (through its `.fai`). The +//! output is unsorted: the workflow sorts it with bcftools. +//! +//! Both stream their VCFs. hg002-gvcf keeps, per chromosome, the merged +//! spans the GIAB records cover instead of every covered position, and +//! reads one reference base per site with a positioned read. +//! +//! Exit 0 when written, 1 when an input cannot be read or parsed, 2 on a +//! usage error. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::os::unix::fs::{FileExt, PermissionsExt}; +use std::path::Path; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +const USAGE: &str = "usage: og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv\n og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf\n"; + +/// Runs `og dev evidence ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev evidence: error: {msg}"); + 2 + }; + let rest = args.get(1..).unwrap_or(&[]); + let result = match args.first().map(String::as_str) { + Some("clashes") if rest.len() == 4 => clashes(&rest[0], &rest[1], &rest[2], &rest[3]), + Some("hg002-gvcf") if rest.len() == 5 => { + hg002_gvcf(&rest[0], &rest[1], &rest[2], &rest[3], &rest[4]).map(|(records, blocks)| { + let _ = writeln!( + out, + "gVCF: {records} GIAB records, {blocks} reference blocks at panel sites" + ); + }) + } + Some("clashes" | "hg002-gvcf") => return usage(err, "wrong number of arguments"), + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown evidence '{other}'")), + None => return usage(err, "the following arguments are required: evidence"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev evidence: {e}"); + 1 + } + } +} + +fn open(path: &str, gzip: bool) -> Result, String> { + let mut file = BufReader::new(File::open(path).map_err(|e| format!("{path}: {e}"))?); + let head = file.fill_buf().map_err(|e| format!("{path}: {e}"))?; + if head.starts_with(&[0x1f, 0x8b]) != gzip { + return Err(format!( + "{path}: wrong compression (expected {})", + if gzip { "gzip" } else { "plain" } + )); + } + if gzip { + og_io::reader(file).map_err(|e| format!("{path}: {e}")) + } else { + Ok(Box::new(file)) + } +} + +/// Plain repository text with Python's universal-newline normalization. +pub(crate) fn read_text(path: &Path) -> Result { + use std::io::Read; + let mut text = String::new(); + open(&path.to_string_lossy(), false)? + .read_to_string(&mut text) + .map_err(|e| format!("{}: invalid text: {e}", path.display()))?; + Ok(text.replace("\r\n", "\n").replace('\r', "\n")) +} + +/// Strict UTF-8 and universal newlines, with a 16 MiB record limit. This +/// local reader can use og-io's bounded reader once that API is available. +pub(crate) fn lines(path: &str, f: impl FnMut(&[u8]) -> Result<(), String>) -> Result<(), String> { + input_lines(path, false, f) +} + +fn input_lines( + path: &str, + gzip: bool, + mut f: impl FnMut(&[u8]) -> Result<(), String>, +) -> Result<(), String> { + const MAX_LINE: usize = 16 << 20; + let mut r = open(path, gzip)?; + let mut line = Vec::new(); + let mut after_cr = false; + loop { + let chunk = r.fill_buf().map_err(|e| format!("{path}: {e}"))?; + if chunk.is_empty() { + if !line.is_empty() { + std::str::from_utf8(&line).map_err(|e| format!("{path}: invalid UTF-8: {e}"))?; + f(&line)?; + } + return Ok(()); + } + if after_cr { + after_cr = false; + if chunk[0] == b'\n' { + r.consume(1); + continue; + } + } + let end = chunk.iter().position(|b| matches!(b, b'\r' | b'\n')); + let n = end.unwrap_or(chunk.len()); + if line.len().checked_add(n).is_none_or(|len| len > MAX_LINE) { + return Err(format!("{path}: record exceeds {MAX_LINE} bytes")); + } + line.extend_from_slice(&chunk[..n]); + if end.is_some() { + after_cr = chunk[n] == b'\r'; + r.consume( + n.checked_add(1) + .ok_or_else(|| format!("{path}: record size overflows"))?, + ); + std::str::from_utf8(&line).map_err(|e| format!("{path}: invalid UTF-8: {e}"))?; + f(&line)?; + line.clear(); + } else { + r.consume(n); + } + } +} + +/// Follow the final symlink, including relative and dangling link targets. +fn resolve_destination(destination: &Path) -> Result { + let mut path = destination.to_owned(); + for _ in 0..40 { + match fs::read_link(&path) { + Ok(target) => { + path = if target.is_absolute() { + target + } else { + path.parent().unwrap_or(Path::new(".")).join(target) + } + } + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound + ) => + { + return Ok(path); + } + Err(e) => return Err(format!("{}: {e}", path.display())), + } + } + Err(format!( + "{}: too many destination symlinks", + destination.display() + )) +} + +fn validate_destination(path: &Path) -> Result, String> { + match fs::metadata(path) { + Ok(meta) => { + if !meta.is_file() { + return Err(format!( + "{}: destination is not a regular file", + path.display() + )); + } + let permissions = meta.permissions(); + if permissions.mode() & 0o222 == 0 { + return Err(format!("{}: read-only destination", path.display())); + } + // Check ACLs and effective-user access without truncating the old file. + File::options() + .write(true) + .open(path) + .map_err(|e| format!("{}: {e}", path.display()))?; + Ok(Some(permissions)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(format!("{}: {e}", path.display())), + } +} + +/// A unique, exclusively created sibling. Dropping it on any read, write +/// or rename error removes this temporary file. A multi-file set is staged +/// together, but its renames are separate transactions. +pub(crate) struct PendingOutput { + path: PathBuf, + destination: PathBuf, +} + +impl PendingOutput { + pub(crate) fn new(destination: &Path) -> Result<(Self, File), String> { + static NEXT: AtomicU64 = AtomicU64::new(0); + let destination = resolve_destination(destination)?; + let permissions = validate_destination(&destination)?; + let parent = destination + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let meta = fs::metadata(parent).map_err(|e| format!("{}: {e}", parent.display()))?; + if meta.permissions().mode() & 0o222 == 0 { + return Err(format!( + "{}: parent directory is not writable", + destination.display() + )); + } + let name = destination.file_name().ok_or("output has no file name")?; + loop { + let mut tmp = std::ffi::OsString::from("."); + tmp.push(name); + tmp.push(format!( + ".tmp-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let path = destination.with_file_name(tmp); + match File::options().write(true).create_new(true).open(&path) { + Ok(file) => { + let pending = Self { + path, + destination: destination.to_owned(), + }; + if let Some(permissions) = permissions { + file.set_permissions(permissions) + .map_err(|e| format!("{}: {e}", destination.display()))?; + } + return Ok((pending, file)); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(format!("{}: {e}", destination.display())), + } + } + } + + pub(crate) fn commit(self) -> Result<(), String> { + if let Some(permissions) = validate_destination(&self.destination)? { + // A write can clear set-user-ID or set-group-ID bits. Apply the + // final mode after every byte is staged, immediately before rename. + fs::set_permissions(&self.path, permissions) + .map_err(|e| format!("{}: {e}", self.destination.display()))?; + } + fs::rename(&self.path, &self.destination) + .map_err(|e| format!("{}: {e}", self.destination.display())) + } +} + +impl Drop for PendingOutput { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +pub(crate) fn publish(path: &Path, bytes: &[u8]) -> Result<(), String> { + let (pending, mut file) = PendingOutput::new(path)?; + file.write_all(bytes) + .and_then(|()| file.flush()) + .map_err(|e| format!("{}: {e}", path.display()))?; + drop(file); + pending.commit() +} + +fn pos(field: &[u8], path: &str) -> Result { + std::str::from_utf8(field) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + format!( + "{path}: position '{}' is not a number", + String::from_utf8_lossy(field) + ) + }) +} + +/// The positions of `path` on `chroms` with two records or more, one of +/// them an SNV. The VCF is sorted, so a position's records are adjacent. +fn clash_positions( + path: &str, + chroms: &BTreeSet>, +) -> Result, u64)>, String> { + let mut hit = BTreeSet::new(); + let (mut key, mut seen, mut multiple, mut snv): (Option<(Vec, u64)>, bool, bool, bool) = + (None, false, false, false); + input_lines(path, true, |line| { + if line.first() == Some(&b'#') { + return Ok(()); + } + let c: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if !chroms.contains(c[0]) { + return Ok(()); + } + if c.len() < 6 { + return Err(format!( + "{path}: a selected record has fewer than 5 columns or missing trailing VCF columns" + )); + } + let k = (c[0].to_vec(), pos(c[1], path)?); + if key.as_ref() != Some(&k) { + key = Some(k.clone()); + (seen, multiple, snv) = (false, false, false); + } + multiple |= seen; + seen = true; + let one = |a: &[u8]| std::str::from_utf8(a).map_or(a.len(), |s| s.chars().count()) == 1; + snv |= one(c[3]) && one(c[4]); + if multiple && snv { + hit.insert(k); + } + Ok(()) + })?; + Ok(hit) +} + +fn clashes(raw: &str, split: &str, data: &str, out: &str) -> Result<(), String> { + let mut tables = BTreeMap::new(); + for e in fs::read_dir(data).map_err(|e| format!("{data}: {e}"))? { + let name = e + .map_err(|e| format!("{data}: {e}"))? + .file_name() + .into_string() + .unwrap_or_default(); + if let Some(chrom) = name + .strip_suffix(".tsv.gz") + .filter(|c| c.starts_with("chr")) + { + tables.insert(chrom.as_bytes().to_vec(), Path::new(data).join(&name)); + } + } + let chroms: BTreeSet> = tables.keys().cloned().collect(); + let sets = [ + ("raw", clash_positions(raw, &chroms)?), + ("split", clash_positions(split, &chroms)?), + ]; + let mut on_table = [0u64; 2]; + for (chrom, table) in &tables { + let want: Vec> = sets + .iter() + .map(|(_, s)| { + s.iter() + .filter(|(c, _)| c == chrom) + .map(|(_, p)| *p) + .collect() + }) + .collect(); + if want.iter().all(BTreeSet::is_empty) { + continue; + } + let path = table.to_string_lossy().into_owned(); + input_lines(&path, true, |line| { + let c: Vec<&[u8]> = line.splitn(3, |&b| b == b'\t').collect(); + if c[0] != chrom.as_slice() { + return Ok(()); + } + if c.len() < 2 { + return Err(format!("{path}: a selected row has fewer than 2 columns")); + } + let p = pos(c[1], &path)?; + for (k, w) in want.iter().enumerate() { + let count = on_table + .get_mut(k) + .ok_or("table counter index out of range")?; + *count = count + .checked_add(u64::from(w.contains(&p))) + .ok_or_else(|| format!("{path}: table count overflows"))?; + } + Ok(()) + })?; + } + let mut text = String::new(); + for (k, (name, s)) in sets.iter().enumerate() { + text += &format!( + "{name}_positions\t{}\n{name}_on_table\t{}\n", + s.len(), + on_table.get(k).ok_or("table counter index out of range")? + ); + } + publish(Path::new(out), text.as_bytes()) +} + +/// One `.fai` row: length, byte offset, bases per line, bytes per line. +struct Fai { + len: u64, + offset: u64, + line_bases: u64, + line_bytes: u64, +} + +fn read_fai(path: &str) -> Result, Fai>, String> { + let mut fai = HashMap::new(); + lines(path, |line| { + let f: Vec<&[u8]> = line.split(|&b| b == b'\t').collect(); + let n = |i: usize| { + f.get(i) + .ok_or_else(|| format!("{path}: a row has fewer than 5 columns")) + .and_then(|x| pos(x, path)) + }; + let row = Fai { + len: n(1)?, + offset: n(2)?, + line_bases: n(3)?, + line_bytes: n(4)?, + }; + if row.line_bases == 0 { + return Err(format!("{path}: a row has 0 bases per line")); + } + fai.insert(f[0].to_vec(), row); + Ok(()) + })?; + Ok(fai) +} + +/// Sorted, merged half-open spans `[start, end)` of 1-based positions. +fn merge(mut spans: Vec<(u64, u64)>) -> Vec<(u64, u64)> { + spans.sort_unstable(); + let mut out: Vec<(u64, u64)> = Vec::with_capacity(spans.len()); + for (s, e) in spans { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + out +} + +/// Is `p` inside one of the sorted, disjoint spans? +fn inside(spans: &[(u64, u64)], p: u64) -> bool { + let i = spans.partition_point(|&(s, _)| s <= p); + i.checked_sub(1) + .and_then(|j| spans.get(j)) + .is_some_and(|&(_, end)| p < end) +} + +fn hg002_gvcf( + truth: &str, + bed: &str, + reference: &str, + sites: &str, + out: &str, +) -> Result<(u64, u64), String> { + let fai = read_fai(&format!("{reference}.fai"))?; + let fasta = File::open(reference).map_err(|e| format!("{reference}: {e}"))?; + let fasta_len = fasta + .metadata() + .map_err(|e| format!("{reference}: {e}"))? + .len(); + let mut magic = [0; 2]; + if fasta + .read_at(&mut magic, 0) + .map_err(|e| format!("{reference}: {e}"))? + == 2 + && magic == [0x1f, 0x8b] + { + return Err(format!("{reference}: wrong compression (expected plain)")); + } + // The benchmark regions, BED 0-based: (start, end) per chromosome, + // sorted. A site p (1-based) is confident when the region with the + // largest start below p ends at p or later, as the Python's bisect did. + let mut conf: HashMap, Vec<(u64, u64)>> = HashMap::new(); + lines(bed, |line| { + let f: Vec<&[u8]> = line + .split(|b| matches!(b, b' ' | b'\t')) + .filter(|x| !x.is_empty()) + .collect(); + if f.iter().any(|field| { + std::str::from_utf8(field) + .expect("strict reader") + .chars() + .any(|ch| ch.is_whitespace() || matches!(ch, '\u{1c}'..='\u{1f}')) + }) { + return Err(format!("{bed}: unsupported whitespace in a field")); + } + if f.len() < 3 { + return Err(format!("{bed}: a row has fewer than 3 columns")); + } + conf.entry(f[0].to_vec()) + .or_default() + .push((pos(f[1], bed)?, pos(f[2], bed)?)); + Ok(()) + })?; + for v in conf.values_mut() { + v.sort_unstable(); + } + let confident = |c: &[u8], p: u64| { + conf.get(c).is_some_and(|v| { + let i = v.partition_point(|&(s, _)| s < p); + i.checked_sub(1) + .and_then(|j| v.get(j)) + .is_some_and(|&(_, end)| end >= p) + }) + }; + + let (pending, file) = PendingOutput::new(Path::new(out))?; + let mut w = BufWriter::with_capacity(1 << 20, file); + let werr = |e: std::io::Error| format!("{out}: {e}"); + let mut covered: HashMap, Vec<(u64, u64)>> = HashMap::new(); + let mut records = 0u64; + input_lines(truth, true, |line| { + if line.first() == Some(&b'#') && records != 0 { + return Err(format!("{truth}: VCF header after first data record")); + } + if line.starts_with(b"##") { + } else if line.first() == Some(&b'#') { + w.write_all(b"##INFO=\n##ALT=\n") + .map_err(werr)?; + } else { + let f: Vec<&[u8]> = line.splitn(6, |&b| b == b'\t').collect(); + if f.len() < 5 { + return Err(format!("{truth}: a record has fewer than 5 columns")); + } + let p = pos(f[1], truth)?; + covered.entry(f[0].to_vec()).or_default().push(( + p, + p.checked_add( + std::str::from_utf8(f[3]) + .expect("strict reader") + .chars() + .count() + .try_into() + .map_err(|_| format!("{truth}: allele length overflows"))?, + ) + .ok_or_else(|| format!("{truth}: position span overflows"))?, + )); + records = records + .checked_add(1) + .ok_or_else(|| format!("{truth}: record count overflows"))?; + } + w.write_all(line) + .and_then(|()| w.write_all(b"\n")) + .map_err(werr) + })?; + let covered: HashMap, Vec<(u64, u64)>> = + covered.into_iter().map(|(c, v)| (c, merge(v))).collect(); + + let mut blocks = 0u64; + let mut first = true; + let mut base = [0u8; 1]; + lines(sites, |line| { + if std::mem::take(&mut first) { + return Ok(()); + } + let f: Vec<&[u8]> = line.splitn(4, |&b| b == b'\t').collect(); + if f.len() < 3 { + return Err(format!("{sites}: a row has fewer than 3 columns")); + } + let (c, p) = (f[1], pos(f[2], sites)?); + let Some(row) = fai.get(c) else { return Ok(()) }; + if p == 0 || p > row.len || covered.get(c).is_some_and(|v| inside(v, p)) || !confident(c, p) + { + return Ok(()); + } + let invalid = || format!("{reference}: invalid FASTA offset: overflow or outside file"); + let zero = p.checked_sub(1).ok_or_else(invalid)?; + let at = zero + .checked_div(row.line_bases) + .and_then(|line| line.checked_mul(row.line_bytes)) + .and_then(|relative| row.offset.checked_add(relative)) + .and_then(|start| { + zero.checked_rem(row.line_bases) + .and_then(|col| start.checked_add(col)) + }) + .ok_or_else(invalid)?; + if at > i64::MAX as u64 || at.checked_add(1).is_none_or(|end| end > fasta_len) { + return Err(invalid()); + } + fasta + .read_exact_at(&mut base, at) + .map_err(|e| format!("{reference}: {e}"))?; + std::str::from_utf8(&base).map_err(|e| format!("{reference}: invalid UTF-8: {e}"))?; + let b = base[0].to_ascii_uppercase(); + if !b"ACGT".contains(&b) { + return Ok(()); + } + blocks = blocks + .checked_add(1) + .ok_or_else(|| format!("{sites}: block count overflows"))?; + w.write_all(c) + .and_then(|()| { + writeln!( + w, + "\t{p}\t.\t{}\t<*>\t.\t.\tEND={p}\tGT\t0/0", + char::from(b) + ) + }) + .map_err(werr) + })?; + if first { + return Err(format!("{sites}: missing header")); + } + let file = w.into_inner().map_err(|e| werr(e.into_error()))?; + drop(file); + pending.commit()?; + Ok((records, blocks)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hg002_gvcf_keeps_truth_and_adds_only_valid_reference_blocks() { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let inputs = repo.join("tests/parity/inputs/dev-evidence-hg002-gvcf"); + let temp = std::env::temp_dir().join(format!( + "og-evidence-gvcf-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&temp).unwrap(); + let output = temp.join("out.vcf"); + let args = [ + "hg002-gvcf".to_owned(), + inputs.join("truth.vcf.gz").to_str().unwrap().to_owned(), + inputs + .join("truth-past-end.bed") + .to_str() + .unwrap() + .to_owned(), + inputs.join("ref.fasta").to_str().unwrap().to_owned(), + inputs + .join("sites-past-end.tsv") + .to_str() + .unwrap() + .to_owned(), + output.to_str().unwrap().to_owned(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 0, "{err:?}"); + assert_eq!( + out, + b"gVCF: 4 GIAB records, 1 reference blocks at panel sites\n" + ); + assert!(err.is_empty()); + let actual = fs::read(&output).unwrap(); + fs::remove_dir_all(&temp).unwrap(); + assert_eq!( + actual, + include_bytes!( + "../../../../tests/parity/golden/dev-evidence-hg002-gvcf/past-the-end/out.vcf" + ) + ); + } + + struct Case { + dir: std::path::PathBuf, + kind: &'static str, + } + + impl Case { + fn new(kind: &'static str, label: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "og-evidence-{kind}-{label}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/inputs") + .join(format!("dev-evidence-{kind}")); + for e in fs::read_dir(src).unwrap() { + let e = e.unwrap(); + fs::copy(e.path(), dir.join(e.file_name())).unwrap(); + } + Self { dir, kind } + } + fn text(&self, name: &str) -> Vec { + use std::io::Read; + let mut bytes = Vec::new(); + open(self.dir.join(name).to_str().unwrap(), name.ends_with(".gz")) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + bytes + } + fn put(&self, name: &str, bytes: &[u8]) { + if name.ends_with(".gz") { + let mut gz = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(bytes).unwrap(); + fs::write(self.dir.join(name), gz.finish().unwrap()).unwrap(); + } else { + fs::write(self.dir.join(name), bytes).unwrap(); + } + } + fn args(&self) -> Vec { + let p = |n: &str| self.dir.join(n).to_str().unwrap().to_owned(); + if self.kind == "clashes" { + vec![ + "clashes".to_owned(), + p("raw.vcf.gz"), + p("split.vcf.gz"), + self.dir.to_str().unwrap().to_owned(), + p("out"), + ] + } else { + vec![ + "hg002-gvcf".to_owned(), + p("truth.vcf.gz"), + p("truth.bed"), + p("ref.fasta"), + p("sites.tsv"), + p("out"), + ] + } + } + fn run(&self) -> Result<(), String> { + let (mut out, mut err) = (Vec::new(), Vec::new()); + match cli(&self.args(), &mut out, &mut err) { + 0 => Ok(()), + 1 => Err(String::from_utf8(err).unwrap()), + code => panic!("unexpected CLI exit {code}: {err:?}"), + } + } + fn fails_without_publication(&self) { + fs::write(self.dir.join("out"), b"PREVIOUS OUTPUT\n").unwrap(); + assert!(self.run().is_err(), "{} must fail", self.dir.display()); + assert_eq!( + fs::read(self.dir.join("out")).unwrap(), + b"PREVIOUS OUTPUT\n" + ); + let leftovers: Vec<_> = fs::read_dir(&self.dir) + .unwrap() + .filter_map(|e| { + let name = e.unwrap().file_name(); + name.to_str() + .filter(|n| n.starts_with(".out.")) + .map(str::to_owned) + }) + .collect(); + assert!(leftovers.is_empty(), "{leftovers:?}"); + } + } + impl Drop for Case { + fn drop(&mut self) { + fs::remove_dir_all(&self.dir).unwrap(); + } + } + + #[test] + fn evidence_status_uses_checked_runner() { + for kind in ["clashes", "hg002-gvcf"] { + let c = Case::new(kind, "streams"); + crate::stream_tests::assert_status_errors(cli, &c.args(), kind == "hg002-gvcf"); + } + } + + #[test] + fn fai_offsets_reject_overflow_and_out_of_file_before_publication() { + for (offset, bases, bytes, position) in [ + (18446744073709551598u64, 1, 2, 13), + (1, 1, u64::MAX, 3), + (u64::MAX, 1, 1, 1), + (u64::MAX - 1, 50, 51, 3), + (1000, 50, 51, 1), + ] { + let c = Case::new("hg002-gvcf", &format!("offset-{offset}-{position}")); + c.put( + "ref.fasta.fai", + format!("chrA\t50\t{offset}\t{bases}\t{bytes}\n").as_bytes(), + ); + c.put( + "sites.tsv", + format!("id\tchr\tpos\nx\tchrA\t{position}\n").as_bytes(), + ); + c.put("truth.bed", b"chrA\t0\t50\n"); + c.put( + "truth.vcf.gz", + b"##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\n", + ); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains(": invalid FASTA offset:")); + } + } + + #[test] + fn late_vcf_headers_fail_before_publication() { + for header in [b"##late=yes\n".as_slice(), b"#CHROM\tPOS\n"] { + let c = Case::new("hg002-gvcf", &format!("late-{}", header.len())); + c.put( + "truth.vcf.gz", + &[c.text("truth.vcf.gz"), header.to_vec()].concat(), + ); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": VCF header after first data record") + ); + } + } + + #[test] + fn empty_gzip_inputs_and_unused_zero_width_fai_rows_fail() { + for (kind, name) in [ + ("clashes", "raw.vcf.gz"), + ("clashes", "split.vcf.gz"), + ("clashes", "chrA.tsv.gz"), + ("hg002-gvcf", "truth.vcf.gz"), + ] { + let c = Case::new(kind, name); + fs::write(c.dir.join(name), b"").unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected gzip)") + ); + } + let c = Case::new("hg002-gvcf", "unused-row"); + c.put( + "ref.fasta.fai", + &[c.text("ref.fasta.fai"), b"chrUnused\t0\t0\t0\t0\n".to_vec()].concat(), + ); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": a row has 0 bases per line") + ); + } + + #[test] + fn gzip_fasta_with_ascii_fname_is_refused_by_format_guard() { + let c = Case::new("hg002-gvcf", "fname"); + let mut gz = flate2::GzBuilder::new() + .filename(vec![b'A'; 200]) + .write(Vec::new(), flate2::Compression::default()); + gz.write_all(&c.text("ref.fasta")).unwrap(); + fs::write(c.dir.join("ref.fasta"), gz.finish().unwrap()).unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected plain)") + ); + } + + #[test] + fn non_writable_parents_preserve_evidence_destinations() { + for kind in ["clashes", "hg002-gvcf"] { + let c = Case::new(kind, "parent-mode"); + fs::write(c.dir.join("out"), b"PREVIOUS OUTPUT\n").unwrap(); + fs::set_permissions(&c.dir, fs::Permissions::from_mode(0o555)).unwrap(); + let result = c.run(); + fs::set_permissions(&c.dir, fs::Permissions::from_mode(0o755)).unwrap(); + assert!( + result + .unwrap_err() + .contains(": parent directory is not writable") + ); + assert_eq!(fs::read(c.dir.join("out")).unwrap(), b"PREVIOUS OUTPUT\n"); + } + } + + #[test] + fn selected_truncated_clash_records_fail() { + for name in ["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"] { + let c = Case::new("clashes", name); + c.put( + name, + if name == "chrA.tsv.gz" { + b"chrA" + } else { + b"chrA\t3\n" + }, + ); + c.fails_without_publication(); + } + } + + #[test] + fn missing_panel_header_and_blank_bed_fail() { + for name in ["sites.tsv", "truth.bed"] { + let c = Case::new("hg002-gvcf", name); + let bytes = if name == "sites.tsv" { + vec![] + } else { + [c.text(name), b"\n".to_vec()].concat() + }; + c.put(name, &bytes); + c.fails_without_publication(); + } + } + + #[test] + fn every_evidence_text_input_rejects_invalid_utf8() { + for (kind, names) in [ + ("clashes", vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"]), + ( + "hg002-gvcf", + vec!["truth.vcf.gz", "truth.bed", "ref.fasta.fai", "sites.tsv"], + ), + ] { + for name in names { + let c = Case::new(kind, name); + // Invalid bytes in a header or filtered-out record must still fail. + let mut bytes = c.text(name); + bytes.extend_from_slice(b"#ignored\xff\n"); + c.put(name, &bytes); + c.fails_without_publication(); + } + } + } + + #[test] + fn every_evidence_text_input_has_universal_newlines() { + for (kind, names) in [ + ( + "clashes", + vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz", "chrB.tsv.gz"], + ), + ( + "hg002-gvcf", + vec!["truth.vcf.gz", "truth.bed", "ref.fasta.fai", "sites.tsv"], + ), + ] { + for name in names { + for ending in [b"\r".as_slice(), b"\r\n".as_slice()] { + let c = Case::new(kind, &format!("{name}-{}", ending.len())); + c.run().unwrap(); + let expected = fs::read(c.dir.join("out")).unwrap(); + let bytes = c + .text(name) + .split(|b| *b == b'\n') + .collect::>() + .join(ending); + c.put(name, &bytes); + c.run().unwrap(); + assert_eq!( + fs::read(c.dir.join("out")).unwrap(), + expected, + "{kind}/{name}/{ending:?}" + ); + } + } + } + } + + #[test] + fn failed_gvcf_inputs_preserve_existing_output() { + for name in ["truth.vcf.gz", "sites.tsv"] { + for fault in ["missing", "directory", "malformed", "truncated"] { + let c = Case::new("hg002-gvcf", &format!("{name}-{fault}")); + let path = c.dir.join(name); + match fault { + "missing" => fs::remove_file(path).unwrap(), + "directory" => { + fs::remove_file(&path).unwrap(); + fs::create_dir(path).unwrap(); + } + "malformed" => { + let mut bytes = c.text(name); + bytes.extend_from_slice(b"bad\n"); + c.put(name, &bytes); + } + _ if name.ends_with(".gz") => { + let mut bytes = fs::read(&path).unwrap(); + bytes.truncate(bytes.len() - 5); + fs::write(path, bytes).unwrap(); + } + _ => c.put(name, b"header\nbad\n"), + } + c.fails_without_publication(); + } + } + } + + #[test] + fn explicit_compression_is_required_at_every_input() { + for (kind, names) in [ + ("clashes", vec!["raw.vcf.gz", "split.vcf.gz", "chrA.tsv.gz"]), + ( + "hg002-gvcf", + vec![ + "truth.vcf.gz", + "truth.bed", + "ref.fasta.fai", + "ref.fasta", + "sites.tsv", + ], + ), + ] { + for name in names { + let c = Case::new(kind, &format!("compression-{name}")); + let bytes = c.text(name); + let wrong = if name.ends_with(".gz") { + bytes + } else { + let mut gz = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(&bytes).unwrap(); + gz.finish().unwrap() + }; + fs::write(c.dir.join(name), wrong).unwrap(); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains(": wrong compression (expected "), + "{name}" + ); + } + } + } + + #[test] + fn bed_rejects_python_whitespace_inside_fields_and_as_separators() { + for ch in [ + '\u{2028}', '\u{85}', '\u{b}', '\u{a0}', '\u{1c}', '\u{1d}', '\u{1e}', '\u{1f}', + ] { + for embedded in [true, false] { + let c = Case::new("hg002-gvcf", &format!("space-{}-{embedded}", ch as u32)); + let text = String::from_utf8(c.text("truth.bed")).unwrap(); + let text = if embedded { + text.replacen("chrA", &format!("chr{ch}A"), 1) + } else { + text.replace('\t', &ch.to_string()) + }; + c.put("truth.bed", text.as_bytes()); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("unsupported whitespace")); + } + } + } + + #[test] + fn publication_preserves_permissions_and_follows_symlinks() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let c = Case::new("clashes", "publication"); + let target = c.dir.join("target"); + let link = c.dir.join("link"); + fs::write(&target, b"old").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o600)).unwrap(); + symlink("target", &link).unwrap(); + publish(&link, b"new").unwrap(); + assert!( + fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(fs::read(&target).unwrap(), b"new"); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o600 + ); + fs::set_permissions(&target, fs::Permissions::from_mode(0o4600)).unwrap(); + publish(&link, b"new").unwrap(); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o7777, + 0o4600 + ); + fs::set_permissions(&target, fs::Permissions::from_mode(0o444)).unwrap(); + assert!(publish(&link, b"bad").unwrap_err().contains("read-only")); + assert_eq!(fs::read(&target).unwrap(), b"new"); + let dir_link = c.dir.join("dir-link"); + symlink(&c.dir, &dir_link).unwrap(); + for path in [&c.dir, &dir_link] { + assert!( + publish(path, b"bad") + .unwrap_err() + .contains("not a regular file") + ); + } + } + + #[test] + fn approved_strict_evidence_shapes_fail_with_stable_errors() { + for value in ["3_0", "٣", "-3", "18446744073709551616"] { + let c = Case::new("hg002-gvcf", value); + c.put( + "truth.vcf.gz", + format!("chrA\t{value}\t.\tA\tC\n").as_bytes(), + ); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("not a number")); + } + let c = Case::new("hg002-gvcf", "short-vcf"); + c.put("truth.vcf.gz", b"chrA\t3\t.\tA\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("fewer than 5 columns")); + let c = Case::new("hg002-gvcf", "blank-fai"); + c.put("ref.fasta.fai", b"\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("fewer than 5 columns")); + let c = Case::new("clashes", "newline-alt"); + c.put("raw.vcf.gz", b"chrA\t3\t.\tA\tC\nchrA\t3\t.\tAA\tA\n"); + c.fails_without_publication(); + assert!(c.run().unwrap_err().contains("VCF columns")); + } + + #[test] + fn approved_record_limit_and_final_lf_are_pinned() { + let c = Case::new("clashes", "record-limit"); + c.put("raw.vcf.gz", &vec![b'#'; (16 << 20) + 1]); + c.fails_without_publication(); + assert!( + c.run() + .unwrap_err() + .contains("record exceeds 16777216 bytes") + ); + let c = Case::new("hg002-gvcf", "final-lf"); + c.run().unwrap(); + let expected = fs::read(c.dir.join("out")).unwrap(); + let mut truth = c.text("truth.vcf.gz"); + assert_eq!(truth.pop(), Some(b'\n')); + c.put("truth.vcf.gz", &truth); + c.run().unwrap(); + let result = fs::read(c.dir.join("out")).unwrap(); + assert_eq!(result, expected); + assert_eq!(result.last(), Some(&b'\n')); + } + + #[test] + fn spans_merge_and_answer_inside() { + let m = merge(vec![(10, 12), (5, 6), (11, 15), (15, 16)]); + assert_eq!(m, [(5, 6), (10, 16)]); + assert!( + inside(&m, 5) && !inside(&m, 6) && inside(&m, 15) && !inside(&m, 16) && !inside(&m, 4) + ); + } +} diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs new file mode 100644 index 000000000..ec74f8dd3 --- /dev/null +++ b/rust/crates/og-dev/src/fixture.rs @@ -0,0 +1,930 @@ +//! `og dev fixture`: writers of committed or generated test fixtures. +//! +//! - `og dev fixture comphet OUT_DIR [--swap]`: the compound-het phasing +//! fixture of `tests/e2e/comphet-phasing-1-reads.sh` (made-up reads on a +//! made-up contig). It replaces `tests/fixtures/comphet/make_fixture.py` +//! and writes the same bytes: the contig is Python's `random.Random(31)` +//! choosing among `ACGT`, so [`Mt19937`] is CPython's generator. +//! - `og dev fixture trim-pharmcat REPORT OUT [MAX_ANNOTATIONS]`: a PharmCAT +//! `report.json` cut down to the fields `og pgx` reads. It replaces +//! `tests/fixtures/pharmcat/trim_report.py` and writes the same bytes. +//! +//! Exit 0 when written, 1 when an input cannot be read or has the wrong +//! shape, 2 on a usage error. + +use crate::py_os_error; +use indexmap::IndexMap; +use pycompat::json::{self, Dump, Value}; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::Path; + +const USAGE: &str = "usage: og dev fixture comphet OUT_DIR [--swap]\n og dev fixture trim-pharmcat REPORT.json OUT.json [MAX_ANNOTATIONS]\n"; + +/// Runs `og dev fixture ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let usage = |err: &mut dyn Write, msg: &str| { + let _ = writeln!(err, "{USAGE}og dev fixture: error: {msg}"); + 2 + }; + let result = match args.first().map(String::as_str) { + Some("comphet") => { + let rest: Vec<&String> = args[1..].iter().filter(|a| *a != "--swap").collect(); + if rest.len() != 1 { + return usage(err, "comphet takes one OUT_DIR and an optional --swap"); + } + comphet::write(Path::new(rest[0]), args[1..].iter().any(|a| a == "--swap")) + } + Some("trim-pharmcat") => { + let rest = &args[1..]; + if !(2..=3).contains(&rest.len()) { + return usage( + err, + "trim-pharmcat takes REPORT.json OUT.json [MAX_ANNOTATIONS]", + ); + } + let max = match rest.get(2).map(|m| m.trim().parse::()) { + None => 4, + Some(Ok(n)) => n, + Some(Err(_)) => { + return usage( + err, + &format!("MAX_ANNOTATIONS '{}' is not a whole number", rest[2]), + ); + } + }; + trim_pharmcat(Path::new(&rest[0]), Path::new(&rest[1]), max) + } + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => return usage(err, &format!("unknown fixture '{other}'")), + None => return usage(err, "the following arguments are required: fixture"), + }; + match result { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev fixture: {e}"); + 1 + } + } +} + +/// CPython's Mersenne Twister (`random.Random`), only as far as +/// `Random(int).choice(seq)` needs it. +pub struct Mt19937 { + state: [u32; 624], + index: usize, +} + +impl Mt19937 { + /// `random.Random(seed)` for a seed below 2^32: `init_by_array([seed])`. + pub fn new(seed: u32) -> Self { + let mut mt = Mt19937 { + state: [0; 624], + index: 624, + }; + mt.init_genrand(19_650_218); + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624.max(key.len()) { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + let prev = mt.state[i - 1] ^ (mt.state[i - 1] >> 30); + mt.state[i] = (mt.state[i] ^ prev.wrapping_mul(1_566_083_941)).wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt.state[0] = mt.state[623]; + i = 1; + } + } + mt.state[0] = 0x8000_0000; + mt + } + + fn init_genrand(&mut self, s: u32) { + self.state[0] = s; + for i in 1..624 { + let prev = self.state[i - 1] ^ (self.state[i - 1] >> 30); + self.state[i] = 1_812_433_253u32.wrapping_mul(prev).wrapping_add(i as u32); + } + self.index = 624; + } + + /// `genrand_uint32`. + pub fn next_u32(&mut self) -> u32 { + if self.index >= 624 { + for k in 0..624 { + let y = (self.state[k] & 0x8000_0000) | (self.state[(k + 1) % 624] & 0x7fff_ffff); + let mag = if y & 1 == 1 { 0x9908_b0df } else { 0 }; + self.state[k] = self.state[(k + 397) % 624] ^ (y >> 1) ^ mag; + } + self.index = 0; + } + let mut y = self.state[self.index]; + self.index += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + /// `_randbelow(n)` for `0 < n < 2^32`: rejection sampling on + /// `getrandbits(n.bit_length())`. + pub fn below(&mut self, n: u32) -> u32 { + let k = 32 - n.leading_zeros(); + loop { + let r = self.next_u32() >> (32 - k); + if r < n { + return r; + } + } + } +} + +pub mod comphet { + //! The compound-het phasing fixture: sample CHPH on contig chrP (80 kb + //! of seeded random sequence), read pairs of 100 bp from 500 bp + //! fragments, and three pairs of heterozygous variants: + //! + //! - GENEC: SNV at 1000 and a 2-base deletion at 1003, both on copy 1 + //! (cis); single reads cover both. `--swap` moves the deletion to + //! copy 2 (trans). + //! - GENET: SNVs at 5000 and 5400, one on each copy (trans); only a read + //! and its mate cover both. + //! - GENEF: SNVs at 20000 and 70000, one on each copy, 50 kb apart: no + //! read pair covers both (unknown). + //! + //! Files: `reads.sam`, `vep.vcf` (with a VEP CSQ, for step 31) and + //! `comphet.vcf` (the pairs as slivar names them, for `og comphet + //! phase`). + + use super::Mt19937; + use std::fmt::Write as _; + use std::fs; + use std::path::Path; + + pub const SAMPLE: &str = "CHPH"; + pub const CONTIG: &str = "chrP"; + pub const LENGTH: usize = 80_000; + const READ: usize = 100; + const FRAG: usize = 500; + const FIELDS: &str = "Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|Existing_variation|MAX_AF|gnomADe_AF|gnomADg_AF|CLIN_SIG"; + + /// One variant: 1-based position, alleles, consequence and the copy + /// (0 or 1) that carries it. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct Variant { + pub gene: &'static str, + pub pos: usize, + pub alt_ref: String, + pub alt: String, + pub consequence: &'static str, + pub copy: u8, + } + + /// The contig: `random.Random(31)` choosing among `ACGT` 80,000 times. + pub fn reference() -> Vec { + let mut rng = Mt19937::new(31); + (0..LENGTH) + .map(|_| b"ACGT"[rng.below(4) as usize]) + .collect() + } + + /// The base before `b` in `ACGT`, wrapping (`A` gives `T`). + fn other(b: u8) -> u8 { + let i = b"ACGT" + .iter() + .position(|&x| x == b) + .expect("a base of ACGT"); + b"ACGT"[(i + 3) % 4] + } + + pub fn variants(seq: &[u8], swap: bool) -> Vec { + let s = |a: usize, b: usize| String::from_utf8_lossy(&seq[a..b]).into_owned(); + let snv = |gene, pos: usize, consequence, copy| Variant { + gene, + pos, + alt_ref: s(pos - 1, pos), + alt: (other(seq[pos - 1]) as char).to_string(), + consequence, + copy, + }; + vec![ + snv("GENEC", 1000, "stop_gained", 0), + Variant { + gene: "GENEC", + pos: 1003, + alt_ref: s(1002, 1005), + alt: s(1002, 1003), + consequence: "frameshift_variant", + copy: u8::from(swap), + }, + snv("GENET", 5000, "stop_gained", 0), + snv("GENET", 5400, "stop_gained", 1), + snv("GENEF", 20000, "stop_gained", 0), + snv("GENEF", 70000, "stop_gained", 1), + ] + } + + /// The sequence and CIGAR of a read of `READ` bases from 0-based + /// `start` on one copy. + fn read_on(seq: &[u8], vs: &[Variant], copy: u8, start: usize) -> (String, String) { + let mine = || vs.iter().filter(|v| v.copy == copy); + let snv_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() == v.alt.len() && v.pos - 1 == p) + .map(|v| v.alt.as_bytes()[0]) + }; + // A deletion is keyed on the first deleted base. + let del_at = |p: usize| { + mine() + .find(|v| v.alt_ref.len() > v.alt.len() && v.pos + v.alt.len() - 1 == p) + .map(|v| v.alt_ref.len() - v.alt.len()) + }; + let (mut out, mut ops, mut p) = + (Vec::with_capacity(READ), Vec::<(u8, usize)>::new(), start); + while out.len() < READ { + if let Some(n) = del_at(p) { + ops.push((b'D', n)); + p += n; + continue; + } + out.push(snv_at(p).unwrap_or(seq[p])); + ops.push((b'M', 1)); + p += 1; + } + let mut cigar = String::new(); + let mut i = 0; + while i < ops.len() { + let (op, mut n) = ops[i]; + let mut j = i + 1; + while j < ops.len() && ops[j].0 == op { + n += ops[j].1; + j += 1; + } + let _ = write!(cigar, "{n}{}", op as char); + i = j; + } + (String::from_utf8_lossy(&out).into_owned(), cigar) + } + + pub fn sam(seq: &[u8], vs: &[Variant]) -> String { + let mut s = format!( + "@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:{CONTIG}\tLN:{LENGTH}\n@RG\tID:{SAMPLE}\tSM:{SAMPLE}\n" + ); + // Fragment starts (0-based) around each pair: read 1 covers the + // first position, its mate the second one when 400 bp apart. + let starts: [(&str, std::iter::StepBy>); 4] = [ + ("c", (920..991).step_by(7)), + ("t", (4905..4996).step_by(10)), + ("f", (19905..19996).step_by(10)), + ("g", (69905..69996).step_by(10)), + ]; + let q = "I".repeat(READ); + for (region, rs) in starts { + for copy in 0..2u8 { + for st in rs.clone() { + let m = st + FRAG - READ; + let (s1, c1) = read_on(seq, vs, copy, st); + let (s2, c2) = read_on(seq, vs, copy, m); + let name = format!("{region}{copy}_{st}"); + let _ = writeln!( + s, + "{name}\t99\t{CONTIG}\t{}\t60\t{c1}\t=\t{}\t{FRAG}\t{s1}\t{q}\tRG:Z:{SAMPLE}", + st + 1, + m + 1 + ); + let _ = writeln!( + s, + "{name}\t147\t{CONTIG}\t{}\t60\t{c2}\t=\t{}\t-{FRAG}\t{s2}\t{q}\tRG:Z:{SAMPLE}", + m + 1, + st + 1 + ); + } + } + } + s + } + + fn vcf_head(info: &str) -> String { + format!( + "##fileformat=VCFv4.2\n##contig=\n##FILTER=\n{info}\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n" + ) + } + + pub fn vep_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head(&format!( + "##INFO=\n" + )); + for v in vs { + let (g, p) = (v.gene, v.pos); + let _ = writeln!( + out, + "{CONTIG}\t{p}\t.\t{}\t{}\t50\tPASS\tCSQ={}|{}|HIGH|{g}|ENSG{g}|Transcript|ENST{p}|protein_coding|YES|NM_{p}.1|rs{p}|0.0001|0.0001|0.0001|\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.alt_ref, v.alt, v.alt, v.consequence + ); + } + out + } + + pub fn comphet_vcf(vs: &[Variant]) -> String { + let mut out = vcf_head( + "##INFO=\n", + ); + for (i, v) in vs.iter().enumerate() { + let mate = if i % 2 == 0 { &vs[i + 1] } else { &vs[i - 1] }; + let pid = match v.gene { + "GENEC" => 1, + "GENET" => 2, + _ => 3, + }; + let _ = writeln!( + out, + "{CONTIG}\t{}\t.\t{}\t{}\t50\tPASS\tslivar_comphet={SAMPLE}/{}/{pid}/{CONTIG}/{}/{}/{}\tGT:GQ:DP:AD\t0/1:50:30:15,15", + v.pos, v.alt_ref, v.alt, v.gene, mate.pos, mate.alt_ref, mate.alt + ); + } + out + } + + /// Writes the three files into `dir`, made if missing. + pub fn write(dir: &Path, swap: bool) -> Result<(), String> { + let seq = reference(); + let vs = variants(&seq, swap); + fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let mut staged = Vec::new(); + for (name, text) in [ + ("reads.sam", sam(&seq, &vs)), + ("vep.vcf", vep_vcf(&vs)), + ("comphet.vcf", comphet_vcf(&vs)), + ] { + let path = dir.join(name); + let (pending, mut file) = crate::evidence::PendingOutput::new(&path)?; + std::io::Write::write_all(&mut file, text.as_bytes()) + .map_err(|e| format!("{}: {e}", path.display()))?; + std::io::Write::flush(&mut file).map_err(|e| format!("{}: {e}", path.display()))?; + drop(file); + staged.push(pending); + } + // Every file is staged and every destination checked before publication. + // Three renames are still not one transaction. + for pending in staged { + pending.commit()?; + } + Ok(()) + } +} + +/// `x.get(k)`: the value, or JSON null. +fn get(x: &IndexMap, k: &str) -> Value { + x.get(k).cloned().unwrap_or(Value::Null) +} + +/// Python's truth of a JSON value. +fn truthy(v: &Value) -> bool { + pycompat::value::truthy(v) +} + +/// The old trimmer used `v or []`: null and every false value are empty. +/// `v or []` as a list; a truthy value that is not a list is an error. +fn list<'a>(v: Option<&'a Value>, what: &str) -> Result<&'a [Value], String> { + match v { + Some(Value::Array(a)) => Ok(a), + Some(x) if truthy(x) => Err(format!("{what} is not a list")), + _ => Ok(&[]), + } +} + +fn object<'a>(v: &'a Value, what: &str) -> Result<&'a IndexMap, String> { + match v { + Value::Object(o) => Ok(o), + _ => Err(format!("{what} is not an object")), + } +} + +/// The keys of `x` among `keys`, in that order. +fn pick(x: &IndexMap, keys: &[&str]) -> IndexMap { + keys.iter() + .filter_map(|k| x.get(*k).map(|v| ((*k).to_owned(), v.clone()))) + .collect() +} + +/// One listed diplotype with the fields `og pgx` reads. +fn diplotype(x: &Value) -> Result { + let Value::Object(x) = x else { + // The Python dip() explicitly passes non-object listed diplotypes through. + return Ok(x.clone()); + }; + let mut out = pick( + x, + &[ + "gene", + "label", + "phenotypes", + "activityScore", + "outsidePhenotype", + ], + ); + for a in ["allele1", "allele2"] { + let Some(v) = x.get(a) else { continue }; + let kept = match v { + Value::Object(o) if !o.is_empty() => { + Value::Object(pick(o, &["gene", "name", "function"])) + } + v if !truthy(v) => v.clone(), + _ => return Err(format!("diplotype {a} is not an object")), + }; + out.insert(a.to_owned(), kept); + } + Ok(Value::Object(out)) +} + +/// `items[:n]` with Python's slice rule for a negative `n`. +#[allow( + clippy::manual_saturating_arithmetic, + reason = "Python slices clamp a checked negative end at zero" +)] +fn head(items: &[Value], n: i64) -> &[Value] { + // Clamp before slicing, including i64::MIN, without narrowing lengths. + let count = usize::try_from(n.unsigned_abs()).unwrap_or(usize::MAX); + let end = if n < 0 { + items.len().checked_sub(count).unwrap_or_default() + } else { + count.min(items.len()) + }; + &items[..end] +} + +/// The trimmed report: every gene with its listed diplotypes and related +/// drugs, and for each drug of each guidance source up to `max` of the +/// annotations PharmCAT matched, with only the fields `og pgx` reads. +pub fn trim(d: Value, max: i64) -> Result { + let Value::Object(mut d) = d else { + return Err("the report is not an object".to_owned()); + }; + let mut genes = IndexMap::new(); + let Some(Value::Object(input_genes)) = d.swap_remove("genes") else { + return Err("genes is not an object".to_owned()); + }; + // Consume one source gene at a time: do not retain a second full report. + for (g, v) in input_genes { + let v = object(&v, &format!("gene {g}"))?; + let mut gene = pick(v, &["geneSymbol", "chr", "phased", "callSource"]); + let mut related = Vec::new(); + for r in list(v.get("relatedDrugs"), "relatedDrugs")? { + let r = object(r, "a related drug")?; + let name = r.get("name").ok_or("a related drug has no name")?.clone(); + related.push(Value::Object(IndexMap::from([ + ("name".to_owned(), name), + ("id".to_owned(), get(r, "id")), + ]))); + } + gene.insert("relatedDrugs".to_owned(), Value::Array(related)); + // sourceDiplotypes is what og pgx reads; recommendationDiplotypes + // only when sourceDiplotypes is absent, so it is kept only then. + let k = if v.get("sourceDiplotypes").is_some_and(truthy) { + "sourceDiplotypes" + } else { + "recommendationDiplotypes" + }; + if v.contains_key(k) { + let dips = list(v.get(k), k)? + .iter() + .map(diplotype) + .collect::>()?; + gene.insert(k.to_owned(), Value::Array(dips)); + } + genes.insert(g, Value::Object(gene)); + } + let mut drugs: IndexMap = IndexMap::new(); + let Some(Value::Object(input_drugs)) = d.swap_remove("drugs") else { + return Err("drugs is not an object".to_owned()); + }; + for (source, by) in input_drugs { + for (name, rep) in object(&by, &format!("drugs of {source}"))? { + let rep = object(rep, &format!("drug {name}"))?; + let mut gls = Vec::new(); + for gl in list(rep.get("guidelines"), "guidelines")? { + let gl = object(gl, "a guideline")?; + let mut anns = Vec::new(); + for a in head(list(gl.get("annotations"), "annotations")?, max) { + let a = object(a, "an annotation")?; + let mut gts = Vec::new(); + for gt in list(a.get("genotypes"), "genotypes")? { + let gt = object(gt, "a genotype")?; + let dips = list(gt.get("diplotypes"), "diplotypes")? + .iter() + .filter_map(|x| match x { + Value::Object(x) => Some(Value::Object(IndexMap::from([ + ("gene".to_owned(), get(x, "gene")), + ("label".to_owned(), get(x, "label")), + ]))), + // The Python comprehension has `if isinstance(x, dict)`. + _ => None, + }) + .collect(); + gts.push(Value::Object(IndexMap::from([( + "diplotypes".to_owned(), + Value::Array(dips), + )]))); + } + anns.push(Value::Object(IndexMap::from([ + ("phenotypes".to_owned(), get(a, "phenotypes")), + ("classification".to_owned(), get(a, "classification")), + ( + "drugRecommendation".to_owned(), + get(a, "drugRecommendation"), + ), + ("genotypes".to_owned(), Value::Array(gts)), + ]))); + } + if !anns.is_empty() { + gls.push(Value::Object(IndexMap::from([ + ("name".to_owned(), get(gl, "name")), + ("source".to_owned(), get(gl, "source")), + ("annotations".to_owned(), Value::Array(anns)), + ]))); + } + } + if !gls.is_empty() { + let entry = drugs + .entry(source.clone()) + .or_insert_with(|| Value::Object(IndexMap::new())); + if let Value::Object(e) = entry { + e.insert( + name.clone(), + Value::Object(IndexMap::from([ + ("name".to_owned(), get(rep, "name")), + ("id".to_owned(), get(rep, "id")), + ("guidelines".to_owned(), Value::Array(gls)), + ])), + ); + } + } + } + } + let mut out = pick( + &d, + &["title", "timestamp", "pharmcatVersion", "dataVersion"], + ); + out.insert("genes".to_owned(), Value::Object(genes)); + out.insert("drugs".to_owned(), Value::Object(drugs)); + let mut note = String::new(); + let _ = write!( + note, + "Trimmed by og dev fixture trim-pharmcat: every gene, and up to {max} matched annotations per drug guideline, with only the fields og pgx reads." + ); + out.insert("_fixture_note".to_owned(), Value::Str(note)); + Ok(Value::Object(out)) +} + +fn trim_pharmcat(src: &Path, dst: &Path, max: i64) -> Result<(), String> { + let text = fs::read(src).map_err(|e| py_os_error(&e, &src.to_string_lossy()))?; + let text = String::from_utf8(text).map_err(|_| format!("{}: not UTF-8", src.display()))?; + let d = json::loads(&text).map_err(|e| format!("{}: {e}", src.display()))?; + drop(text); + let out = trim(d, max)?; + let opts = Dump { + separators: Some((",", ":")), + ..Dump::default() + }; + let text = json::dumps(&out, &opts).map_err(|e| e.to_string())?; + crate::evidence::publish(dst, text.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixture_verbs_use_checked_runner_even_with_empty_status() { + let d = std::env::temp_dir().join(format!("og-fixture-streams-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for args in [ + vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ], + vec!["comphet".into(), d.to_str().unwrap().into()], + ] { + crate::stream_tests::assert_status_errors(cli, &args, false); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn annotation_slices_clamp_extreme_counts_without_narrowing() { + let items = vec![Value::Null; 3]; + for (n, expected) in [ + (i64::MIN, 0), + (-4, 0), + (-3, 0), + (-2, 1), + (-1, 2), + (0, 0), + (1, 1), + (i64::MAX, 3), + ] { + assert_eq!(head(&items, n).len(), expected); + } + } + + #[test] + fn non_ascii_or_out_of_range_annotation_limits_fail_at_cli() { + let d = std::env::temp_dir().join(format!("og-limits-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for limit in ["1_0", "٤", "18446744073709551616"] { + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + limit.into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 2); + let err = String::from_utf8(err).unwrap(); + assert!( + err.contains("error: MAX_ANNOTATIONS") && err.contains("is not a whole number"), + "{err}" + ); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn non_writable_parents_preserve_fixture_destinations() { + use std::os::unix::fs::PermissionsExt; + let d = std::env::temp_dir().join(format!("og-parent-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("in"), br#"{"genes":{},"drugs":{}}"#).unwrap(); + for name in ["out", "reads.sam", "vep.vcf", "comphet.vcf"] { + fs::write(d.join(name), b"old").unwrap(); + } + for args in [ + vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ], + vec!["comphet".into(), d.to_str().unwrap().into()], + ] { + fs::set_permissions(&d, fs::Permissions::from_mode(0o555)).unwrap(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + fs::set_permissions(&d, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(code, 1); + assert!( + String::from_utf8(err) + .unwrap() + .contains(": parent directory is not writable") + ); + for name in ["out", "reads.sam", "vep.vcf", "comphet.vcf"] { + assert_eq!(fs::read(d.join(name)).unwrap(), b"old"); + } + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn pharmcat_input_is_plain_json() { + let d = + std::env::temp_dir().join(format!("og-pharmcat-compression-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(br#"{"genes":{},"drugs":{}}"#).unwrap(); + fs::write(d.join("in"), gz.finish().unwrap()).unwrap(); + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 1); + assert!(String::from_utf8_lossy(&err).contains("not UTF-8")); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn comphet_validates_later_destinations_before_publishing() { + let d = std::env::temp_dir().join(format!("og-comphet-preflight-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("comphet.vcf")).unwrap(); + for name in ["reads.sam", "vep.vcf"] { + fs::write(d.join(name), b"PREVIOUS OUTPUT").unwrap(); + } + assert!(comphet::write(&d, false).is_err()); + for name in ["reads.sam", "vep.vcf"] { + assert_eq!(fs::read(d.join(name)).unwrap(), b"PREVIOUS OUTPUT"); + } + assert_eq!(fs::read_dir(&d).unwrap().count(), 3); + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn downloader_preflights_destinations_and_cleans_handled_failures() { + use std::os::unix::fs::{PermissionsExt, symlink}; + use std::process::Command; + let d = std::env::temp_dir().join(format!("og-download-safety-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("bin")).unwrap(); + let stub = d.join("bin/curl"); + fs::write(&stub, "#!/bin/sh\nprintf called >> \"$CALLED\"\nwhile [ \"$1\" != -o ]; do shift; done\nprintf partial > \"$2\"\nexit 22\n").unwrap(); + fs::set_permissions(&stub, fs::Permissions::from_mode(0o755)).unwrap(); + let script = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../tests/smoke/vntyper_testdata.sh"); + for fault in [ + "directory", + "directory-link", + "curl", + "crc", + "sha", + "signal", + ] { + let out = d.join(fault); + fs::create_dir(&out).unwrap(); + let dst = out.join("negative.bam"); + match fault { + "directory" => fs::create_dir(&dst).unwrap(), + "directory-link" => symlink(&d, &dst).unwrap(), + _ => { + fs::write(&dst, b"PREVIOUS OUTPUT").unwrap(); + symlink("negative.bam", out.join("negative.bam.part")).unwrap(); + } + } + if fault == "crc" { + fs::write(&stub, "#!/bin/sh\nprintf called >> \"$CALLED\"\nwhile [ \"$1\" != -o ]; do shift; done\ndd if=/dev/zero of=\"$2\" bs=3376349 count=1 2>/dev/null\n").unwrap(); + } else if fault == "sha" { + // Supply an inflater with correct size but the wrong SHA. + fs::write(d.join("bin/gzip"), "#!/bin/sh\nprintf wrong-sha\n").unwrap(); + fs::set_permissions(d.join("bin/gzip"), fs::Permissions::from_mode(0o755)).unwrap(); + } else if fault == "signal" { + fs::remove_file(d.join("bin/gzip")).unwrap(); + fs::write(&stub, "#!/bin/sh\nkill -TERM \"$PPID\"\nexit 22\n").unwrap(); + } + let called = d.join("called"); + let _ = fs::remove_file(&called); + let result = Command::new("sh") + .arg(&script) + .arg(&out) + .arg("negative.bam") + .env( + "PATH", + format!( + "{}:{}", + d.join("bin").display(), + std::env::var("PATH").unwrap() + ), + ) + .env("CALLED", &called) + .output() + .unwrap(); + assert!(!result.status.success(), "{fault}"); + if fault.starts_with("directory") { + assert!(!called.exists(), "must reject before curl: {fault}"); + } else { + assert_eq!(fs::read(&dst).unwrap(), b"PREVIOUS OUTPUT", "{fault}"); + let entries: Vec<_> = fs::read_dir(&out) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(entries.len(), 2, "{fault}: {entries:?}"); + } + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn approved_strict_pharmcat_containers_fail_at_the_cli() { + let d = std::env::temp_dir().join(format!("og-pharmcat-shapes-{}", std::process::id())); + fs::create_dir_all(&d).unwrap(); + for (input, class) in [ + ( + r#"{"genes":{"G":{"sourceDiplotypes":"x"}},"drugs":{}}"#, + "sourceDiplotypes is not a list", + ), + ( + r#"{"genes":{"G":{"recommendationDiplotypes":"x"}},"drugs":{}}"#, + "recommendationDiplotypes is not a list", + ), + ( + r#"{"genes":{"G":{"sourceDiplotypes":[{"allele1":"x"}]}},"drugs":{}}"#, + "allele1 is not an object", + ), + ( + r#"{"genes":{"G":{"sourceDiplotypes":[{"allele2":["x"]}]}},"drugs":{}}"#, + "allele2 is not an object", + ), + ( + r#"{"genes":{},"drugs":{"S":{"D":{"guidelines":[{"annotations":[{"genotypes":[{"diplotypes":{"x":1}}]}]}]}}}}"#, + "diplotypes is not a list", + ), + ] { + fs::write(d.join("in"), input).unwrap(); + fs::write(d.join("out"), b"old").unwrap(); + let args = vec![ + "trim-pharmcat".into(), + d.join("in").to_str().unwrap().into(), + d.join("out").to_str().unwrap().into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&args, &mut out, &mut err), 1); + assert!(String::from_utf8_lossy(&err).contains(class), "{err:?}"); + assert_eq!(fs::read(d.join("out")).unwrap(), b"old"); + } + fs::remove_dir_all(d).unwrap(); + } + + #[test] + fn mt19937_matches_cpython() { + // random.Random(31): [r.getrandbits(32) for _ in range(3)] and the + // first ten of "".join(r.choice("ACGT") ...) from a fresh seed. + let mut r = Mt19937::new(31); + let first: Vec = (0..3).map(|_| r.next_u32()).collect(); + assert_eq!(first, [52_734_659, 2_016_822_975, 482_749_413]); + assert_eq!(&comphet::reference()[..10], b"ATATCACACC"); + } + + #[test] + fn swap_moves_only_the_deletion() { + let seq = comphet::reference(); + let (a, b) = ( + comphet::variants(&seq, false), + comphet::variants(&seq, true), + ); + assert_eq!(a[1].copy, 0); + assert_eq!(b[1].copy, 1); + assert_eq!(a[0], b[0]); + assert_eq!(a[2..], b[2..]); + } + + #[test] + fn a_read_over_the_deletion_has_a_d_in_its_cigar() { + let seq = comphet::reference(); + let vs = comphet::variants(&seq, false); + let sam = comphet::sam(&seq, &vs); + let line = sam + .lines() + .find(|l| l.starts_with("c0_920\t99")) + .expect("read c0_920"); + assert!(line.contains("\t83M2D17M\t"), "{line}"); + let line = sam + .lines() + .find(|l| l.starts_with("c1_920\t99")) + .expect("read c1_920"); + assert!(line.contains("\t100M\t"), "{line}"); + } + + #[test] + fn trim_keeps_the_fields_og_pgx_reads() { + let d = json::loads( + r#"{"title":"t","extra":1,"genes":{"G":{"geneSymbol":"G","x":1,"relatedDrugs":[{"name":"d"}], + "sourceDiplotypes":[],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1","y":2},"allele2":null,"z":3},"str"]}}, + "drugs":{"CPIC":{"d":{"name":"d","guidelines":[{"name":"g","annotations":[{"genotypes":[{"diplotypes":[{"gene":"G"},5]}]},{},{}]},{"annotations":[]}]}}}}"#, + ) + .expect("json"); + let out = trim(d.clone(), 2).expect("trim"); + let text = json::dumps( + &out, + &Dump { + separators: Some((",", ":")), + ..Dump::default() + }, + ) + .expect("dumps"); + assert!(text.starts_with(r#"{"title":"t","genes":{"G":{"geneSymbol":"G","relatedDrugs":[{"name":"d","id":null}],"recommendationDiplotypes":[{"label":"*1/*2","allele1":{"name":"*1"},"allele2":null},"str"]}}"#), "{text}"); + assert!(text.contains(r#""annotations":[{"phenotypes":null,"classification":null,"drugRecommendation":null,"genotypes":[{"diplotypes":[{"gene":"G","label":null}]}]},{"phenotypes":null"#), "{text}"); + assert_eq!(text.matches("drugRecommendation").count(), 2, "{text}"); + // A negative count is Python's slice: all but the last. + let out = trim(d, -2).expect("trim"); + let text = json::dumps(&out, &Dump::default()).expect("dumps"); + assert_eq!(text.matches("drugRecommendation").count(), 1, "{text}"); + } +} diff --git a/rust/crates/og-dev/src/images.rs b/rust/crates/og-dev/src/images.rs new file mode 100644 index 000000000..34f2d540d --- /dev/null +++ b/rust/crates/og-dev/src/images.rs @@ -0,0 +1,443 @@ +//! `og dev scan images` and `og dev versions coupled`: the two checks of +//! `scripts/ci/check-images.sh` that were Python. The script keeps the +//! lists (which paths are scanned, which files and single literals are +//! exempt, with the reason) and passes them in. +//! +//! ```text +//! og dev scan images --root DIR --exempt-files TEXT --exempt-pairs TEXT < FILES +//! og dev versions coupled --env versions.env +//! ``` +//! +//! `scan images` reads the files to scan (one path relative to DIR per +//! line) on stdin and prints one `FAIL: : names the image +//! ; ...` per literal image, then `scanned N files`. An image is +//! `registry/name:tag`, `name@sha256:...`, or a Docker Hub official name +//! (those `versions.env` uses and the usual base images) with a tag. A line +//! of `--exempt-pairs` (`file image reason`) allows one occurrence of that +//! image in that file; listing a pair twice allows two. +//! +//! `versions coupled` checks that `MINIMAP2_SAMTOOLS_IMAGE`, the one image +//! of the Nextflow alignment task, names in its `versions.env` comment the +//! minimap2 and samtools releases of `MINIMAP2_IMAGE` and `SAMTOOLS_IMAGE`. +//! +//! Exit 0 clean, 1 on a finding, 2 on a usage error or a `versions.env` +//! the strict reader refuses. +//! +//! Where it differs from the Python on purpose: an exempt pair whose file +//! no longer names its image as often as the list allows fails +//! (`FAIL: no longer names ; drop its line ...`), so an +//! exemption cannot outlive its reason; and `versions.env` is read strictly +//! ([`crate::versions_env`]). + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read, Write}; +use std::path::Path; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::lookaround::Rule; +use crate::versions_env::{self, Pin}; + +const USAGE: &str = "usage: og dev scan images --root DIR --exempt-files TEXT --exempt-pairs TEXT < FILES\n og dev versions coupled --env versions.env\n"; + +/// Docker Hub official images that `versions.env` may not use today. +const BASES: [&str; 15] = [ + "ubuntu", + "debian", + "alpine", + "busybox", + "centos", + "rockylinux", + "fedora", + "python", + "node", + "perl", + "r-base", + "openjdk", + "eclipse-temurin", + "golang", + "rust", +]; + +/// Any tag: a version, or a word such as latest or stable. +const TAG: &str = r"(?::[A-Za-z0-9_][A-Za-z0-9._-]*|@sha256:[0-9a-f]{64})"; +/// What may not stand right before an image, nor right after it. +const AHEAD: &str = r"(?:[^A-Za-z0-9_/-]|$)"; + +fn lead(c: char) -> bool { + c.is_ascii_alphanumeric() || "_./:@$-".contains(c) +} + +static URL: LazyLock = + LazyLock::new(|| Regex::new(r"[a-z][a-z0-9+.-]*://[^\s\x1c-\x1f]+").expect("a valid pattern")); +/// A path to a file, such as modules/local/vep/main.nf:23, is not an image. +static FILE: LazyLock = LazyLock::new(|| { + Regex::new( + r"\.(nf|sh|md|py|config|ya?ml|json|txt|tsv|csv|vcf|gz|html|css|js|R|r):[0-9]+(-[0-9]+)?$", + ) + .expect("a valid pattern") +}); + +fn bare_names(pins: &[Pin]) -> BTreeSet { + let mut bare: BTreeSet = BASES.iter().map(|s| (*s).to_owned()).collect(); + for p in pins.iter().filter(|p| p.name.ends_with("_IMAGE")) { + if let Some(i) = p.value.find([':', '@']) { + let name = &p.value[..i]; + if !name.is_empty() && !name.contains('/') { + bare.insert(name.to_owned()); + } + } + } + bare +} + +/// The two patterns: `ns/name:tag`, and an official name with a tag. +fn patterns(bare: &BTreeSet) -> [Rule; 2] { + let names: Vec = bare.iter().map(|b| regex::escape(b)).collect(); + [ + Rule::new( + &format!(r"([a-z][a-z0-9._-]*/[a-z][a-z0-9._/-]*{TAG}){AHEAD}"), + Some(lead), + ) + .expect("a valid pattern"), + Rule::new( + &format!(r"((?:{}){TAG}){AHEAD}", names.join("|")), + Some(lead), + ) + .expect("a valid pattern"), + ] +} + +/// The text of a file as Python's text mode reads it: strict UTF-8, any +/// newline turned into `\n`. None for a file that is not UTF-8 or not a +/// readable file. +fn read_text(path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + let text = String::from_utf8(bytes).ok()?; + Some(text.replace("\r\n", "\n").replace('\r', "\n")) +} + +/// The literal images of one file, with their line numbers; exempt +/// occurrences are taken off `left`. +fn scan_file( + rel: &str, + text: &str, + rules: &[Rule; 2], + left: &mut BTreeMap<(String, String), usize>, + out: &mut dyn Write, +) -> bool { + let mut bad = false; + for (i, raw) in text.split('\n').enumerate() { + // docker://image:tag (a step image in a workflow) is an image, not a URL. + let line = raw.replace("docker://", " "); + let line = URL.replace_all(&line, " "); + for r in rules { + for (s, e) in r.find_iter(&line) { + // A full stop ends the sentence, not the tag. + let image = line[s..e].trim_end_matches('.'); + if FILE.is_match(image) || image.starts_with("example/") { + continue; // example/ is the placeholder namespace of the self-tests + } + if let Some(n) = left.get_mut(&(rel.to_owned(), image.to_owned())) + && *n > 0 + { + *n -= 1; + continue; + } + let _ = writeln!( + out, + "FAIL: {rel}:{} names the image {image}; use its *_IMAGE variable from versions.env", + i + 1 + ); + bad = true; + } + } + } + bad +} + +struct ScanArgs { + root: String, + exempt_files: Vec, + pairs: BTreeMap<(String, String), usize>, +} + +fn parse_scan(args: &[String]) -> Result { + let (mut root, mut files, mut pairs) = (None, None, None); + let mut it = args.iter(); + while let Some(a) = it.next() { + let v = it.next().ok_or_else(|| format!("{a} needs a value")); + match a.as_str() { + "--root" => root = Some(v?.clone()), + "--exempt-files" => files = Some(v?.clone()), + "--exempt-pairs" => pairs = Some(v?.clone()), + s => return Err(format!("unknown argument {s}")), + } + } + let exempt_files = files + .ok_or("--exempt-files is required")? + .lines() + .filter_map(|l| l.split_whitespace().next().map(str::to_owned)) + .collect(); + let mut counted = BTreeMap::new(); + for l in pairs.ok_or("--exempt-pairs is required")?.lines() { + let w: Vec<&str> = l.split_whitespace().take(2).collect(); + if let [f, image] = w[..] { + *counted.entry((f.to_owned(), image.to_owned())).or_insert(0) += 1; + } else if !w.is_empty() { + return Err(format!("an exempt pair needs a file and an image: {l}")); + } + } + Ok(ScanArgs { + root: root.ok_or("--root is required")?, + exempt_files, + pairs: counted, + }) +} + +fn scan(a: &ScanArgs, list: &str, out: &mut dyn Write) -> Result { + let files: Vec<&str> = list.lines().filter(|f| !f.is_empty()).collect(); + if files.is_empty() { + let _ = writeln!(out, "FAIL: no file to scan"); + return Ok(false); + } + let root = Path::new(&a.root); + let pins = versions_env::read(&root.join("versions.env"))?; + let rules = patterns(&bare_names(&pins)); + let mut left = a.pairs.clone(); + let mut bad = false; + for f in &files { + let skipped = a + .exempt_files + .iter() + .any(|s| *f == s || (s.ends_with('/') && f.starts_with(s.as_str()))); + if skipped { + continue; + } + if let Some(text) = read_text(&root.join(f)) { + bad |= scan_file(f, &text, &rules, &mut left, out); + } + } + for ((f, image), n) in &left { + if *n > 0 { + let _ = writeln!( + out, + "FAIL: {f} no longer names {image}{}; drop its line from EXEMPT_PAIRS in scripts/ci/check-images.sh", + if *n > 1 { + format!(" {n} more times") + } else { + String::new() + } + ); + bad = true; + } + } + let _ = writeln!(out, "scanned {} files", files.len()); + Ok(!bad) +} + +/// `og dev scan images ARGS`. +pub fn cli_scan(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let a = match parse_scan(args) { + Ok(a) => a, + Err(e) => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og dev scan images: error: {e}"); + return 2; + } + }; + let mut list = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut list) { + let _ = writeln!(err, "ERROR: cannot read the file list: {e}"); + return 2; + } + match scan(&a, &list, out) { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +/// The release in an image's tag: `quay.io/x/samtools:1.24--h0_0` is 1.24. +fn release(value: &str) -> Option<&str> { + let tag = value.rsplit('/').next().unwrap_or(value); + let (_, t) = tag.split_once(':')?; + Some(t.split("--").next().unwrap_or(t)) +} + +static HELD: LazyLock = LazyLock::new(|| { + Regex::new(r"\b(minimap2|samtools)=([0-9][0-9A-Za-z.]*)").expect("a valid pattern") +}); + +/// The check on parsed pins; prints one line per tool. +pub fn coupled(pins: &[Pin], out: &mut dyn Write) -> bool { + let Some(image) = versions_env::get(pins, "MINIMAP2_SAMTOOLS_IMAGE") else { + let _ = writeln!( + out, + "FAIL: MINIMAP2_SAMTOOLS_IMAGE is not set in versions.env" + ); + return false; + }; + let mut held: BTreeMap<&str, &str> = BTreeMap::new(); + for c in HELD.captures_iter(&image.rest) { + let (tool, v) = ( + c.get(1).map_or("", |m| m.as_str()), + c.get(2).map_or("", |m| m.as_str()), + ); + held.insert(tool, v.trim_end_matches('.')); + } + let mut ok = true; + for (tool, var) in [ + ("minimap2", "MINIMAP2_IMAGE"), + ("samtools", "SAMTOOLS_IMAGE"), + ] { + let want = versions_env::get(pins, var).and_then(|p| release(&p.value)); + let got = held.get(tool).copied(); + match (want, got) { + (Some(w), Some(g)) if !w.is_empty() && w == g => { + let _ = writeln!( + out, + "OK: MINIMAP2_SAMTOOLS_IMAGE holds {tool} {g}, the release of {var}" + ); + } + _ => { + let want = want.filter(|w| !w.is_empty()); + let _ = writeln!( + out, + "FAIL: MINIMAP2_SAMTOOLS_IMAGE says {tool}={} in its versions.env comment, but {var} is {}: pin a BioContainers mulled image with {tool} {} and name it in that comment (docs/updating.md, Held pins)", + got.unwrap_or("(nothing)"), + want.unwrap_or("(unset)"), + want.unwrap_or("?") + ); + ok = false; + } + } + } + ok +} + +/// `og dev versions coupled --env FILE`. +pub fn cli_coupled(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let [flag, path] = args else { + let _ = write!(err, "{USAGE}"); + return 2; + }; + if flag != "--env" { + let _ = write!(err, "{USAGE}"); + return 2; + } + match versions_env::read(Path::new(path)) { + Ok(pins) => u8::from(!coupled(&pins, out)), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rules() -> [Rule; 2] { + let pins = versions_env::parse("PYTHON_IMAGE=\"python:3.15.2\"\n").unwrap(); + patterns(&bare_names(&pins)) + } + + fn found(line: &str) -> Vec { + let mut left = BTreeMap::new(); + let mut out = Vec::new(); + scan_file("f", line, &rules(), &mut left, &mut out); + String::from_utf8(out) + .unwrap() + .lines() + .map(|l| { + l.split(" names the image ") + .nth(1) + .unwrap() + .split(';') + .next() + .unwrap() + .to_owned() + }) + .collect() + } + + // Planted images are split by "~", which is dropped, so this file + // names none of them whole. + fn p(s: &str) -> String { + s.replace('~', "") + } + + #[test] + fn literals_are_found_as_python_finds_them() { + assert_eq!( + found(&p("docker run staphb/bcftools:~1.21 x")), + [p("staphb/bcftools:~1.21")] + ); + assert_eq!( + found(&p("runs staphb/samtools:~1.19.")), + [p("staphb/samtools:~1.19")] + ); + assert_eq!(found(&p("// container python:~3.12")), [p("python:~3.12")]); + assert_eq!(found(&p("uses: docker://alpine:~3.8")), [p("alpine:~3.8")]); + // A tag followed by a slash backs off to the part before a dot. + assert_eq!( + found(&p("x staphb/samtools:~1.2/x")), + [p("staphb/samtools:~1")] + ); + for clean in [ + "see https://quay.io/repository/biocontainers/samtools:1.0", + "modules/local/vep/main.nf:23", + "example/bcftools:1.0", + "$VAR/name:tag", + "python3 x.py", + ] { + assert!(found(clean).is_empty(), "{clean}: {:?}", found(clean)); + } + } + + #[test] + fn a_stale_exemption_fails() { + let dir = std::env::temp_dir().join(format!("og-dev-images-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("versions.env"), "A_IMAGE=\"x/y:1\"\n").unwrap(); + std::fs::write(dir.join("a.md"), p("uses quay.io/x/y:~2\n")).unwrap(); + let a = ScanArgs { + root: dir.to_string_lossy().into_owned(), + exempt_files: vec![], + pairs: BTreeMap::from([ + (("a.md".into(), p("quay.io/x/y:~2")), 1), + (("a.md".into(), p("quay.io/x/y:~3")), 1), + ]), + }; + let mut out = Vec::new(); + let ok = scan(&a, "a.md\n", &mut out).unwrap(); + let out = String::from_utf8(out).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert!(!ok); + assert!( + out.contains(&p( + "FAIL: a.md no longer names quay.io/x/y:~3; drop its line" + )), + "{out}" + ); + assert!(!out.contains(&p("y:~2;")), "{out}"); + } + + #[test] + fn coupled_releases() { + let env = "MINIMAP2_IMAGE=\"q/minimap2:2.31--h1_0\"\nSAMTOOLS_IMAGE=\"q/samtools:1.24--h2_0\"\nMINIMAP2_SAMTOOLS_IMAGE=\"q/mulled:abc\" # minimap2=2.31,samtools=1.24.\n"; + let mut out = Vec::new(); + assert!(coupled(&versions_env::parse(env).unwrap(), &mut out)); + let bumped = env.replace("samtools:1.24", "samtools:9.99"); + let mut out = Vec::new(); + assert!(!coupled(&versions_env::parse(&bumped).unwrap(), &mut out)); + let out = String::from_utf8(out).unwrap(); + assert!(out.contains("FAIL: MINIMAP2_SAMTOOLS_IMAGE says samtools=1.24 in its versions.env comment, but SAMTOOLS_IMAGE is 9.99:"), "{out}"); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index d222ec760..a5f776c46 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -15,22 +15,50 @@ //! (`scripts/ci/check-container-helpers.sh`). //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev scan personal-data` ([`personal_data`]): no tracked file holds +//! a home path, a private address or a genotype. +//! - `og dev scan images`, `og dev versions coupled` ([`images`]): no file +//! names an image outside `versions.env`; the alignment task's image holds +//! the releases of step 02's two images. +//! - `og dev versions check-run`, `og dev versions expand` ([`versions`]): a +//! run used the images `conf/containers.config` names; download URLs with +//! their `versions.env` variables filled. +//! - `og dev pgs` ([`pgs`]): the PGS Catalog's answers against +//! `assets/pgs_scores.tsv`. +//! - `og dev fixture` ([`fixture`]): writers of test fixtures (the +//! compound-het reads, a trimmed PharmCAT report). +//! - `og dev evidence` ([`evidence`]): the measurements of the public +//! evidence workflow on GIAB's HG002. +//! - `og dev renovate-check LOG` ([`renovate`]): what a Renovate dry run +//! found, against every pin in the repository. //! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): //! rebuild a committed asset table from its pinned downloads. pub mod build; pub mod contract; +pub mod evidence; +pub mod fixture; pub mod helpers; +pub mod images; +mod lookaround; +pub mod nf_trace; pub mod parity; +pub mod personal_data; +pub mod pgs; +pub mod pharmcat; pub mod remnants; +pub mod renovate; pub mod report_only; pub mod schema; pub mod shlex; +pub mod smoke; +pub mod versions; +pub mod versions_env; mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -50,11 +78,80 @@ pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { /// Runs a verb that writes to stdout and stderr, flushing both. fn run(f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, args: &[String]) -> u8 { - let mut out = std::io::stdout().lock(); - let mut err = std::io::stderr().lock(); + use std::os::fd::AsFd; + // Stdout/StdoutLock suppress EBADF on Unix. Duplicating the descriptor + // preserves its access mode and exposes write errors without unsafe code. + let out = match std::io::stdout().as_fd().try_clone_to_owned() { + Ok(fd) => std::fs::File::from(fd), + Err(e) => { + let _ = writeln!(std::io::stderr().lock(), "og dev: stdout: {e}"); + return 1; + } + }; + let mut err = match std::io::stderr().as_fd().try_clone_to_owned() { + Ok(fd) => std::fs::File::from(fd), + Err(_) => return 1, + }; + let mut out = std::io::BufWriter::new(out); + run_with_writers(f, args, &mut out, &mut err) +} + +pub(crate) fn run_with_writers( + f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, + args: &[String], + out: &mut dyn Write, + err: &mut dyn Write, +) -> u8 { + let mut out = CheckedWriter { + inner: out, + failure: None, + }; + let mut err = CheckedWriter { + inner: err, + failure: None, + }; let code = f(args, &mut out, &mut err); let _ = out.flush(); - code + if let Some(reason) = &out.failure { + let _ = writeln!(err, "og dev: stdout: {reason}"); + } + let _ = err.flush(); + if code == 0 && (out.failure.is_some() || err.failure.is_some()) { + 1 + } else { + code + } +} + +/// Remember errors even when an older verb ignores its Write result. +struct CheckedWriter<'a> { + inner: &'a mut dyn Write, + failure: Option, +} +impl Write for CheckedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let result = match self.inner.write(bytes) { + Ok(0) if !bytes.is_empty() => Err(std::io::Error::from(std::io::ErrorKind::WriteZero)), + result => result, + }; + if let Err(e) = &result { + self.failure.get_or_insert_with(|| e.to_string()); + } + result + } + fn flush(&mut self) -> std::io::Result<()> { + let result = self.inner.flush(); + if let Err(e) = &result { + self.failure.get_or_insert_with(|| e.to_string()); + } + result + } +} + +fn unknown(verb: &str) -> u8 { + eprint!("{USAGE}"); + eprintln!("og dev: error: unknown verb '{verb}'"); + 2 } /// Runs `og dev ARGS`; returns the exit code. @@ -65,17 +162,31 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("scan") => match args.get(1).map(String::as_str) { + Some("personal-data") => run(personal_data::cli, &args[2..]), + Some("images") => run(images::cli_scan, &args[2..]), + _ => unknown(args.get(1).map_or("scan", String::as_str)), + }, + Some("versions") => match args.get(1).map(String::as_str) { + Some("check-run") => run(versions::cli_check_run, &args[2..]), + Some("expand") => run(versions::cli_expand, &args[2..]), + Some("coupled") => run(images::cli_coupled, &args[2..]), + _ => unknown(args.get(1).map_or("versions", String::as_str)), + }, + Some("pgs") => run(pgs::cli, &args[1..]), + Some("fixture") => run(fixture::cli, &args[1..]), + Some("evidence") => run(evidence::cli, &args[1..]), + Some("renovate-check") => run(renovate::cli, &args[1..]), + Some("pharmcat") => run(pharmcat::cli, &args[1..]), + Some("nf-trace") => run(nf_trace::cli, &args[1..]), + Some("smoke") => run(smoke::cli, &args[1..]), Some("build") => run(build::cli, &args[1..]), Some("helpers") => run(helpers::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 } - Some(other) => { - eprint!("{USAGE}"); - eprintln!("og dev: error: unknown verb '{other}'"); - 2 - } + Some(other) => unknown(other), None => { eprint!("{USAGE}"); eprintln!("og dev: error: the following arguments are required: verb"); @@ -83,3 +194,109 @@ pub fn main(args: &[String]) -> u8 { } } } + +#[cfg(test)] +mod stream_tests { + use super::*; + struct Fails { + flush_only: bool, + } + impl Write for Fails { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.flush_only { + Ok(bytes.len()) + } else { + Err(std::io::Error::other("write control")) + } + } + fn flush(&mut self) -> std::io::Result<()> { + if self.flush_only { + Err(std::io::Error::other("flush control")) + } else { + Ok(()) + } + } + } + #[test] + fn stdio_process_control() { + if let Ok(verb) = std::env::var("OG_DEV_STREAM_CONTROL") { + std::process::exit(i32::from(main(&[verb, "--help".into()]))); + } + } + + #[test] + fn read_only_stdout_descriptor_makes_added_verbs_fail() { + let path = std::env::temp_dir().join(format!("og-stream-fd-{}", std::process::id())); + std::fs::write(&path, b"previous bytes").unwrap(); + for verb in ["fixture", "evidence", "renovate-check"] { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "stream_tests::stdio_process_control", + "--nocapture", + ]) + .env("OG_DEV_STREAM_CONTROL", verb) + .stdout(std::fs::File::open(&path).unwrap()) + .output() + .unwrap(); + assert!(!result.status.success(), "{verb} silently lost stdout"); + assert!( + String::from_utf8_lossy(&result.stderr).contains("og dev: stdout:"), + "{verb}: {:?}", + result.stderr + ); + assert_eq!(std::fs::read(&path).unwrap(), b"previous bytes"); + } + std::fs::remove_file(path).unwrap(); + } + + pub(crate) fn assert_status_errors( + verb: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, + args: &[String], + writes: bool, + ) { + for flush_only in [false, true] { + let mut bad = Fails { flush_only }; + let mut err = Vec::new(); + assert_eq!( + run_with_writers(verb, args, &mut bad, &mut err), + u8::from(writes || flush_only), + "{}", + String::from_utf8_lossy(&err) + ); + } + } + + #[test] + fn added_verbs_propagate_write_and_flush_errors() { + let verbs = [fixture::cli, evidence::cli, renovate::cli]; + for verb in verbs { + for flush_only in [false, true] { + let mut bad = Fails { flush_only }; + let mut err = Vec::new(); + assert_eq!( + run_with_writers(verb, &["--help".into()], &mut bad, &mut err), + 1 + ); + assert!(String::from_utf8(err).unwrap().contains("stdout:")); + } + } + } + #[test] + fn shared_runner_checks_stderr_and_an_empty_stdout() { + fn clean(_: &[String], _: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "status"); + 0 + } + for flush_only in [false, true] { + assert_eq!( + run_with_writers(clean, &[], &mut Vec::new(), &mut Fails { flush_only }), + 1 + ); + } + assert_eq!( + run_with_writers(clean, &[], &mut Fails { flush_only: true }, &mut Vec::new()), + 1 + ); + } +} diff --git a/rust/crates/og-dev/src/lookaround.rs b/rust/crates/og-dev/src/lookaround.rs new file mode 100644 index 000000000..75e27dcb1 --- /dev/null +++ b/rust/crates/og-dev/src/lookaround.rs @@ -0,0 +1,915 @@ +//! Image rules keep their original linear matcher. Personal-data rules use +//! a Python-subset compiler and a backtracking matcher with assertions. +//! +//! Every look-around of the image scanner is a single character class: "not +//! after one of these" before the match and "not followed by one of these" +//! after it. A [`Rule`] keeps the matched text in capture group 1 and +//! writes the look-ahead as a consumed `(?:[^...]|$)` after it, so the +//! engine still finds, at each start, the match Python's backtracking would +//! pick. The look-behind is the character before group 1, checked in code; +//! when it fails, the search starts again one character later, as Python +//! tries the next start. The next match is searched from the end of group +//! 1, so the character the look-ahead read is not used up. + +use regex::Regex; + +/// A pattern with its look-behind. +pub struct Rule { + re: Regex, + /// True for a character the match must not follow. + not_after: Option bool>, +} + +impl Rule { + /// `pattern` holds the match in group 1, or is the match when it has no + /// group. + pub fn new(pattern: &str, not_after: Option bool>) -> Result { + Ok(Rule { + re: Regex::new(pattern)?, + not_after, + }) + } + + /// The spans of the image rule over `line`, in order. + pub fn find_iter<'a>(&'a self, line: &'a str) -> impl Iterator + 'a { + let mut pos = 0; + std::iter::from_fn(move || { + while pos <= line.len() { + let caps = self.re.captures_at(line, pos)?; + let m = caps.get(1).or_else(|| caps.get(0))?; + let (start, end) = (m.start(), m.end()); + let refused = self + .not_after + .is_some_and(|bad| line[..start].chars().next_back().is_some_and(bad)); + if refused { + pos = next_char(line, start); + continue; + } + pos = if end > start { + end + } else { + next_char(line, end) + }; + return Some((start, end)); + } + None + }) + } +} + +/// The byte offset of the character after `i`, or one past the end. +fn next_char(s: &str, i: usize) -> usize { + s[i..] + .chars() + .next() + .map_or(s.len() + 1, |c| i + c.len_utf8()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digit_or_dot(c: char) -> bool { + c.is_ascii_digit() || c == '.' + } + + #[test] + fn behind_and_ahead_as_python() { + // Python: re.finditer(r'(?(r: &Rule, s: &'a str) -> Vec<&'a str> { + r.find_iter(s).map(|(a, b)| &s[a..b]).collect() + } + assert_eq!(spans(&r, "17/19"), ["17/19"]); + assert_eq!(spans(&r, "1.17/19 x"), Vec::<&str>::new()); + assert_eq!(spans(&r, "17/19/3"), ["19/3"]); + assert_eq!(spans(&r, "a 17/19 and 2/3"), ["17/19", "2/3"]); + // The look-ahead character is not used up: two matches side by side. + let w = Rule::new(r"(ab)(?:[^b]|$)", None).unwrap(); + assert_eq!(w.find_iter("abab").count(), 2); + } + + #[test] + fn a_shorter_match_where_the_longest_fails_ahead() { + // Python finds "a/b:1" in "a/b:1.2/x": the tag backs off to "1", + // which a "." follows. + let r = Rule::new(r"([a-z]+/[a-z]+:[0-9][0-9.]*)(?:[^0-9a-z/]|$)", None).unwrap(); + let s = "a/b:1.2/x"; + let got: Vec<_> = r.find_iter(s).map(|(a, b)| &s[a..b]).collect(); + assert_eq!(got, ["a/b:1"]); + } +} + +// Personal-data patterns have one translation and one search path. Assertions +// stay in the engine so a failed assertion can backtrack at the same start. +const WORD: &str = r"[\p{L}\p{N}_]"; +const SPACE: &str = r"[\x09-\x0d\x1c-\x20\x85\x{a0}\x{1680}\x{2000}-\x{200a}\x{2028}\x{2029}\x{202f}\x{205f}\x{3000}]"; + +/// Why a pattern was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// It names, under an active `i` flag, a letter whose case pairing + /// differs between Unicode versions. + CaseDependent, + Other, +} + +#[derive(Clone)] +pub struct PythonRule { + re: fancy_regex::Regex, + after_empty: fancy_regex::Regex, +} + +impl PythonRule { + pub fn compile(pattern: &str, builtin: bool) -> Result { + Self::compile_explained(pattern, builtin).map_err(|_| ()) + } + + /// As `compile`, and says when the refusal is the case-pairing guard, so + /// the caller can tell the user what to change. + pub fn compile_explained(pattern: &str, builtin: bool) -> Result { + Self::compile_inner(pattern, builtin).map_err(|case| { + if case { + Refusal::CaseDependent + } else { + Refusal::Other + } + }) + } + + /// The error is true when the case-pairing guard refused the pattern. + fn compile_inner(pattern: &str, builtin: bool) -> Result { + if pattern.chars().any(crate::personal_data::version_dependent) { + return Err(false); + } + if pattern.len() > 16_384 { + return Err(false); + } + let mut parser = PythonPattern { + chars: pattern.chars().collect(), + at: 0, + builtin, + depth: 0, + names: std::collections::HashSet::new(), + case_refused: false, + }; + let (translated, _) = parser + .group(Flags::default(), false) + .map_err(|()| parser.case_refused)?; + // \G is the search offset. After yielding an empty match, forbid only + // a match ending at that offset; the engine then tries nonempty + // alternatives at the same start, exactly as Python finditer does. + let after_empty = format!(r"(?:{translated})(?!\G)"); + let build = |p: &str| { + fancy_regex::RegexBuilder::new(p) + .backtrack_limit(1_000_000) + .build() + .map_err(|_| false) + }; + Ok(Self { + re: build(&translated)?, + after_empty: build(&after_empty)?, + }) + } + + pub fn find_iter<'a>( + &'a self, + text: &'a str, + ) -> impl Iterator> + 'a { + let (mut pos, mut empty, mut done) = (0, false, false); + std::iter::from_fn(move || { + if done { + return None; + } + let re = if empty { &self.after_empty } else { &self.re }; + match re.find_from_pos(text, pos) { + Ok(Some(m)) => { + pos = m.end(); + empty = m.start() == m.end(); + Some(Ok((m.start(), m.end()))) + } + Ok(None) => { + done = true; + None + } + Err(_) => { + done = true; + Some(Err(())) + } + } + }) + } + + pub fn is_match(&self, text: &str) -> Result { + self.find_iter(text).next().transpose().map(|m| m.is_some()) + } +} + +#[derive(Clone, Copy, Default)] +struct Flags { + i: bool, + m: bool, + s: bool, + x: bool, +} + +struct PythonPattern { + chars: Vec, + at: usize, + builtin: bool, + depth: usize, + names: std::collections::HashSet, + /// Set when a refusal comes from the case-pairing guard. + case_refused: bool, +} + +impl PythonPattern { + /// `literal`, recording a refusal by the case-pairing guard. + fn lit(&mut self, c: char, ci: bool) -> Result { + self.case_refused |= ci && crate::personal_data::range_regex_case_dependent(c, c); + literal(c, ci) + } + + fn take(&mut self) -> Result { + let c = self.chars.get(self.at).copied().ok_or(())?; + self.at += 1; + Ok(c) + } + + fn peek(&self) -> Option { + self.chars.get(self.at).copied() + } + + fn group(&mut self, mut flags: Flags, nested: bool) -> Result<(String, bool), ()> { + let mut out = String::new(); + // Nullability is structural: concatenate with AND, alternate with OR. + // Keep the last atom separate until its quantifier is known. + let (mut prefix_nullable, mut atom_nullable, mut alt_nullable) = (true, true, false); + let (mut repeatable, mut lazy) = (false, false); + while let Some(c) = self.peek() { + self.at += 1; + if flags.x && matches!(c, ' ' | '\t' | '\n' | '\r' | '\u{b}' | '\u{c}') { + continue; + } + if flags.x && c == '#' { + while self.peek().is_some_and(|c| c != '\n') { + self.at += 1; + } + continue; + } + if !matches!(c, ')' | '|' | '*' | '+' | '?' | '{') { + prefix_nullable &= atom_nullable; + atom_nullable = true; + } + match c { + ')' if nested => { + return Ok((out, alt_nullable || (prefix_nullable && atom_nullable))); + } + ')' => return Err(()), + '(' => { + repeatable = true; + lazy = false; + if self.depth >= 64 { + return Err(()); + } + let mut prefix = "?:".to_owned(); + let mut inner = flags; + if self.peek() == Some('?') { + self.at += 1; + match self.take()? { + ':' => (), + '=' if self.builtin => prefix = "?=".into(), + '!' if self.builtin => prefix = "?!".into(), + '<' if self.builtin => { + let c = self.take()?; + if !matches!(c, '=' | '!') { + return Err(()); + } + prefix = format!("?<{c}"); + } + 'P' => { + if self.take()? != '<' { + return Err(()); + } + let mut name = String::new(); + loop { + let c = self.take()?; + if c == '>' { + break; + } + name.push(c); + } + if name.is_empty() + || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + || name.starts_with(|c: char| c.is_ascii_digit()) + || !self.names.insert(name.clone()) + { + return Err(()); + } + } + c if "imsx-".contains(c) => { + self.at -= 1; + let mut on = true; + let mut seen = std::collections::HashSet::new(); + let mut disabled = false; + loop { + let flag = self.take()?; + if matches!(flag, 'i' | 'm' | 's' | 'x') { + if !seen.insert(flag) { + return Err(()); + } + disabled |= !on; + } + match flag { + '-' if on => on = false, + 'i' => inner.i = on, + 'm' => inner.m = on, + 's' => inner.s = on, + 'x' => inner.x = on, + ':' if !seen.is_empty() && (on || disabled) => break, + ')' if on + && !seen.is_empty() + && out.is_empty() + && !nested => + { + flags = inner; + break; + } + _ => return Err(()), + } + if self.chars[self.at - 1] == ')' { + break; + } + } + if self.chars[self.at - 1] == ')' { + repeatable = false; + continue; + } + } + _ => return Err(()), + } + } + out.push('('); + out.push_str(&prefix); + self.depth += 1; + let (body, nullable) = self.group(inner, true)?; + atom_nullable = prefix != "?:" || nullable; + out.push_str(&body); + self.depth -= 1; + out.push(')'); + } + '[' => { + out.push_str(&self.class(flags.i)?); + atom_nullable = false; + repeatable = true; + lazy = false; + } + '\\' => { + repeatable = !matches!(self.peek(), Some('b' | 'B' | 'A' | 'Z')); + atom_nullable = !repeatable; + lazy = false; + out.push_str(&self.escape(flags.i, false)?); + } + '.' => { + out.push_str(if flags.s { r"[\s\S]" } else { r"[^\n]" }); + atom_nullable = false; + repeatable = true; + lazy = false; + } + '^' => { + out.push_str(if flags.m { r"(?:\A|(?<=\n))" } else { r"\A" }); + repeatable = false; + lazy = false; + } + '$' => { + out.push_str(if flags.m { r"(?=\n|\z)" } else { r"(?=\n?\z)" }); + repeatable = false; + lazy = false; + } + '|' => { + alt_nullable |= prefix_nullable && atom_nullable; + prefix_nullable = true; + atom_nullable = true; + out.push('|'); + repeatable = false; + lazy = false; + } + '*' | '+' | '?' => { + if c == '?' && lazy { + out.push(c); + lazy = false; + continue; + } + if !repeatable || atom_nullable { + return Err(()); + } + repeatable = false; + lazy = true; + // Possessive quantifiers are outside the supported subset. + if matches!(c, '*' | '+' | '?') && self.peek() == Some('+') { + return Err(()); + } + atom_nullable = c != '+'; + out.push(c); + } + '{' => { + if !repeatable || atom_nullable { + return Err(()); + } + repeatable = false; + lazy = true; + out.push(c); + let minimum_at = self.at; + loop { + let c = self.take()?; + out.push(c); + if c == '}' { + break; + } + if !c.is_ascii_digit() && c != ',' { + return Err(()); + } + } + // Python permits {,n}; a missing minimum means zero. + atom_nullable = self.chars[minimum_at..self.at - 1] + .iter() + .take_while(|&&c| c != ',') + .all(|&c| c == '0'); + if self.peek() == Some('+') { + return Err(()); + } + } + '}' => return Err(()), + c => { + out.push_str(&self.lit(c, flags.i)?); + atom_nullable = false; + repeatable = true; + lazy = false; + } + } + } + if nested { + Err(()) + } else { + Ok((out, alt_nullable || (prefix_nullable && atom_nullable))) + } + } + + fn escape(&mut self, ci: bool, class: bool) -> Result { + let c = self.take()?; + Ok(match c { + 'w' => WORD.into(), + 'W' => r"[^\p{L}\p{N}_]".into(), + 's' => SPACE.into(), + 'S' => format!("[^{SPACE}]"), + 'd' => r"[\p{Nd}]".into(), + 'D' => r"[^\p{Nd}]".into(), + 'b' if class => r"\x08".into(), + 'b' => format!(r"(?:(? format!(r"(?:(?<={WORD})(?={WORD})|(? r"\A".into(), + 'Z' if !class => r"\z".into(), + 'a' => r"\x07".into(), + 'f' => r"\x0c".into(), + 'n' => r"\x0a".into(), + 'r' => r"\x0d".into(), + 't' => r"\x09".into(), + 'v' => r"\x0b".into(), + 'x' | 'u' | 'U' => { + let n = match c { + 'x' => 2, + 'u' => 4, + _ => 8, + }; + let mut hex = String::new(); + for _ in 0..n { + let c = self.take()?; + if !c.is_ascii_hexdigit() { + return Err(()); + } + hex.push(c); + } + let c = char::from_u32(u32::from_str_radix(&hex, 16).map_err(|_| ())?).ok_or(())?; + self.lit(c, ci)? + } + c if c.is_ascii_alphanumeric() => return Err(()), + c => self.lit(c, ci)?, + }) + } + + fn class(&mut self, ci: bool) -> Result { + let negative = self.peek() == Some('^'); + if negative { + self.at += 1; + } + let mut atoms = Vec::new(); + let mut first = true; + loop { + let c = self.take()?; + if c == ']' && !first { + break; + } + first = false; + match c { + '\\' => atoms.push((self.escape(false, true)?, false)), + '[' => return Err(()), + '-' => atoms.push((literal('-', false)?, true)), + c => atoms.push((literal(c, false)?, false)), + } + } + let mut body = String::new(); + let mut i = 0; + while i < atoms.len() { + let atom = &atoms[i].0; + if i + 2 < atoms.len() && atoms[i + 1].1 { + // Categories are not case folded in Python, even inside + // classes. Only literal range endpoints can form a range. + let start = class_literal(atom).ok_or(())?; + let end = class_literal(&atoms[i + 2].0).ok_or(())?; + self.case_refused |= + ci && crate::personal_data::range_regex_case_dependent(start, end); + if start > end + || crate::personal_data::range_version_dependent(start, end) + || self.case_refused + { + return Err(()); + } + let range = format!("[{atom}-{}]", atoms[i + 2].0); + body.push_str(&if ci { fold_class(&range)? } else { range }); + i += 3; + } else { + if atoms[i].1 && i != 0 && i + 1 != atoms.len() { + return Err(()); + } + if ci && let Some(c) = class_literal(atom) { + if crate::personal_data::range_regex_case_dependent(c, c) { + self.case_refused = true; + return Err(()); + } + body.push_str(&fold_class(&format!("[{atom}]"))?); + } else { + body.push_str(atom); + } + i += 1; + } + } + Ok(format!("[{}{body}]", if negative { "^" } else { "" })) + } +} + +fn class_literal(atom: &str) -> Option { + let hir = regex_syntax::Parser::new().parse(atom).ok()?; + let regex_syntax::hir::HirKind::Literal(value) = hir.kind() else { + return None; + }; + let text = std::str::from_utf8(&value.0).ok()?; + (text.chars().count() == 1) + .then(|| text.chars().next()) + .flatten() +} + +fn literal(c: char, ci: bool) -> Result { + if crate::personal_data::version_dependent(c) + || (ci && crate::personal_data::range_regex_case_dependent(c, c)) + { + return Err(()); + } + let escaped = format!(r"\x{{{:x}}}", u32::from(c)); + if ci { + fold_class(&format!("[{escaped}]")) + } else { + Ok(escaped) + } +} + +fn fold_class(class: &str) -> Result { + use regex_syntax::hir::{Class, HirKind}; + // Rust's simple fold follows Python except the extra dotted/dotless I. + let hir = regex_syntax::ParserBuilder::new() + .case_insensitive(true) + .build() + .parse(class) + .map_err(|_| ())?; + let set = match hir.kind() { + HirKind::Class(Class::Unicode(set)) => set.clone(), + HirKind::Literal(value) => { + let text = std::str::from_utf8(&value.0).map_err(|_| ())?; + let c = text.chars().next().ok_or(())?; + if text.chars().count() != 1 { + return Err(()); + } + regex_syntax::hir::ClassUnicode::new([regex_syntax::hir::ClassUnicodeRange::new(c, c)]) + } + _ => return Err(()), + }; + let mut out = String::from("["); + let mut i = false; + for range in set.iter() { + out.push_str(&format!( + r"\x{{{:x}}}-\x{{{:x}}}", + u32::from(range.start()), + u32::from(range.end()) + )); + i |= range.start() <= 'I' && 'I' <= range.end() + || range.start() <= 'i' && 'i' <= range.end() + || range.start() <= 'İ' && 'İ' <= range.end() + || range.start() <= 'ı' && 'ı' <= range.end(); + } + if i { + out.push_str(r"Ii\x{130}\x{131}"); + } + out.push(']'); + Ok(out) +} + +#[cfg(test)] +mod python_tests { + use super::*; + + #[test] + fn finditer_spans_match_real_python() { + for pattern in [ + r"^|a", + r"a*?", + r"a|", + r"\w*", + r"\b|\W+", + r"a$", + r"(?m)^|$", + r"(?s:.)*?", + r"(?i)[^I]+", + r"(?i:İ|ı|S|k)+", + r"[\s\S]", + r"(?i)[^\W]+", + r"(?i)[\w]+", + r"\D*\d?", + r"\Bz", + r"(?x) a + # comment", + ] { + for text in ["", "aa", "a\n", "a\n\n", "\u{301}aİıſK9\u{1c}\u{345}Ιι"] { + let output = std::process::Command::new("python3").args(["-I", "-c", + "import re,sys; print(','.join(str(m.start())+':'+str(m.end()) for m in re.finditer(sys.argv[1],sys.argv[2])))", pattern, text]).output().unwrap(); + assert!(output.status.success(), "span oracle failed"); + let rule = PythonRule::compile(pattern, false).expect("supported span pattern"); + let actual = rule + .find_iter(text) + .map(|m| { + let (a, b) = m.unwrap(); + format!( + "{}:{}", + text[..a].chars().count(), + text[..b].chars().count() + ) + }) + .collect::>() + .join(","); + assert_eq!(actual, String::from_utf8(output.stdout).unwrap().trim()); + } + } + } + + #[test] + fn round3_nullable_atoms_are_refused() { + for (pattern, text, spans) in [ + ("(a*|b*?){1,3}", "ba", "0:0,0:2,2:2"), + ("(?:a*|[ab]{0,2}){0,3}", "baa", "0:0,0:3,3:3"), + ] { + let output = std::process::Command::new("python3").args(["-I", "-c", + "import re,sys; print(','.join(str(m.start())+':'+str(m.end()) for m in re.finditer(sys.argv[1],sys.argv[2])))", pattern, text]).output().unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), spans); + assert!( + PythonRule::compile(pattern, false).is_err(), + "unsupported span case accepted" + ); + } + for pattern in [ + "(a*|b*?){1,3}", + "(?:a*|[ab]{0,2}){0,3}", + "(?:a?)+", + "(?:a*)?", + "(?:|a)*?", + "()*", + "(?:a{0}){2}", + "(?i:(?:a?|b)){1,2}", + "(?:^|a)+", + "(?:a|$)?", + ] { + assert!( + PythonRule::compile(pattern, false).is_err(), + "nullable operand accepted" + ); + } + } + + // A finite grammar, exhaustively enumerated by AST node count. Every + // terminal costs one node; each unary/binary operator costs one more. + // No random selection or accepted-pattern sampling occurs in this model. + const MODEL_ATOMS: &[&str] = &[ + "", "a", "b", "0", " ", "\u{301}", ".", "[ab]", "[^ab]", "[a-b]", r"\w", r"\W", r"\d", + r"\D", r"\s", r"\S", r"\b", r"\B", "^", "$", r"\A", r"\Z", r"\x61", r"\u0301", r"[\s\S]", + r"[\w0]", + ]; + const MODEL_GROUPS: &[&str] = &["", "?:", "?P", "?i:", "?m:", "?s:", "?x:", "?-i:"]; + const MODEL_QUANTIFIERS: &[&str] = &[ + "?", "*", "+", "??", "*?", "+?", "{0}", "{1}", "{0,2}", "{1,3}", "{2,}", "{,2}", "{0,2}?", + "{1,3}?", + ]; + + fn unary_patterns(p: &str) -> impl Iterator + '_ { + MODEL_GROUPS + .iter() + .map(move |g| format!("({g}{p})")) + .chain(MODEL_QUANTIFIERS.iter().map(move |q| format!("(?:{p}){q}"))) + } + + fn model_patterns(size: usize) -> Vec { + use std::collections::BTreeSet; + let mut levels = vec![ + BTreeSet::new(), + MODEL_ATOMS + .iter() + .map(|s| s.to_string()) + .collect::>(), + ]; + for n in 2..=size { + let mut level = BTreeSet::new(); + for p in &levels[n - 1] { + level.extend(unary_patterns(p)); + } + for left in 1..n - 1 { + for a in &levels[left] { + for b in &levels[n - 1 - left] { + level.insert(format!("(?:{a})(?:{b})")); + level.insert(format!("(?:{a}|{b})")); + } + } + } + levels.push(level); + } + levels + .into_iter() + .flatten() + .collect::>() + .into_iter() + .collect() + } + + fn model_texts(length: usize) -> Vec { + let mut all = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..length { + level = level + .iter() + .flat_map(|p| ['a', 'b', '0', ' ', '\u{301}', 'é'].map(|c| format!("{p}{c}"))) + .collect(); + all.extend(level.iter().cloned()); + } + all + } + + fn span_list(rule: &PythonRule, text: &str) -> String { + rule.find_iter(text) + .map(|m| { + let (a, b) = m.expect("model matcher exhausted"); + format!( + "{}:{}", + text[..a].chars().count(), + text[..b].chars().count() + ) + }) + .collect::>() + .join(",") + } + + fn compare_model(label: &str, patterns: &[String], texts: &[String]) -> usize { + use std::io::Write; + use std::process::{Command, Stdio}; + let (mut accepted, mut refused) = (0, 0); + for (batch, chunk) in patterns.chunks(128).enumerate() { + let rules: Vec<_> = chunk + .iter() + .filter_map(|p| match PythonRule::compile(p, false) { + Ok(r) => Some((p, r)), + Err(()) => { + refused += 1; + None + } + }) + .collect(); + if rules.is_empty() { + continue; + } + accepted += rules.len(); + let array = |items: Vec<&str>| { + let mut out = String::from("["); + for (i, item) in items.iter().enumerate() { + if i != 0 { + out.push(','); + } + pycompat::json::quote(&mut out, item, true); + } + out.push(']'); + out + }; + let input = format!( + "[{},{}]", + array(rules.iter().map(|(p, _)| p.as_str()).collect()), + array(texts.iter().map(String::as_str).collect()) + ); + let mut child = Command::new("python3") + .args([ + "-I", + "-c", + r#" +import json,re,sys +patterns,texts=json.load(sys.stdin) +for p in patterns: + r=re.compile(p) + for text in texts: + print(','.join(str(m.start())+':'+str(m.end()) for m in r.finditer(text))) +"#, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let writer = std::thread::spawn(move || stdin.write_all(input.as_bytes())); + let output = child.wait_with_output().unwrap(); + writer.join().unwrap().unwrap(); + assert!(output.status.success(), "model oracle failed"); + assert!(output.stderr.is_empty(), "model oracle diagnostic"); + let oracle = String::from_utf8(output.stdout).unwrap(); + let mut spans = oracle.lines(); + for (index, (_, rule)) in rules.iter().enumerate() { + for (t, text) in texts.iter().enumerate() { + assert_eq!( + Some(span_list(rule, text).as_str()), + spans.next(), + "{label}: batch {batch}, accepted pattern {index}, text {t}" + ); + } + } + assert!(spans.next().is_none(), "extra oracle rows"); + } + let pairs = accepted * texts.len(); + println!( + "{label}: {} patterns, {accepted} accepted, {refused} refused, {} texts, {pairs} pairs, zero span differences", + patterns.len(), + texts.len() + ); + pairs + } + + #[test] + fn exhaustive_small_model_spans() { + // CI: every <=2-node pattern against all <=2-character texts. + assert!(compare_model("exhaustive-ci", &model_patterns(2), &model_texts(2)) > 10_000); + } + + #[test] + #[ignore = "large local model; run explicitly with --ignored --nocapture"] + fn exhaustive_large_model_spans() { + // Local: every <=3-node pattern against all <=3-character texts. + assert!(compare_model("exhaustive-local", &model_patterns(3), &model_texts(3)) > 1_000_000); + } + + fn fuzz_pattern(state: &mut u64, depth: usize) -> String { + fn next(state: &mut u64) -> usize { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state as usize + } + if depth == 0 || next(state).is_multiple_of(4) { + return MODEL_ATOMS[next(state) % MODEL_ATOMS.len()].into(); + } + let p = fuzz_pattern(state, depth - 1); + match next(state) % 4 { + 0 => format!("({}{p})", MODEL_GROUPS[next(state) % MODEL_GROUPS.len()]), + 1 => format!( + "(?:{p}){}", + MODEL_QUANTIFIERS[next(state) % MODEL_QUANTIFIERS.len()] + ), + 2 => format!("(?:{p}|{})", fuzz_pattern(state, depth - 1)), + _ => format!("(?:{p})(?:{})", fuzz_pattern(state, depth - 1)), + } + } + + #[test] + fn round3_two_seed_span_fuzz() { + for mut seed in [0x401ad33, 0x3ad401] { + let label = format!("fuzz-{seed:x}"); + let mut patterns = Vec::new(); + // Focus the same seeds on nullable alternatives and nested repeats. + for _ in 0..1500 { + let p = fuzz_pattern(&mut seed, 4); + patterns.push(p.clone()); + patterns.push(format!("(?:{p}|a*?){{1,3}}")); + } + assert!(compare_model(&label, &patterns, &model_texts(3)) > 50_000); + } + } +} diff --git a/rust/crates/og-dev/src/nf_trace.rs b/rust/crates/og-dev/src/nf_trace.rs new file mode 100644 index 000000000..2015c9e0d --- /dev/null +++ b/rust/crates/og-dev/src/nf_trace.rs @@ -0,0 +1,323 @@ +//! Strict checks over raw Nextflow traces. Only one explicit trace path is accepted. +use pycompat::csv::{Quoting, Reader}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; + +const GB: u64 = 1 << 30; +const H: u64 = 3_600_000; +type Record = BTreeMap; + +fn g(value: u64, unit: u64) -> String { + pycompat::fmt::general(value as f64 / unit as f64, 6) +} +fn records(path: &str, mut f: impl FnMut(Record) -> Result<(), String>) -> Result<(), String> { + if path.contains(['*', '?', '[', ']']) { + return Err("trace glob refused: pass exactly one explicit trace path".into()); + } + let lines = super::smoke::csv_lines(path)?; + let mut csv = Reader::new(lines, '\t', Quoting::Minimal); + let header = csv + .next() + .ok_or("no trace header")? + .map_err(|e| e.to_string())?; + for row in csv { + let row = row.map_err(|e| e.to_string())?; + if row.is_empty() { + continue; + } + if row.len() != header.len() { + return Err(format!("{path}: row width does not match header")); + } + f(header.iter().cloned().zip(row).collect())?; + } + Ok(()) +} +fn cell<'a>(r: &'a Record, k: &str) -> Result<&'a str, String> { + r.get(k) + .map(String::as_str) + .ok_or_else(|| format!("missing column {k}")) +} +fn num(r: &Record, k: &str) -> Result { + pycompat::num::parse_int(cell(r, k)?) + .ok() + .and_then(|v| u64::try_from(v).ok()) + .ok_or_else(|| format!("column {k} is not an unsigned ASCII integer")) +} +fn traces(path: &str, out: &mut dyn Write) -> Result, String> { + let mut rows: Vec = Vec::new(); + let mut ids = BTreeMap::new(); + records(path, |r| { + let id = cell(&r, "task_id")?.to_owned(); + if let Some(&i) = ids.get(&id) { + let first: &mut Record = &mut rows[i]; + let a = cell(first, "exit")?; + let b = cell(&r, "exit")?; + if (a != "-" && b != "-" && a != b) + || r.iter() + .any(|(k, v)| k != "exit" && first.get(k) != Some(v)) + { + return Err(format!("task {id} has two different rows in the trace")); + } + if a == "-" && b != "-" { + first.insert("exit".into(), b.into()); + } + writeln!( + out, + "{}: attempt {} is written twice in the trace; counted once", + cell(&r, "name")?, + cell(&r, "attempt")? + ) + .map_err(|e| e.to_string())?; + } else { + ids.insert(id, rows.len()); + rows.push(r); + } + Ok(()) + })?; + if rows.is_empty() { + return Err("no trace rows".into()); + } + Ok(rows) +} +fn options(args: &[String]) -> Result, String> { + if !args.len().is_multiple_of(2) { + return Err("expected named option and value; pass exactly one trace path".into()); + } + let mut opts = BTreeMap::new(); + for pair in args.as_chunks::<2>().0 { + if !pair[0].starts_with("--") || opts.insert(pair[0].as_str(), pair[1].as_str()).is_some() { + return Err("unexpected or repeated argument".into()); + } + } + Ok(opts) +} +fn opt<'a>(opts: &BTreeMap<&str, &'a str>, key: &str) -> Result<&'a str, String> { + opts.get(key) + .copied() + .ok_or_else(|| format!("{key} is required")) +} +fn glob(pattern: &str, text: &str) -> bool { + // Escaping makes punctuation literal. Only '*' and '?' have glob meaning. + let re = regex::escape(pattern) + .replace("\\*", ".*") + .replace("\\?", "."); + regex::Regex::new(&format!("(?s)^{re}$")).is_ok_and(|r| r.is_match(text)) +} +fn attempts(path: &str, opts: &BTreeMap<&str, &str>, out: &mut dyn Write) -> Result { + for k in opts.keys() { + if !["--name-glob", "--want", "--nextflow-exit"].contains(k) { + return Err(format!("unknown option {k}")); + } + } + let pattern = opt(opts, "--name-glob")?; + let want: Vec<(u64, String, u64)> = opt(opts, "--want")? + .split(',') + .map(|t| { + let parts: Vec<_> = t.split(':').collect(); + if parts.len() != 3 { + return Err("--want must be attempt:exit:GB,...".into()); + } + Ok(( + parts[0].parse().map_err(|_| "bad attempt")?, + parts[1].to_owned(), + parts[2].parse().map_err(|_| "bad memory")?, + )) + }) + .collect::>()?; + let mut got = Vec::new(); + for r in traces(path, out)? { + if glob(pattern, cell(&r, "name")?) { + got.push(( + num(&r, "attempt")?, + cell(&r, "exit")?.to_owned(), + num(&r, "memory")? / GB, + )); + } + } + got.sort(); + let tuples = |xs: &[(u64, String, u64)]| { + format!( + "[{}]", + xs.iter() + .map(|(a, e, m)| format!("({a}, '{e}', {m})")) + .collect::>() + .join(", ") + ) + }; + let label = if pattern.contains("ALIGN_MINIMAP2") { + "ALIGN_MINIMAP2" + } else { + pattern + }; + writeln!( + out, + "{label} attempts (attempt, exit, GB): {}; want {}", + tuples(&got), + tuples(&want) + ) + .map_err(|e| e.to_string())?; + let ok = got == want && opt(opts, "--nextflow-exit")? == "0"; + writeln!( + out, + "{}", + if ok { + "OK" + } else { + "FAIL: a memory kill did not reach Nextflow as exit 137 and a retry that passes" + } + ) + .map_err(|e| e.to_string())?; + Ok(ok) +} +fn requests(path: &str, opts: &BTreeMap<&str, &str>, out: &mut dyn Write) -> Result { + for k in opts.keys() { + if !["--want", "--executor-cpus", "--executor-mem-gb"].contains(k) { + return Err(format!("unknown option {k}")); + } + } + let mut want = BTreeMap::new(); + records(opt(opts, "--want")?, |r| { + let name = cell(&r, "process")?.to_owned(); + let v = ( + num(&r, "cpus")?, + num(&r, "memory_gb")? + .checked_mul(GB) + .ok_or("memory overflow")?, + num(&r, "time_h")?.checked_mul(H).ok_or("time overflow")?, + ); + if want.insert(name, v).is_some() { + return Err("duplicate process in --want".into()); + } + Ok(()) + })?; + if want.is_empty() { + return Err("empty request expectations".into()); + } + let rows = traces(path, out)?; + type Seen<'a> = BTreeMap<&'a str, BTreeMap>>>; + let mut seen: Seen<'_> = BTreeMap::new(); + let mut all_cpus = 0u64; + let mut all_mem = 0u64; + let mut bad = 0; + for r in &rows { + let cpus = num(r, "cpus")?; + let mem = num(r, "memory")?; + all_cpus = all_cpus.checked_add(cpus).ok_or("CPU sum overflow")?; + all_mem = all_mem.checked_add(mem).ok_or("memory sum overflow")?; + let name = cell(r, "name")?; + let proc = name + .split(" (") + .next() + .unwrap_or(name) + .rsplit(':') + .next() + .unwrap_or(name); + if !want.contains_key(proc) { + continue; + } + let attempt = num(r, "attempt")?; + let time = num(r, "time")?; + writeln!( + out, + "{name}: attempt {attempt}, exit {}, cpus {cpus}, memory {} GB, time {} h", + cell(r, "exit")?, + g(mem, GB), + g(time, H) + ) + .map_err(|e| e.to_string())?; + seen.entry(proc) + .or_default() + .entry(attempt) + .or_default() + .entry(name) + .or_default() + .push((cpus, mem, time)); + } + for (proc, &expected) in &want { + let attempts = seen.get(proc.as_str()); + if attempts.is_some_and(|a| a.keys().any(|k| ![1, 2].contains(k))) { + writeln!(out, "FAIL {proc}: want only a first attempt and one retry") + .map_err(|e| e.to_string())?; + bad += 1; + } + let first = attempts.and_then(|a| a.get(&1)); + let second = attempts.and_then(|a| a.get(&2)); + if first.is_none_or(|a| a.is_empty()) { + writeln!(out, "FAIL {proc}: no first attempt in the trace") + .map_err(|e| e.to_string())?; + bad += 1; + continue; + } + let names: BTreeSet<_> = first + .into_iter() + .chain(second) + .flat_map(|a| a.keys()) + .collect(); + for name in names { + let a = first.and_then(|a| a.get(name)).map_or(0, Vec::len); + let b = second.and_then(|a| a.get(name)).map_or(0, Vec::len); + if a != 1 || b != 1 { + writeln!( + out, + "FAIL {name}: {a} first and {b} second attempts in the trace, want one of each" + ) + .map_err(|e| e.to_string())?; + bad += 1; + } + } + for got in first.into_iter().flat_map(|a| a.values()).flatten() { + if *got != expected { + writeln!(out,"FAIL {proc}: first attempt asks for cpus {}, memory {} GB, time {} h; want {}, {} GB, {} h",got.0,g(got.1,GB),g(got.2,H),expected.0,g(expected.1,GB),g(expected.2,H)).map_err(|e| e.to_string())?; + bad += 1; + } + } + for got in second.into_iter().flat_map(|a| a.values()).flatten() { + if Some(got.1) != expected.1.checked_mul(2) { + writeln!( + out, + "FAIL {proc}: the retry asks for {} GB, want {} GB", + g(got.1, GB), + pycompat::fmt::general(expected.1 as f64 * 2. / GB as f64, 6) + ) + .map_err(|e| e.to_string())?; + bad += 1; + } + } + } + let cpus: u64 = opt(opts, "--executor-cpus")? + .parse() + .map_err(|_| "invalid executor CPUs")?; + let mem: u64 = opt(opts, "--executor-mem-gb")? + .parse() + .map_err(|_| "invalid executor memory")?; + writeln!(out,"{} task attempts ask for {all_cpus} CPUs and {} GB added together; the executor is told it has {cpus} CPUs and {} GB",rows.len(),g(all_mem,GB),pycompat::fmt::general(mem as f64,6)).map_err(|e| e.to_string())?; + if all_cpus > cpus || Some(all_mem) > mem.checked_mul(GB) { + writeln!(out,"FAIL: executor.cpus or executor.memory of this step is below that sum; raise exec_cpus / exec_mem_gb").map_err(|e| e.to_string())?; + bad += 1; + } + writeln!(out, "{bad} failure(s)").map_err(|e| e.to_string())?; + Ok(bad == 0) +} + +/// Checks `requests TRACE ...` or `attempts TRACE ...`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let result = (|| { + if args.len() < 2 { + return Err("usage: og dev nf-trace requests|attempts TRACE [options]".into()); + } + let opts = options(&args[2..])?; + match args[0].as_str() { + "requests" => requests(&args[1], &opts, out), + "attempts" => attempts(&args[1], &opts, out), + _ => Err("unknown nf-trace verb".into()), + } + })(); + match result { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + let _ = writeln!(err, "FAIL: {e}"); + 2 + } + } +} diff --git a/rust/crates/og-dev/src/personal_data.rs b/rust/crates/og-dev/src/personal_data.rs new file mode 100644 index 000000000..40159c19d --- /dev/null +++ b/rust/crates/og-dev/src/personal_data.rs @@ -0,0 +1,3555 @@ +//! `og dev scan personal-data`: fail when a tracked text file holds +//! something that looks like personal data. It replaces the Python inside +//! `scripts/ci/check-personal-data.sh`, which keeps the lists of documented +//! examples (`--allow`) and skipped files (`--skip`) and passes them in. +//! +//! ```text +//! og dev scan personal-data --root DIR --allow TEXT --skip TEXT +//! [--patterns FILE] [--cached] [--] [FILE...] +//! ``` +//! +//! With no FILE, every file `git ls-files` lists under DIR. `--cached` +//! reads each file's staged blob (one `git cat-file --batch` process) +//! instead of the working copy. `--patterns` adds one regular expression +//! per line (kind `own`). +//! +//! Each line of a text file (no NUL byte in its first 8 KiB) is matched +//! against the rules below; a match inside an allowed example of that file +//! is skipped, the rest of the line still counts. Output: one +//! `FAIL :` per rule and line, then +//! `scanned N text files`. Exit 0 clean, 1 on a finding or when no text +//! file was scanned, 2 on a usage error or a file or pattern that cannot be +//! read. +//! +//! Where it differs from the Python on purpose: a scan that read no text +//! file fails (`FAIL: no text file was scanned`) instead of passing, and a +//! file is placed under DIR after resolving symbolic links in both, so +//! DIR given as `/tmp/x` still finds `/private/tmp/x/CLAUDE.md`'s examples. + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::LazyLock; + +use pycompat::mtime::fnmatch; +use pycompat::num::py_strip; + +use crate::lookaround::{PythonRule as Rule, Refusal}; + +// Property drift and astral casing guard text, patterns and exception rows. +// BMP regex case drift is guarded separately, only where a pattern enables i. +// BEGIN GENERATED UNICODE GUARD +// Generated by personal-unicode-generator.rs; do not edit. +// UCD 13.0.0, 15.0.0, 16.0.0, 17.0.0; sources pinned in personal-unicode-sources.tsv. +const VERSION_DEPENDENT: &[(u32, u32)] = &[ + (0x19b, 0x19b), + (0x264, 0x264), + (0x378, 0x379), + (0x380, 0x383), + (0x38b, 0x38b), + (0x38d, 0x38d), + (0x3a2, 0x3a2), + (0x530, 0x530), + (0x557, 0x558), + (0x58b, 0x58c), + (0x590, 0x590), + (0x5c8, 0x5cf), + (0x5eb, 0x5ee), + (0x5f5, 0x5ff), + (0x70e, 0x70e), + (0x74b, 0x74c), + (0x7b2, 0x7bf), + (0x7fb, 0x7fc), + (0x82e, 0x82f), + (0x83f, 0x83f), + (0x85c, 0x85d), + (0x85f, 0x85f), + (0x86b, 0x887), + (0x889, 0x88f), + (0x892, 0x89f), + (0x8b5, 0x8b5), + (0x8c8, 0x8d2), + (0x984, 0x984), + (0x98d, 0x98e), + (0x991, 0x992), + (0x9a9, 0x9a9), + (0x9b1, 0x9b1), + (0x9b3, 0x9b5), + (0x9ba, 0x9bb), + (0x9c5, 0x9c6), + (0x9c9, 0x9ca), + (0x9cf, 0x9d6), + (0x9d8, 0x9db), + (0x9de, 0x9de), + (0x9e4, 0x9e5), + (0x9ff, 0xa00), + (0xa04, 0xa04), + (0xa0b, 0xa0e), + (0xa11, 0xa12), + (0xa29, 0xa29), + (0xa31, 0xa31), + (0xa34, 0xa34), + (0xa37, 0xa37), + (0xa3a, 0xa3b), + (0xa3d, 0xa3d), + (0xa43, 0xa46), + (0xa49, 0xa4a), + (0xa4e, 0xa50), + (0xa52, 0xa58), + (0xa5d, 0xa5d), + (0xa5f, 0xa65), + (0xa77, 0xa80), + (0xa84, 0xa84), + (0xa8e, 0xa8e), + (0xa92, 0xa92), + (0xaa9, 0xaa9), + (0xab1, 0xab1), + (0xab4, 0xab4), + (0xaba, 0xabb), + (0xac6, 0xac6), + (0xaca, 0xaca), + (0xace, 0xacf), + (0xad1, 0xadf), + (0xae4, 0xae5), + (0xaf2, 0xaf8), + (0xb00, 0xb00), + (0xb04, 0xb04), + (0xb0d, 0xb0e), + (0xb11, 0xb12), + (0xb29, 0xb29), + (0xb31, 0xb31), + (0xb34, 0xb34), + (0xb3a, 0xb3b), + (0xb45, 0xb46), + (0xb49, 0xb4a), + (0xb4e, 0xb54), + (0xb58, 0xb5b), + (0xb5e, 0xb5e), + (0xb64, 0xb65), + (0xb78, 0xb81), + (0xb84, 0xb84), + (0xb8b, 0xb8d), + (0xb91, 0xb91), + (0xb96, 0xb98), + (0xb9b, 0xb9b), + (0xb9d, 0xb9d), + (0xba0, 0xba2), + (0xba5, 0xba7), + (0xbab, 0xbad), + (0xbba, 0xbbd), + (0xbc3, 0xbc5), + (0xbc9, 0xbc9), + (0xbce, 0xbcf), + (0xbd1, 0xbd6), + (0xbd8, 0xbe5), + (0xbfb, 0xbff), + (0xc0d, 0xc0d), + (0xc11, 0xc11), + (0xc29, 0xc29), + (0xc3a, 0xc3c), + (0xc45, 0xc45), + (0xc49, 0xc49), + (0xc4e, 0xc54), + (0xc57, 0xc57), + (0xc5b, 0xc5f), + (0xc64, 0xc65), + (0xc70, 0xc76), + (0xc8d, 0xc8d), + (0xc91, 0xc91), + (0xca9, 0xca9), + (0xcb4, 0xcb4), + (0xcba, 0xcbb), + (0xcc5, 0xcc5), + (0xcc9, 0xcc9), + (0xcce, 0xcd4), + (0xcd7, 0xcdd), + (0xcdf, 0xcdf), + (0xce4, 0xce5), + (0xcf0, 0xcf0), + (0xcf3, 0xcff), + (0xd0d, 0xd0d), + (0xd11, 0xd11), + (0xd45, 0xd45), + (0xd49, 0xd49), + (0xd50, 0xd53), + (0xd64, 0xd65), + (0xd80, 0xd80), + (0xd84, 0xd84), + (0xd97, 0xd99), + (0xdb2, 0xdb2), + (0xdbc, 0xdbc), + (0xdbe, 0xdbf), + (0xdc7, 0xdc9), + (0xdcb, 0xdce), + (0xdd5, 0xdd5), + (0xdd7, 0xdd7), + (0xde0, 0xde5), + (0xdf0, 0xdf1), + (0xdf5, 0xe00), + (0xe3b, 0xe3e), + (0xe5c, 0xe80), + (0xe83, 0xe83), + (0xe85, 0xe85), + (0xe8b, 0xe8b), + (0xea4, 0xea4), + (0xea6, 0xea6), + (0xebe, 0xebf), + (0xec5, 0xec5), + (0xec7, 0xec7), + (0xece, 0xecf), + (0xeda, 0xedb), + (0xee0, 0xeff), + (0xf48, 0xf48), + (0xf6d, 0xf70), + (0xf98, 0xf98), + (0xfbd, 0xfbd), + (0xfcd, 0xfcd), + (0xfdb, 0xfff), + (0x10c6, 0x10c6), + (0x10c8, 0x10cc), + (0x10ce, 0x10cf), + (0x1249, 0x1249), + (0x124e, 0x124f), + (0x1257, 0x1257), + (0x1259, 0x1259), + (0x125e, 0x125f), + (0x1289, 0x1289), + (0x128e, 0x128f), + (0x12b1, 0x12b1), + (0x12b6, 0x12b7), + (0x12bf, 0x12bf), + (0x12c1, 0x12c1), + (0x12c6, 0x12c7), + (0x12d7, 0x12d7), + (0x1311, 0x1311), + (0x1316, 0x1317), + (0x135b, 0x135c), + (0x137d, 0x137f), + (0x139a, 0x139f), + (0x13f6, 0x13f7), + (0x13fe, 0x13ff), + (0x169d, 0x169f), + (0x16f9, 0x16ff), + (0x170d, 0x170d), + (0x1715, 0x171f), + (0x1737, 0x173f), + (0x1754, 0x175f), + (0x176d, 0x176d), + (0x1771, 0x1771), + (0x1774, 0x177f), + (0x17de, 0x17df), + (0x17ea, 0x17ef), + (0x17fa, 0x17ff), + (0x180f, 0x180f), + (0x181a, 0x181f), + (0x1879, 0x187f), + (0x18ab, 0x18af), + (0x18f6, 0x18ff), + (0x191f, 0x191f), + (0x192c, 0x192f), + (0x193c, 0x193f), + (0x1941, 0x1943), + (0x196e, 0x196f), + (0x1975, 0x197f), + (0x19ac, 0x19af), + (0x19ca, 0x19cf), + (0x19db, 0x19dd), + (0x1a1c, 0x1a1d), + (0x1a5f, 0x1a5f), + (0x1a7d, 0x1a7e), + (0x1a8a, 0x1a8f), + (0x1a9a, 0x1a9f), + (0x1aae, 0x1aaf), + (0x1ac1, 0x1aff), + (0x1b4c, 0x1b4d), + (0x1bf4, 0x1bfb), + (0x1c38, 0x1c3a), + (0x1c4a, 0x1c4c), + (0x1c89, 0x1c8f), + (0x1cbb, 0x1cbc), + (0x1cc8, 0x1ccf), + (0x1cfb, 0x1cff), + (0x1dfa, 0x1dfa), + (0x1f16, 0x1f17), + (0x1f1e, 0x1f1f), + (0x1f46, 0x1f47), + (0x1f4e, 0x1f4f), + (0x1f58, 0x1f58), + (0x1f5a, 0x1f5a), + (0x1f5c, 0x1f5c), + (0x1f5e, 0x1f5e), + (0x1f7e, 0x1f7f), + (0x1fb5, 0x1fb5), + (0x1fc5, 0x1fc5), + (0x1fd4, 0x1fd5), + (0x1fdc, 0x1fdc), + (0x1ff0, 0x1ff1), + (0x1ff5, 0x1ff5), + (0x1fff, 0x1fff), + (0x2065, 0x2065), + (0x2072, 0x2073), + (0x208f, 0x208f), + (0x209d, 0x209f), + (0x20c2, 0x20cf), + (0x20f1, 0x20ff), + (0x218c, 0x218f), + (0x242a, 0x243f), + (0x244b, 0x245f), + (0x2b74, 0x2b75), + (0x2c2f, 0x2c2f), + (0x2c5f, 0x2c5f), + (0x2cf4, 0x2cf8), + (0x2d26, 0x2d26), + (0x2d28, 0x2d2c), + (0x2d2e, 0x2d2f), + (0x2d68, 0x2d6e), + (0x2d71, 0x2d7e), + (0x2d97, 0x2d9f), + (0x2da7, 0x2da7), + (0x2daf, 0x2daf), + (0x2db7, 0x2db7), + (0x2dbf, 0x2dbf), + (0x2dc7, 0x2dc7), + (0x2dcf, 0x2dcf), + (0x2dd7, 0x2dd7), + (0x2ddf, 0x2ddf), + (0x2e5e, 0x2e7f), + (0x2e9a, 0x2e9a), + (0x2ef4, 0x2eff), + (0x2fd6, 0x2fef), + (0x3040, 0x3040), + (0x3097, 0x3098), + (0x3100, 0x3104), + (0x3130, 0x3130), + (0x318f, 0x318f), + (0x31e6, 0x31ee), + (0x321f, 0x321f), + (0x9ffd, 0x9fff), + (0xa48d, 0xa48f), + (0xa4c7, 0xa4cf), + (0xa62c, 0xa63f), + (0xa6f8, 0xa6ff), + (0xa7c0, 0xa7c1), + (0xa7cb, 0xa7f4), + (0xa82d, 0xa82f), + (0xa83a, 0xa83f), + (0xa878, 0xa87f), + (0xa8c6, 0xa8cd), + (0xa8da, 0xa8df), + (0xa954, 0xa95e), + (0xa97d, 0xa97f), + (0xa9ce, 0xa9ce), + (0xa9da, 0xa9dd), + (0xa9ff, 0xa9ff), + (0xaa37, 0xaa3f), + (0xaa4e, 0xaa4f), + (0xaa5a, 0xaa5b), + (0xaac3, 0xaada), + (0xaaf7, 0xab00), + (0xab07, 0xab08), + (0xab0f, 0xab10), + (0xab17, 0xab1f), + (0xab27, 0xab27), + (0xab2f, 0xab2f), + (0xab6c, 0xab6f), + (0xabee, 0xabef), + (0xabfa, 0xabff), + (0xd7a4, 0xd7af), + (0xd7c7, 0xd7ca), + (0xd7fc, 0xd7ff), + (0xfa6e, 0xfa6f), + (0xfada, 0xfaff), + (0xfb07, 0xfb12), + (0xfb18, 0xfb1c), + (0xfb37, 0xfb37), + (0xfb3d, 0xfb3d), + (0xfb3f, 0xfb3f), + (0xfb42, 0xfb42), + (0xfb45, 0xfb45), + (0xfdd0, 0xfdef), + (0xfe1a, 0xfe1f), + (0xfe53, 0xfe53), + (0xfe67, 0xfe67), + (0xfe6c, 0xfe6f), + (0xfe75, 0xfe75), + (0xfefd, 0xfefe), + (0xff00, 0xff00), + (0xffbf, 0xffc1), + (0xffc8, 0xffc9), + (0xffd0, 0xffd1), + (0xffd8, 0xffd9), + (0xffdd, 0xffdf), + (0xffe7, 0xffe7), + (0xffef, 0xfff8), + (0xfffe, 0xffff), + (0x1000c, 0x1000c), + (0x10027, 0x10027), + (0x1003b, 0x1003b), + (0x1003e, 0x1003e), + (0x1004e, 0x1004f), + (0x1005e, 0x1007f), + (0x100fb, 0x100ff), + (0x10103, 0x10106), + (0x10134, 0x10136), + (0x1018f, 0x1018f), + (0x1019d, 0x1019f), + (0x101a1, 0x101cf), + (0x101fe, 0x1027f), + (0x1029d, 0x1029f), + (0x102d1, 0x102df), + (0x102fc, 0x102ff), + (0x10324, 0x1032c), + (0x1034b, 0x1034f), + (0x1037b, 0x1037f), + (0x1039e, 0x1039e), + (0x103c4, 0x103c7), + (0x103d6, 0x1044f), + (0x1049e, 0x1049f), + (0x104aa, 0x104ff), + (0x10528, 0x1052f), + (0x10564, 0x1056e), + (0x10570, 0x105ff), + (0x10737, 0x1073f), + (0x10756, 0x1075f), + (0x10768, 0x107ff), + (0x10806, 0x10807), + (0x10809, 0x10809), + (0x10836, 0x10836), + (0x10839, 0x1083b), + (0x1083d, 0x1083e), + (0x10856, 0x10856), + (0x1089f, 0x108a6), + (0x108b0, 0x108df), + (0x108f3, 0x108f3), + (0x108f6, 0x108fa), + (0x1091c, 0x1091e), + (0x1093a, 0x1093e), + (0x10940, 0x1097f), + (0x109b8, 0x109bb), + (0x109d0, 0x109d1), + (0x10a04, 0x10a04), + (0x10a07, 0x10a0b), + (0x10a14, 0x10a14), + (0x10a18, 0x10a18), + (0x10a36, 0x10a37), + (0x10a3b, 0x10a3e), + (0x10a49, 0x10a4f), + (0x10a59, 0x10a5f), + (0x10aa0, 0x10abf), + (0x10ae7, 0x10aea), + (0x10af7, 0x10aff), + (0x10b36, 0x10b38), + (0x10b56, 0x10b57), + (0x10b73, 0x10b77), + (0x10b92, 0x10b98), + (0x10b9d, 0x10ba8), + (0x10bb0, 0x10bff), + (0x10c49, 0x10cf9), + (0x10d28, 0x10d2f), + (0x10d3a, 0x10d6d), + (0x10d6f, 0x10d8d), + (0x10d90, 0x10e5f), + (0x10e7f, 0x10e7f), + (0x10eaa, 0x10eaa), + (0x10eae, 0x10eaf), + (0x10eb2, 0x10ecf), + (0x10ed9, 0x10eff), + (0x10f28, 0x10f2f), + (0x10f5a, 0x10f85), + (0x10f8a, 0x10faf), + (0x10fcc, 0x10fdf), + (0x10ff7, 0x10fff), + (0x1104e, 0x11051), + (0x11070, 0x1107e), + (0x110c2, 0x110cc), + (0x110ce, 0x110cf), + (0x110e9, 0x110ef), + (0x110fa, 0x110ff), + (0x11135, 0x11135), + (0x11148, 0x1114f), + (0x11177, 0x1117f), + (0x111e0, 0x111e0), + (0x111f5, 0x111ff), + (0x11212, 0x11212), + (0x1123f, 0x1127f), + (0x11287, 0x11287), + (0x11289, 0x11289), + (0x1128e, 0x1128e), + (0x1129e, 0x1129e), + (0x112aa, 0x112af), + (0x112eb, 0x112ef), + (0x112fa, 0x112ff), + (0x11304, 0x11304), + (0x1130d, 0x1130e), + (0x11311, 0x11312), + (0x11329, 0x11329), + (0x11331, 0x11331), + (0x11334, 0x11334), + (0x1133a, 0x1133a), + (0x11345, 0x11346), + (0x11349, 0x1134a), + (0x1134e, 0x1134f), + (0x11351, 0x11356), + (0x11358, 0x1135c), + (0x11364, 0x11365), + (0x1136d, 0x1136f), + (0x11375, 0x113d3), + (0x113d6, 0x113d6), + (0x113d9, 0x113ff), + (0x1145c, 0x1145c), + (0x11462, 0x1147f), + (0x114c8, 0x114cf), + (0x114da, 0x1157f), + (0x115b6, 0x115b7), + (0x115de, 0x115ff), + (0x11645, 0x1164f), + (0x1165a, 0x1165f), + (0x1166d, 0x1167f), + (0x116ba, 0x116bf), + (0x116ca, 0x116ff), + (0x1171b, 0x1171c), + (0x1172c, 0x1172f), + (0x11740, 0x117ff), + (0x1183c, 0x118df), + (0x118f3, 0x118fe), + (0x11907, 0x11908), + (0x1190a, 0x1190b), + (0x11914, 0x11914), + (0x11917, 0x11917), + (0x11936, 0x11936), + (0x11939, 0x1193a), + (0x11947, 0x1194f), + (0x1195a, 0x1199f), + (0x119a8, 0x119a9), + (0x119d8, 0x119d9), + (0x119e5, 0x119ff), + (0x11a48, 0x11a4f), + (0x11aa3, 0x11abf), + (0x11af9, 0x11aff), + (0x11b0a, 0x11be0), + (0x11be2, 0x11bff), + (0x11c09, 0x11c09), + (0x11c37, 0x11c37), + (0x11c46, 0x11c4f), + (0x11c6d, 0x11c6f), + (0x11c90, 0x11c91), + (0x11ca8, 0x11ca8), + (0x11cb7, 0x11cff), + (0x11d07, 0x11d07), + (0x11d0a, 0x11d0a), + (0x11d37, 0x11d39), + (0x11d3b, 0x11d3b), + (0x11d3e, 0x11d3e), + (0x11d48, 0x11d4f), + (0x11d5a, 0x11d5f), + (0x11d66, 0x11d66), + (0x11d69, 0x11d69), + (0x11d8f, 0x11d8f), + (0x11d92, 0x11d92), + (0x11d99, 0x11d9f), + (0x11daa, 0x11edf), + (0x11ef9, 0x11f42), + (0x11f50, 0x11faf), + (0x11fb1, 0x11fbf), + (0x11ff2, 0x11ffe), + (0x1239a, 0x123ff), + (0x1246f, 0x1246f), + (0x12475, 0x1247f), + (0x12544, 0x12ff0), + (0x12ff3, 0x12fff), + (0x1342f, 0x1342f), + (0x13440, 0x143ff), + (0x14647, 0x167ff), + (0x16a39, 0x16a3f), + (0x16a5f, 0x16a5f), + (0x16a6a, 0x16a6d), + (0x16a70, 0x16acf), + (0x16aee, 0x16aef), + (0x16af6, 0x16aff), + (0x16b46, 0x16b4f), + (0x16b5a, 0x16b5a), + (0x16b62, 0x16b62), + (0x16b78, 0x16b7c), + (0x16b90, 0x16d6c), + (0x16d70, 0x16e7f), + (0x16e9b, 0x16eff), + (0x16f4b, 0x16f4e), + (0x16f88, 0x16f8e), + (0x16fa0, 0x16fdf), + (0x16fe5, 0x16fef), + (0x16ff2, 0x16fff), + (0x187f8, 0x187ff), + (0x18cd6, 0x18cff), + (0x18d09, 0x1afff), + (0x1b11f, 0x1b14f), + (0x1b153, 0x1b163), + (0x1b168, 0x1b16f), + (0x1b2fc, 0x1bbff), + (0x1bc6b, 0x1bc6f), + (0x1bc7d, 0x1bc7f), + (0x1bc89, 0x1bc8f), + (0x1bc9a, 0x1bc9b), + (0x1bca4, 0x1cbff), + (0x1ccf0, 0x1ccf9), + (0x1ccfd, 0x1ccff), + (0x1ceb4, 0x1ceb9), + (0x1ced1, 0x1cedf), + (0x1cef1, 0x1cf4f), + (0x1cfc4, 0x1cfff), + (0x1d0f6, 0x1d0ff), + (0x1d127, 0x1d128), + (0x1d1eb, 0x1d1ff), + (0x1d246, 0x1d2df), + (0x1d2f4, 0x1d2ff), + (0x1d357, 0x1d35f), + (0x1d379, 0x1d3ff), + (0x1d455, 0x1d455), + (0x1d49d, 0x1d49d), + (0x1d4a0, 0x1d4a1), + (0x1d4a3, 0x1d4a4), + (0x1d4a7, 0x1d4a8), + (0x1d4ad, 0x1d4ad), + (0x1d4ba, 0x1d4ba), + (0x1d4bc, 0x1d4bc), + (0x1d4c4, 0x1d4c4), + (0x1d506, 0x1d506), + (0x1d50b, 0x1d50c), + (0x1d515, 0x1d515), + (0x1d51d, 0x1d51d), + (0x1d53a, 0x1d53a), + (0x1d53f, 0x1d53f), + (0x1d545, 0x1d545), + (0x1d547, 0x1d549), + (0x1d551, 0x1d551), + (0x1d6a6, 0x1d6a7), + (0x1d7cc, 0x1d7cd), + (0x1da8c, 0x1da9a), + (0x1daa0, 0x1daa0), + (0x1dab0, 0x1dfff), + (0x1e007, 0x1e007), + (0x1e019, 0x1e01a), + (0x1e022, 0x1e022), + (0x1e025, 0x1e025), + (0x1e02b, 0x1e0ff), + (0x1e12d, 0x1e12f), + (0x1e13e, 0x1e13f), + (0x1e14a, 0x1e14d), + (0x1e150, 0x1e2bf), + (0x1e2fa, 0x1e2fe), + (0x1e300, 0x1e5fe), + (0x1e600, 0x1e7ff), + (0x1e8c5, 0x1e8c6), + (0x1e8d7, 0x1e943), + (0x1e94c, 0x1e94f), + (0x1e95a, 0x1e95d), + (0x1e960, 0x1ec70), + (0x1ecb5, 0x1ed00), + (0x1ed3e, 0x1edff), + (0x1ee04, 0x1ee04), + (0x1ee20, 0x1ee20), + (0x1ee23, 0x1ee23), + (0x1ee25, 0x1ee26), + (0x1ee28, 0x1ee28), + (0x1ee33, 0x1ee33), + (0x1ee38, 0x1ee38), + (0x1ee3a, 0x1ee3a), + (0x1ee3c, 0x1ee41), + (0x1ee43, 0x1ee46), + (0x1ee48, 0x1ee48), + (0x1ee4a, 0x1ee4a), + (0x1ee4c, 0x1ee4c), + (0x1ee50, 0x1ee50), + (0x1ee53, 0x1ee53), + (0x1ee55, 0x1ee56), + (0x1ee58, 0x1ee58), + (0x1ee5a, 0x1ee5a), + (0x1ee5c, 0x1ee5c), + (0x1ee5e, 0x1ee5e), + (0x1ee60, 0x1ee60), + (0x1ee63, 0x1ee63), + (0x1ee65, 0x1ee66), + (0x1ee6b, 0x1ee6b), + (0x1ee73, 0x1ee73), + (0x1ee78, 0x1ee78), + (0x1ee7d, 0x1ee7d), + (0x1ee7f, 0x1ee7f), + (0x1ee8a, 0x1ee8a), + (0x1ee9c, 0x1eea0), + (0x1eea4, 0x1eea4), + (0x1eeaa, 0x1eeaa), + (0x1eebc, 0x1eeef), + (0x1eef2, 0x1efff), + (0x1f02c, 0x1f02f), + (0x1f094, 0x1f09f), + (0x1f0af, 0x1f0b0), + (0x1f0c0, 0x1f0c0), + (0x1f0d0, 0x1f0d0), + (0x1f0f6, 0x1f0ff), + (0x1f1ae, 0x1f1e5), + (0x1f203, 0x1f20f), + (0x1f23c, 0x1f23f), + (0x1f249, 0x1f24f), + (0x1f252, 0x1f25f), + (0x1f266, 0x1f2ff), + (0x1f6d9, 0x1f6db), + (0x1f6ed, 0x1f6ef), + (0x1f6fd, 0x1f6ff), + (0x1f7da, 0x1f7df), + (0x1f7ec, 0x1f7ef), + (0x1f7f1, 0x1f7ff), + (0x1f80c, 0x1f80f), + (0x1f848, 0x1f84f), + (0x1f85a, 0x1f85f), + (0x1f888, 0x1f88f), + (0x1f8ae, 0x1f8af), + (0x1f8bc, 0x1f8bf), + (0x1f8c2, 0x1f8cf), + (0x1f8d9, 0x1f8ff), + (0x1fa58, 0x1fa5f), + (0x1fa6e, 0x1fa6f), + (0x1fa7d, 0x1fa7f), + (0x1fa8b, 0x1fa8d), + (0x1fac7, 0x1fac7), + (0x1fac9, 0x1facc), + (0x1fadd, 0x1fade), + (0x1faeb, 0x1faee), + (0x1faf9, 0x1faff), + (0x1fb93, 0x1fb93), + (0x1fbfb, 0x1ffff), + (0x2a6de, 0x2a6ff), + (0x2b735, 0x2b73f), + (0x2b81e, 0x2b81f), + (0x2cea2, 0x2ceaf), + (0x2ebe1, 0x2f7ff), + (0x2fa1e, 0x2ffff), + (0x3134b, 0xe0000), + (0xe0002, 0xe001f), + (0xe0080, 0xe00ff), + (0xe01f0, 0xeffff), + (0xffffe, 0xfffff), + (0x10fffe, 0x10ffff), +]; +// END GENERATED UNICODE GUARD + +// BEGIN GENERATED REGEX CASE RELATIONS +// Generated by round5_regex_case_table_matches_interpreters; decimal Unicode scalar pairs. +// Differing BMP re.IGNORECASE relations seed the pattern-only fold-class guard. +// Python 3.11.16 14.0.0 +// Python 3.12.14 15.0.0 +// Python 3.13.15 15.1.0 +// Python 3.14.7 16.0.0 +// Python 3.15.0rc2 17.0.0 +// Python 3.9.6 13.0.0 +const REGEX_CASE_RELATIONS: &[(u32, u32)] = &[ + (411, 42972), + (612, 42955), + (1042, 7296), + (1044, 7297), + (1054, 7298), + (1057, 7299), + (1058, 7300), + (1058, 7301), + (1066, 7302), + (1074, 7296), + (1076, 7297), + (1086, 7298), + (1089, 7299), + (1090, 7300), + (1090, 7301), + (1098, 7302), + (1122, 7303), + (1123, 7303), + (7300, 7301), + (7304, 42570), + (7304, 42571), + (7305, 7306), + (11311, 11359), + (42944, 42945), + (42956, 42957), + (42958, 42959), + (42960, 42961), + (42962, 42963), + (42964, 42965), + (42966, 42967), + (42968, 42969), + (42970, 42971), +]; +// END GENERATED REGEX CASE RELATIONS + +// Generated separately from real Python regex engines. String case mappings +// do not describe re.IGNORECASE, notably the Python 3.9 Cyrillic relations. +static REGEX_CASE_ENDPOINTS: LazyLock> = LazyLock::new(|| { + let mut points: Vec = REGEX_CASE_RELATIONS + .iter() + .flat_map(|&(a, b)| [a, b]) + .collect(); + // Close over the engine's fold classes too. Older Python-only relations + // already contribute both endpoints through the generated pair table. + let class = format!( + "[{}]", + points + .iter() + .map(|c| format!(r"\x{{{c:x}}}")) + .collect::() + ); + let hir = regex_syntax::ParserBuilder::new() + .case_insensitive(true) + .build() + .parse(&class) + .expect("fixed regex case table"); + if let regex_syntax::hir::HirKind::Class(regex_syntax::hir::Class::Unicode(set)) = hir.kind() { + for range in set.iter() { + points.extend(u32::from(range.start())..=u32::from(range.end())); + } + } + points.sort_unstable(); + points.dedup(); + points +}); + +pub(super) fn version_dependent(c: char) -> bool { + if c.is_ascii() { + return false; + } + let c = u32::from(c); + let at = VERSION_DEPENDENT.partition_point(|&(_, end)| end < c); + VERSION_DEPENDENT + .get(at) + .is_some_and(|&(start, _)| start <= c) +} + +/// Check the closed range without walking its code points. Endpoints alone +/// cannot validate a custom class: its interior can name a guarded scalar. +pub(super) fn range_version_dependent(start: char, end: char) -> bool { + let (start, end) = (u32::from(start), u32::from(end)); + let at = VERSION_DEPENDENT.partition_point(|&(_, last)| last < start); + VERSION_DEPENDENT + .get(at) + .is_some_and(|&(first, _)| first <= end) +} + +/// Only named literals and class ranges under active IGNORECASE use this +/// relation guard. Categories and dot do not name a member of a fold class. +pub(super) fn range_regex_case_dependent(start: char, end: char) -> bool { + let (start, end) = (u32::from(start), u32::from(end)); + REGEX_CASE_ENDPOINTS + .get(REGEX_CASE_ENDPOINTS.partition_point(|&c| c < start)) + .is_some_and(|&c| c <= end) +} + +fn validate_unicode(text: &str, file: &str, mode: Lines) -> Result<(), String> { + for (line, text) in lines(text, mode).enumerate() { + if text.chars().any(version_dependent) { + return Err(format!( + "{}:{}: Unicode version-dependent text", + file.escape_debug(), + line + 1 + )); + } + } + Ok(()) +} + +const USAGE: &str = "usage: og dev scan personal-data --root DIR --allow TEXT --skip TEXT [--patterns FILE] [--cached] [--] [FILE...]\n"; + +/// This file's own path: it holds every allowed text, so all of them apply. +const SELF: &str = "scripts/ci/check-personal-data.sh"; + +/// How many leading bytes are searched for a NUL to call a file binary. +const BINARY_PROBE: usize = 8192; + +const STR_GENES: &str = "HTT|FMR1|C9orf72|C9ORF72|ATXN[0-9]+|DMPK|FXN|CNBP|RFC1|PABPN1|CACNA1A|TCF4|AR|PPP2R2B|TBP|JPH3|NOP56|ATN1|CSTB|AFF2|FMR2|NOTCH2NLC|DAB1|BEAN1|STARD7|YEATS2"; +const OCTET: &str = "(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"; + +/// Python has three distinct line operations here. Pattern files use text +/// mode universal newlines; exception tables use str.splitlines(); scanned +/// blobs are decoded from bytes and split on LF, preserving the last empty +/// line. Keep the distinction in this one helper, including CRLF as one row. +#[derive(Clone, Copy)] +enum Lines { + Blob, + TextFile, + ExceptionTable, +} + +fn lines(text: &str, mode: Lines) -> impl Iterator { + let (mut rest, mut finished) = (text, false); + std::iter::from_fn(move || { + if finished || (rest.is_empty() && !matches!(mode, Lines::Blob)) { + return None; + } + let at = rest.char_indices().find(|&(_, c)| match mode { + Lines::Blob => c == '\n', + Lines::TextFile => matches!(c, '\n' | '\r'), + Lines::ExceptionTable => matches!( + c, + '\n' | '\r' + | '\u{b}' + | '\u{c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ), + }); + match at { + Some((i, c)) => { + let line = &rest[..i]; + let mut end = i + c.len_utf8(); + if c == '\r' && rest[end..].starts_with('\n') { + end += 1; + } + rest = &rest[end..]; + Some(line) + } + None => { + finished = true; + Some(rest) + } + } + }) +} + +/// A rule: its kind, its pattern, and a pattern the line must also match. +#[derive(Clone)] +struct Check { + kind: &'static str, + rule: Rule, + cond: Option<&'static [Rule]>, +} + +impl Check { + /// Necessary literal bytes only. This can admit extra lines, but cannot + /// decide a match or discard a candidate span. Custom rules always run. + fn candidate(&self, line: &str) -> bool { + match self.kind { + "home" => { + line.contains("/home/") || line.contains("/mnt/use") || line.contains("Users") + } + "address" => line.contains('.'), + "diplotype" => line.contains('*') && line.contains('/'), + "rsid" => line.contains("rs"), + "repeat" => line.contains('/'), + _ => true, + } + } +} + +fn rule(pattern: &str) -> Rule { + Rule::compile(pattern, true).expect("a valid built-in pattern") +} + +static LABEL: LazyLock<[Rule; 1]> = LazyLock::new(|| { + [rule( + r"(?:(?i:(? = LazyLock::new(|| [rule(r"\bREPCN\b")]); + +/// Built-ins use the same compiler and iterator as custom patterns. +fn builtin() -> Vec { + static RULES: LazyLock> = LazyLock::new(|| { + let gt = r"\(?[ACGTDI][/|;]?[ACGTDI]\)?(?![A-Za-z0-9])"; + vec![ + Check { + kind: "home", + rule: rule( + r"(?, + skip: Vec, + patterns: Option, + cached: bool, + files: Vec, +} + +fn parse(args: &[String]) -> Result { + let (mut root, mut allow, mut skip, mut patterns, mut cached) = (None, None, None, None, false); + let mut files = Vec::new(); + let mut it = args.iter(); + while let Some(a) = it.next() { + let mut value = |name: &str| { + it.next() + .cloned() + .ok_or_else(|| format!("{name} needs a value")) + }; + match a.as_str() { + "--root" => root = Some(value("--root")?), + "--allow" => allow = Some(value("--allow")?), + "--skip" => skip = Some(value("--skip")?), + "--patterns" => patterns = Some(value("--patterns")?), + "--cached" => cached = true, + "--" => { + files.extend(it.by_ref().map(PathBuf::from)); + break; + } + s if s.starts_with('-') => return Err(format!("unknown option {s}")), + s => files.push(PathBuf::from(s)), + } + } + let root = root.ok_or("--root is required")?; + let allow_text = allow.ok_or("--allow is required")?; + validate_unicode(&allow_text, "ALLOW", Lines::ExceptionTable)?; + let allow = lines(&allow_text, Lines::ExceptionTable) + .filter(|l| !py_strip(l).is_empty()) + .map(|l| { + let l = py_strip(l); + let (path, text) = l.split_once(' ').unwrap_or((l, "")); + (path.to_owned(), py_strip(text).to_owned()) + }) + .collect(); + let skip_text = skip.ok_or("--skip is required")?; + validate_unicode(&skip_text, "SKIP", Lines::ExceptionTable)?; + let skip = lines(&skip_text, Lines::ExceptionTable) + .filter_map(|l| { + pycompat::text::split(l, None, None) + .first() + .map(|s| (*s).to_owned()) + }) + .collect(); + Ok(Args { + root: PathBuf::from(root), + allow, + skip, + patterns: patterns.filter(|p| !p.is_empty()), + cached, + files, + }) +} + +/// Said when a pattern is refused by the case-pairing guard. +const CASE_REFUSAL: &str = "unsupported custom pattern: it matches case-insensitively a letter whose upper/lower pairing differs between Unicode versions (some Cyrillic letters, for example). Turn the i flag off for that part and list the exact spellings instead, such as (?:Name|NAME|name)"; + +/// The user's own patterns, one per line; `#` lines and blank lines skipped. +fn own_rules(path: &str) -> Result, String> { + let text = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?; + validate_unicode(&text, path, Lines::TextFile)?; + let mut out = Vec::new(); + for (line, l) in lines(&text, Lines::TextFile).enumerate() { + let p = py_strip(l); + if p.is_empty() || l.starts_with('#') { + continue; + } + // Rule reports capture 1 for the built-in look-around patterns. + // A user's captures must not narrow the span checked against ALLOW. + // The message never quotes the pattern: it is private. + let rule = Rule::compile_explained(p, false).map_err(|why| { + format!( + "{}:{}: {}", + path.escape_debug(), + line + 1, + match why { + Refusal::CaseDependent => CASE_REFUSAL, + Refusal::Other => "invalid or unsupported custom pattern", + } + ) + })?; + out.push(Check { + kind: "own", + rule, + cond: None, + }); + } + Ok(out) +} + +/// `path` relative to `root` when it lies below it (both with symbolic +/// links resolved), else `path` as given. +fn relative(path: &Path, root: &Path) -> Result { + validate_filename(path)?; + let abs = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().map_or_else(|_| path.to_path_buf(), |d| d.join(path)) + }; + // The folder is resolved, not the file: a tracked symbolic link keeps + // its own name. + let placed = match (abs.parent(), abs.file_name()) { + (Some(dir), Some(name)) => dir + .canonicalize() + .map_or_else(|_| abs.clone(), |d| d.join(name)), + _ => abs.clone(), + }; + validate_filename(&placed)?; + match placed.strip_prefix(root) { + Ok(rel) if !rel.as_os_str().is_empty() => utf8_path(rel).map(str::to_owned), + _ => utf8_path(path).map(str::to_owned), + } +} + +fn validate_filename(path: &Path) -> Result<(), String> { + if utf8_path(path)?.chars().any(version_dependent) { + return Err("selected filename has Unicode version-dependent text".into()); + } + Ok(()) +} + +fn utf8_path(path: &Path) -> Result<&str, String> { + path.to_str() + .ok_or_else(|| "path is not valid UTF-8".into()) +} + +fn git_files(root: &Path) -> Result, String> { + let out = Command::new("git") + .arg("-C") + .arg(root) + .args(["ls-files", "-z"]) + .stderr(Stdio::piped()) + .output() + .map_err(|e| format!("cannot run git: {e}"))?; + if !out.status.success() { + return Err(format!("git ls-files failed in {}", root.display())); + } + out.stdout + .split(|&b| b == 0) + .filter(|f| !f.is_empty()) + .map(|f| { + std::str::from_utf8(f) + .map(|s| root.join(s)) + .map_err(|_| "path is not valid UTF-8".into()) + }) + .collect::, String>>() +} + +/// The index owns symlink identity in cached mode, even if the working copy +/// is absent, replaced by a regular file, or has a different target. +fn index_entries(root: &Path) -> Result, String> { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(["ls-files", "--stage", "-z"]) + .output() + .map_err(|_| "cannot read Git index")?; + if !output.status.success() { + return Err("cannot read Git index".into()); + } + let mut entries = HashMap::new(); + for entry in output.stdout.split(|&b| b == 0).filter(|e| !e.is_empty()) { + let tab = entry + .iter() + .position(|&b| b == b'\t') + .ok_or("invalid Git index entry")?; + let name = std::str::from_utf8(&entry[tab + 1..]).map_err(|_| "path is not valid UTF-8")?; + let mode = std::str::from_utf8(&entry[..tab]) + .map_err(|_| "invalid Git index entry")? + .split(' ') + .next() + .ok_or("invalid Git index entry")?; + entries.insert(name.to_owned(), mode.to_owned()); + } + Ok(entries) +} + +/// The staged blob of each path (relative to `root`), from one +/// `git cat-file --batch`. A path with no blob in the index is absent. +fn staged(root: &Path, rels: &[String]) -> Result>, String> { + let entries = index_entries(root)?; + let unsupported = |rel: &str| { + format!( + "{}: indexed entry is not a readable blob", + rel.escape_debug() + ) + }; + for rel in rels { + if entries + .get(rel) + .is_some_and(|mode| !matches!(mode.as_str(), "100644" | "100755" | "120000")) + { + return Err(unsupported(rel)); + } + } + let wanted: Vec<&String> = rels.iter().collect(); + let mut blobs = HashMap::new(); + if wanted.is_empty() { + return Ok(blobs); + } + let mut child = Command::new("git") + .arg("-C") + .arg(root) + .args(["cat-file", "--batch", "-Z"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| format!("cannot run git: {e}"))?; + let mut stdin = child.stdin.take().ok_or("no stdin for git")?; + let input: Vec = wanted + .iter() + .flat_map(|r| format!(":{r}\0").into_bytes()) + .collect(); + let writer = std::thread::spawn(move || stdin.write_all(&input)); + let mut out = BufReader::new(child.stdout.take().ok_or("no stdout for git")?); + let mut unreadable = None; + for rel in &wanted { + let mut header = Vec::new(); + if out.read_until(0, &mut header).map_err(|e| e.to_string())? == 0 { + return Err("git cat-file stopped early".into()); + } + if header.pop() != Some(0) { + return Err("git cat-file: incomplete header".into()); + } + let head = + std::str::from_utf8(&header).map_err(|_| "git cat-file: invalid header encoding")?; + let fields: Vec<&str> = head.split(' ').collect(); + let [_, kind, size] = fields[..] else { + if entries.contains_key(*rel) { + unreadable.get_or_insert_with(|| unsupported(rel)); + } + continue; // An explicit untracked path has no index object. + }; + let size: usize = size + .parse() + .map_err(|_| "git cat-file: invalid object size".to_owned())?; + let mut data = vec![0; size + 1]; + out.read_exact(&mut data).map_err(|e| e.to_string())?; + data.pop(); + if kind == "blob" { + blobs.insert((*rel).clone(), data); + } else { + unreadable.get_or_insert_with(|| unsupported(rel)); + } + } + writer + .join() + .map_err(|_| "git input writer stopped")? + .map_err(|e| e.to_string())?; + if !child.wait().map_err(|e| e.to_string())?.success() { + return Err("git cat-file failed".into()); + } + if let Some(error) = unreadable { + return Err(error); + } + Ok(blobs) +} + +/// Validate components as well as terminal links: the OS can follow a link +/// in a directory component of any target. A bounded queue avoids recursion +/// and catches cycles before the subsequent read follows them. +fn validate_symlink_chain(path: &Path) -> Result<(), String> { + let mut pending: std::collections::VecDeque<_> = path + .components() + .map(|c| c.as_os_str().to_os_string()) + .collect(); + let mut prefix = PathBuf::new(); + let mut hops = 0; + while let Some(component) = pending.pop_front() { + prefix.push(component); + match std::fs::read_link(&prefix) { + Ok(target) => { + validate_filename(&target)?; + hops += 1; + if hops > 40 { + return Err("cannot validate symlink target".into()); + } + prefix.pop(); + if target.is_absolute() { + prefix.clear(); + } + for part in target.components().rev() { + pending.push_front(part.as_os_str().to_os_string()); + } + } + Err(e) if e.kind() == std::io::ErrorKind::InvalidInput => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(_) => return Err("cannot validate symlink target".into()), + } + } + Ok(()) +} + +/// Reports metadata only. Escape control characters in paths so each +/// finding occupies one line, including filenames containing newlines. +fn report(out: &mut dyn Write, kind: &str, rel: &str, n: usize) { + let _ = writeln!(out, "FAIL {kind:<9} {}:{n}", rel.escape_debug()); +} + +/// Scans one file's text; true when something was reported. +fn scan_text( + out: &mut dyn Write, + rules: &[Check], + texts: &[&str], + rel: &str, + data: &[u8], +) -> Result { + let mut bad = false; + // Intentional content decoding, matching Python errors="replace". Paths + // and Git protocol headers are validated strictly before any conversion. + let text = String::from_utf8_lossy(data); + for (i, line) in lines(&text, Lines::Blob).enumerate() { + let spans: Vec<(usize, usize)> = texts + .iter() + .flat_map(|t| line.match_indices(*t).map(|(a, s)| (a, a + s.len()))) + .collect(); + for c in rules { + if !c.candidate(line) { + continue; + } + if let Some(conditions) = c.cond { + let mut matched = false; + for condition in conditions { + matched |= condition.is_match(line)?; + } + if !matched { + continue; + } + } + for found in c.rule.find_iter(line) { + let (s, e) = found?; + if !spans.iter().any(|&(a, b)| a <= s && e <= b) { + report(out, c.kind, rel, i + 1); + bad = true; + break; + } + } + } + } + Ok(bad) +} + +fn run(a: &Args, out: &mut dyn Write) -> Result { + utf8_path(&a.root)?; + let root = a + .root + .canonicalize() + .map_err(|e| format!("--root {}: {e}", a.root.display()))?; + utf8_path(&root)?; + let mut rules = builtin(); + if let Some(p) = &a.patterns { + rules.extend(own_rules(p)?); + } + let files = if a.files.is_empty() { + git_files(&root)? + } else { + a.files.clone() + }; + if files.is_empty() { + let _ = writeln!(out, "FAIL: no file to scan"); + return Ok(false); + } + let mut todo: Vec<(&PathBuf, String)> = files + .iter() + .map(|f| relative(f, &root).map(|rel| (f, rel))) + .collect::, _>>()?; + // Validate every selected target before SKIP, and before following any + // symlink for a read. Cached scans validate the target stored in Git. + if a.cached { + let entries = index_entries(&root)?; + let rels: Vec<_> = todo + .iter() + .filter(|(_, r)| entries.get(r).is_some_and(|mode| mode == "120000")) + .map(|(_, r)| r.clone()) + .collect(); + for target in staged(&root, &rels)?.values() { + let target = std::str::from_utf8(target).map_err(|_| "path is not valid UTF-8")?; + validate_filename(Path::new(target))?; + } + } else { + for (path, _) in &todo { + validate_symlink_chain(path)?; + } + } + todo.retain(|(_, rel)| !a.skip.iter().any(|g| fnmatch(rel, g))); + let blobs = if a.cached { + staged( + &root, + &todo.iter().map(|(_, r)| r.clone()).collect::>(), + )? + } else { + HashMap::new() + }; + let (mut bad, mut scanned) = (false, 0usize); + for (path, rel) in &todo { + let data = if a.cached { + match blobs.get(rel) { + Some(d) => std::borrow::Cow::Borrowed(d.as_slice()), + None => continue, + } + } else { + match std::fs::read(path) { + Ok(d) => std::borrow::Cow::Owned(d), + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::IsADirectory + ) => + { + continue; + } + Err(e) => return Err(format!("cannot read {}: {e}", path.display())), + } + }; + if data[..data.len().min(BINARY_PROBE)].contains(&0) { + continue; + } + validate_unicode(&String::from_utf8_lossy(&data), rel, Lines::Blob)?; + scanned += 1; + let texts: Vec<&str> = a + .allow + .iter() + .filter(|(p, _)| rel == SELF || fnmatch(rel, p)) + .map(|(_, t)| t.as_str()) + .collect(); + bad |= scan_text(out, &rules, &texts, rel, &data).map_err(|()| { + format!( + "{}: pattern matching could not complete", + rel.escape_debug() + ) + })?; + } + let _ = writeln!(out, "scanned {scanned} text files"); + if scanned == 0 { + let _ = writeln!(out, "FAIL: no text file was scanned"); + return Ok(false); + } + Ok(!bad) +} + +/// `og dev scan personal-data ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let a = match parse(args) { + Ok(a) => a, + Err(e) => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!(err, "og dev scan personal-data: error: {e}"); + return 2; + } + }; + match run(&a, out) { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + fn kinds(line: &str) -> Vec<&'static str> { + let rules = builtin(); + let mut out = Vec::new(); + for c in &rules { + if c.cond + .is_some_and(|cs| !cs.iter().any(|r| r.is_match(line).unwrap())) + { + continue; + } + if c.rule.is_match(line).unwrap() { + out.push(c.kind); + } + } + out + } + + // Every planted value is split by "~", which is dropped, so this file + // holds none of them whole. + fn p(s: &str) -> String { + s.replace('~', "") + } + + #[test] + fn each_kind_is_caught() { + for (line, kind) in [ + ("cp results.html /home/~alice/genome/", "home"), + ("C:\\~Users\\bob", "home"), + ("scp out.vcf.gz root@192.~168.7.20:/data/", "address"), + ("ssh nas.tail1234.ts.~net", "address"), + ("| sample7 | CYP2D6 | *1/~*4 |", "diplotype"), + ("sample_01 CYP2D6 *1/~*4", "diplotype"), + ("HG0~02 *1/~*2", "diplotype"), + ("rs4244285~ AG", "rsid"), + ("rs123: (C;~T)", "rsid"), + ("rs4477212\t1\t82154\t~AA", "rsid"), + ("HTT CAG repeats: 17/~19", "repeat"), + ("GT:REPCN 0~/1:17/~19", "repeat"), + ] { + let line = p(line); + assert!(kinds(&line).contains(&kind), "{line}: {:?}", kinds(&line)); + } + } + + #[test] + fn edges_as_python_reads_them() { + for line in [ + "- `*1/*4` -- Intermediate metabolizer", + "samtools 1.21, GATK 4.6.2.0", + "https://example.org/home/page", + "CYP2C19*2 is rs4244285 (G>A)", + // An address with a fifth part, or a version that looks like one. + "v10.1.2.3.4", + "1.10.1.2.3", + "10.1.2.256", + // The word must stand alone: "samples" is a label, "sampler" is not. + "sampler *1/*4", + // A repeat count with another digit or slash on either side. + "REPCN 1.17/19", + "REPCN 17/19/3", + "/home/1x", + "my/home/user", + ] { + assert!(kinds(line).is_empty(), "{line}: {:?}", kinds(line)); + } + // Python takes the label "sample" out of "sample7a" by backing off + // the number: the line still counts as labelled. + assert_eq!(kinds(&p("sample7a *1/~*4")), ["diplotype"]); + assert_eq!(kinds(&p("samples: *1/~*4")), ["diplotype"]); + assert_eq!(kinds(&p("10.1.2.~3.")), ["address"]); + assert_eq!(kinds(&p("FMR1 has 29/~31")), ["repeat"]); + } + + #[test] + fn an_allowed_example_covers_only_its_own_text() { + let rules = builtin(); + let ex = p("(`/mnt/u~ser/`, `/ho~me/username/`, etc.)"); + let mut out = Vec::new(); + assert!(!scan_text(&mut out, &rules, &[ex.as_str()], "x.md", ex.as_bytes()).unwrap()); + let line = p(&format!("{ex} see /home/~bob/x")); + assert!(scan_text(&mut out, &rules, &[ex.as_str()], "x.md", line.as_bytes()).unwrap()); + let got = String::from_utf8(out).unwrap(); + assert!(got.starts_with("FAIL home x.md:1"), "{got}"); + } + + #[test] + fn own_patterns_use_the_complete_match() { + let path = + std::env::temp_dir().join(format!("og-personal-patterns-{}", std::process::id())); + std::fs::write( + &path, + "PREFIX (PUBLIC) SUFFIX\nPREFIX (?P