From b2eb6c680022136985482b05aadb3d4072579a9e Mon Sep 17 00:00:00 2001 From: Loki Date: Wed, 4 Nov 2026 14:30:45 +0800 Subject: [PATCH] fix(egress): retry a transcribe submit that failed instead of dropping it A failed submit deleted the local MP4, so the recording never got a transcript and winlab-helper was never told (the 11-02 wnc-ahts recording). Co-Authored-By: Claude Opus 5.5 --- deploy/egress/README.md | 3 ++- deploy/egress/bin/egress-transcribe | 38 +++++++++++++++++++++++++++-- deploy/egress/bin/egress-upload | 38 ++++++++++++++++++++--------- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/deploy/egress/README.md b/deploy/egress/README.md index c3f90d9..85a0a9e 100644 --- a/deploy/egress/README.md +++ b/deploy/egress/README.md @@ -20,7 +20,8 @@ Files for the 2 LiveKit Egress VMs (`egress01`, `egress02`). Both run the same c 2. The timer runs the service every minute. 3. `egress-upload` waits for the `EG_*.json` manifest, sends the MP4 to transcribe, and uploads it to Nextcloud under `meet///`. 4. Parts of one booking wait in `/var/lib/egress-hold//` until 1 hour after the booking ends, then ffmpeg joins them. -5. `egress-transcribe` puts the transcript, subtitles and notes next to the MP4 (`.txt`, `.srt`, `.notes.md`, `.notes.json`), then sends the hand-off to winlab-helper's trigger. +5. When transcribe does not take the MP4, it stays in `/var/lib/egress-transcribe/retry-.mp4` with `need_submit=1`. `egress-transcribe` sends it again every 10 minutes. After 6 hours it hands the recording off with `TRANSCRIPT_STATUS=failed` and an empty `TX_JOB_ID`, so the helper runs its own ASR. +6. `egress-transcribe` puts the transcript, subtitles and notes next to the MP4 (`.txt`, `.srt`, `.notes.md`, `.notes.json`), then sends the hand-off to winlab-helper's trigger. ## Empty stretches cut diff --git a/deploy/egress/bin/egress-transcribe b/deploy/egress/bin/egress-transcribe index eccb6c8..854e7b1 100755 --- a/deploy/egress/bin/egress-transcribe +++ b/deploy/egress/bin/egress-transcribe @@ -55,6 +55,39 @@ sha_of() { [ -s "$1" ] && sha256sum "$1" | cut -c1-64; } # pending file with dir=1, the month folder for one written before that. remote_dir() { if [ -n "${dir:-}" ]; then echo "${folder%%/*}/$(urlencode "$stem")"; else echo "$folder"; fi; } tx() { curl -g -sS -H "Authorization: Bearer $TX_TOKEN" "$@"; } +# Sends a file to transcribe; prints the job id, or logs why it failed and +# returns 1. Same function in egress-upload. +tx_submit() { # + local out code + out=$(mktemp) + code=$(curl -g -sS -o "$out" -w "%{http_code}" --max-time 1800 -H "Authorization: Bearer $TX_TOKEN" \ + -F "file=@$1" -F language=zh -F diarize=true "$TX/api/jobs") + if python3 -c 'import json,sys; j=json.load(open(sys.argv[1])).get("job_id",""); print(j); sys.exit(0 if j else 1)' "$out" 2>/dev/null; then + rm -f "$out"; return 0 + fi + echo "transcribe submit -> $code: $(head -c 200 "$out" | tr '\n' ' ')" >&2 + rm -f "$out"; return 1 +} +# A recording transcribe did not take: try again every 10 minutes; after 6 +# hours hand it to the helper as transcript failed, so it runs its own ASR. +resubmit() { # ; needs the pending file's fields in scope + local pending=$1 new tmp now + now=$(date +%s) + [ $((now - $(stat -c %Y "$pending"))) -ge 600 ] || return 0 + if new=$(tx_submit "$STATE/$job.mp4"); then + grep -vE '^(job|need_submit|submit_since)=' "$pending" > "$STATE/$new.pending.tmp" + printf 'job=%q\n' "$new" >> "$STATE/$new.pending.tmp" + mv "$STATE/$job.mp4" "$STATE/$new.mp4" + mv "$STATE/$new.pending.tmp" "$STATE/$new.pending" && rm -f "$pending" + echo "transcribe job $new (resubmitted): $stem" + elif [ $((now - ${submit_since:-0})) -ge 21600 ]; then + echo "transcribe never took it, handing off without a transcript: $stem" >&2 + printf 'need_submit=\ntx_job=\n' >> "$pending" + tmp=$(mktemp -d); finish "$pending" "$job" "$tmp/t.srt" failed none "$tmp"; rm -rf "${tmp:?}" + else + touch "$pending" + fi +} put() { # put local code code=$(curl -g -sS -o /dev/null -w "%{http_code}" -u "$AUTH" -T "$1" "$DAV/$NC_DIR/$2") @@ -128,7 +161,7 @@ hand_off() { # ; needs the pending file's fields in scope [START_TIME]=${start:-} [END_TIME]=${end:-} [BOOKING_REQUEST_ID]=${booking:-}) if [ -n "${dir:-}" ]; then v[DL_URL]="$root$(urlencode "$stem").mp4" - v[RELAY_VERSION]=2 v[TX_JOB_ID]=$job + v[RELAY_VERSION]=2 v[TX_JOB_ID]=${tx_job-$job} v[TRANSCRIPT_STATUS]=${tx_status:-failed} v[TRANSCRIPT_SHA256]=${sha_txt:-} v[SUBTITLE_SHA256]=${sha_srt:-} v[NOTES_STATUS]=${notes_status:-none} v[NOTES_SHA256]=${sha_notes_md:-} v[NOTES_JSON_SHA256]=${sha_notes_json:-} fields="$fields RELAY_VERSION TX_JOB_ID TRANSCRIPT_STATUS TRANSCRIPT_SHA256 SUBTITLE_SHA256 NOTES_STATUS NOTES_SHA256 NOTES_JSON_SHA256" @@ -153,10 +186,11 @@ print(hmac.new(os.environ["HELPER_RELAY_SECRET"].encode(), payload.encode(), has for pending in "$STATE"/*.pending; do unset job folder dir stem room egress start end group person booking subject sha notes_asked finished share secret \ - tx_status notes_status sha_txt sha_srt sha_notes_md sha_notes_json + tx_status notes_status sha_txt sha_srt sha_notes_md sha_notes_json need_submit submit_since tx_job . "$pending" # job, folder, stem, the recording's fields, maybe notes_asked / finished / share / statuses [ -n "${job:-}" ] && [ -n "${stem:-}" ] || { echo "bad state file: $pending" >&2; continue; } if [ -n "${finished:-}" ]; then hand_off "$pending" && rm -f "$STATE/$job.mp4" "$pending"; continue; fi + if [ -n "${need_submit:-}" ]; then resubmit "$pending"; continue; fi stage=$(tx "$TX/api/jobs/$job" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("stage",""))' 2>/dev/null) case "$stage" in done) ;; diff --git a/deploy/egress/bin/egress-upload b/deploy/egress/bin/egress-upload index ca6b21c..d522d92 100755 --- a/deploy/egress/bin/egress-upload +++ b/deploy/egress/bin/egress-upload @@ -58,6 +58,21 @@ PY urlencode() { python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"; } +STATE=/var/lib/egress-transcribe +# Sends a file to transcribe; prints the job id, or logs why it failed and +# returns 1. Same function in egress-transcribe. +tx_submit() { # + local out code + out=$(mktemp) + code=$(curl -g -sS -o "$out" -w "%{http_code}" --max-time 1800 -H "Authorization: Bearer $TX_TOKEN" \ + -F "file=@$1" -F language=zh -F diarize=true "${TX_URL:-https://transcribe.winlab.tw}/api/jobs") + if python3 -c 'import json,sys; j=json.load(open(sys.argv[1])).get("job_id",""); print(j); sys.exit(0 if j else 1)' "$out" 2>/dev/null; then + rm -f "$out"; return 0 + fi + echo "transcribe submit -> $code: $(head -c 200 "$out" | tr '\n' ' ')" >&2 + rm -f "$out"; return 1 +} + # Uploads one file to Nextcloud, hands it to transcribe and removes it. # Each recording gets its own folder, meet///, so its # transcript, subtitles and notes sit beside it and one folder share hands @@ -122,18 +137,19 @@ ship() { # file_with_meet "$month/$stem" "$stem" "$(stat -c %s "$local_path")" "$(date -u -r "$local_path" +%Y-%m-%dT%H:%M:%SZ)" # Hand the recording to transcribe before it goes; egress-transcribe # collects the transcript when the job is done. + # A failed submit keeps the MP4 under a placeholder id with need_submit=1; + # egress-transcribe sends it again until transcribe takes it. if [ -n "${TX_TOKEN:-}" ]; then - job=$(curl -g -sS -H "Authorization: Bearer $TX_TOKEN" -F "file=@$local_path" -F language=zh -F diarize=true "${TX_URL:-https://transcribe.winlab.tw}/api/jobs" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("job_id",""))' 2>/dev/null) - if [ -n "$job" ]; then - mkdir -p /var/lib/egress-transcribe - # dir=1: the recording has its own folder (hand-off v2). - printf 'job=%q\nfolder=%q\ndir=1\nstem=%q\nroom=%q\negress=%q\nstart=%q\nend=%q\ngroup=%q\nperson=%q\nbooking=%q\nsubject=%q\nsha=%q\n' \ - "$job" "$month/$stem" "$stem" "$room" "$egress" "$start" "$end" "$group" "$person" "$booking" "$subject" "$sha" > "/var/lib/egress-transcribe/$job.pending" - mv "$local_path" "/var/lib/egress-transcribe/$job.mp4" - echo "transcribe job $job: $name" - else - echo "transcribe submit failed: $name" >&2 - fi + local retry= + job=$(tx_submit "$local_path") || { job="retry-${sha:0:16}"; retry=1; } + mkdir -p "$STATE" + # dir=1: the recording has its own folder (hand-off v2). + printf 'job=%q\nfolder=%q\ndir=1\nstem=%q\nroom=%q\negress=%q\nstart=%q\nend=%q\ngroup=%q\nperson=%q\nbooking=%q\nsubject=%q\nsha=%q\n' \ + "$job" "$month/$stem" "$stem" "$room" "$egress" "$start" "$end" "$group" "$person" "$booking" "$subject" "$sha" > "$STATE/$job.pending" + [ -z "$retry" ] || printf 'need_submit=1\nsubmit_since=%s\n' "$(date +%s)" >> "$STATE/$job.pending" + mv "$local_path" "$STATE/$job.mp4" + if [ -n "$retry" ]; then echo "transcribe submit failed, will retry as $job: $name" >&2 + else echo "transcribe job $job: $name"; fi fi rm -f "$local_path" return 0