From b8f42b5585e48e44c27c0078966bd32787fc0ccd Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Sun, 13 Sep 2026 17:23:51 +0200 Subject: [PATCH] fix(storage): retry reader lease release on MySQL deadlock Releasing a Storage Reader Lease raced the cascade delete of its storage location, which locks the same rows in the opposite order. MySQL picked the release as the deadlock victim, and since it was fire-and-forget the error surfaced as an unhandled rejection that failed CI. Co-Authored-By: Claude Opus 5 (1M context) --- lib/storage-leases.ts | 6 +++++- lib/storage.ts | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/storage-leases.ts b/lib/storage-leases.ts index 8b38161..c06dc8e 100644 --- a/lib/storage-leases.ts +++ b/lib/storage-leases.ts @@ -1,6 +1,7 @@ import type { Kysely } from 'kysely' import type { Database, StorageReaderLeaseScope } from './db' import { randomUUID } from 'node:crypto' +import { retryOnLockConflict } from './db' import { env } from './env' export const LEASE_DURATION_MS = 2 * 60 * 1000 @@ -86,5 +87,8 @@ export async function renewReaderLease(db: Kysely, id: string) { } export async function releaseReaderLease(db: Kysely, id: string) { - await db.deleteFrom('storage_reader_leases').where('id', '=', id).execute() + // races cascade deletes of the storage location, which lock rows in the opposite order + await retryOnLockConflict(() => + db.deleteFrom('storage_reader_leases').where('id', '=', id).execute(), + ) } diff --git a/lib/storage.ts b/lib/storage.ts index b33bd5d..e56d0f3 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -116,7 +116,10 @@ export class Storage { if (released) return released = true clearInterval(renewalTimer) - void releaseReaderLease(this.db, readerLeaseId) + // an unreleased lease only delays cleanup until it expires + releaseReaderLease(this.db, readerLeaseId).catch((err) => + logger.warn(`Failed to release Storage Reader Lease ${readerLeaseId}`, { error: err }), + ) } stream.once('end', release) stream.once('close', release)