From a80dc3ebb559e637ff4ceb1eb905a3c8efb799b2 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:23:06 +0900 Subject: [PATCH 1/2] fix(server): retain admission until cancelled turns finalize Split cancellation ownership from PR #719 and preserve abort delivery when a listener finalizes the lease. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/server/lifecycle.ts | 18 ++- structure/runtime.md | 2 +- structure/transports/responses.md | 6 +- .../active-turn-disconnect.test.ts | 119 ++++++++++++++---- 4 files changed, 114 insertions(+), 31 deletions(-) diff --git a/src/server/lifecycle.ts b/src/server/lifecycle.ts index addb0c0fa23..395131e2a1b 100644 --- a/src/server/lifecycle.ts +++ b/src/server/lifecycle.ts @@ -172,6 +172,7 @@ export function resetLifecycleDrainStateForTests(): void { } export function tryAdmitTurn(sessionLaneId?: string, clientSignal?: AbortSignal): ActiveTurnLease | null { if (isDraining()) return null; + if (clientSignal?.aborted) return null; const opaqueSessionLaneId = sessionLaneId ? createHash("sha256").update(sessionLaneId).digest("hex").slice(0, SESSION_LANE_ID_BYTES) : undefined; @@ -195,6 +196,7 @@ export function tryAdmitTurn(sessionLaneId?: string, clientSignal?: AbortSignal) const attachedLeases = new Set(); const abortListeners = new Map void>(); let cancellationReason: unknown; + let cancelled = false; let active = true; let transferred = false; let nativeMainClaimed = false; @@ -206,10 +208,12 @@ export function tryAdmitTurn(sessionLaneId?: string, clientSignal?: AbortSignal) bindAbortSignal(signal) { if (!active || abortListeners.has(signal)) return () => {}; const onAbort = () => { + if (cancelled) return; + cancelled = true; cancellationReason = signal.reason; - const boundControllers = [...controllers]; - lease.release(); - for (const controller of boundControllers) controller.abort(cancellationReason); + for (const controller of [...controllers]) { + if (!controller.signal.aborted) controller.abort(cancellationReason); + } }; abortListeners.set(signal, onAbort); signal.addEventListener("abort", onAbort, { once: true }); @@ -229,17 +233,21 @@ export function tryAdmitTurn(sessionLaneId?: string, clientSignal?: AbortSignal) transferred = true; controllers.add(ac); activeTurns.set(ac, lease); + if (cancelled) { + ac.abort(cancellationReason ?? new Error("turn cancelled")); + return; + } lease.bindAbortSignal(ac.signal); }, beginCodexAccountSelection() { const mainProfileDraining = nativeMainDrainOwners.size > 0; - let selectionActive = active && !mainProfileDraining; + let selectionActive = active && !cancelled && !mainProfileDraining; let released = false; if (selectionActive) nativeMainSelections += 1; const selection: CodexAccountSelectionAdmission = { mainProfileDraining, claimMainProfile() { - if (released || mainProfileDraining || !active) return false; + if (released || mainProfileDraining || !active || cancelled) return false; if (!nativeMainClaimed) { nativeMainClaimed = true; nativeMainTurns.add(lease); diff --git a/structure/runtime.md b/structure/runtime.md index 5a33ffbc381..9023b5693fb 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -26,7 +26,7 @@ Native result continuations and function-result injection follow [the mode-speci Native steering follows [the shared WebSocket contract](transports/streaming-health.md#experimental-native-mid-turn-steering); this surface's defaults remain unchanged. -Responses admission and finalization are composed through the [core module ownership](transports/responses.md#core-module-ownership). Kiro's optional account-load admission is process-local and request-owned; its slot ends with the response body or cancellation. Other providers retain their admission path. Cancellation after a slot grant and before the request resumes releases the slot and returns cancellation without dispatch. +Responses admission and finalization are composed through the [core module ownership](transports/responses.md#core-module-ownership). Kiro's optional account-load admission is process-local and request-owned; its slot ends with the response body or cancellation. Other providers retain their admission path. Cancellation after a slot grant and before the request resumes releases the slot and returns cancellation without dispatch. In `src/server/lifecycle.ts`, cancellation promptly aborts bound controllers but retains active-turn, session-lane and workflow/native ownership until the owner finalizes; late controllers abort immediately, cancelled turns cannot select native accounts, and shutdown may release turns while admission is fenced. Catalog HTTP acquisition follows the [proxy-routing contract](catalog.md#remote-catalog-http-proxy-routing). CLI model lists expose shared catalog estimates as `price`; `models price` retains the saved override in `cost` and adds `effectiveCost`. Explicit zero overrides win; unknown prices remain null, and automatic defaults are never persisted to `modelCosts`. Covered by `tests/cli/cli-models.test.ts` and `tests/cli/cli-models-price.test.ts`. Ordinary model discovery, startup sync, and explicit sync apply the [new-arrival catalog contract](catalog.md#shared-catalog) before publishing rows. The scoped persisted decision also updates the live config merge baseline, so subsequent visibility writes preserve manual re-enablement. A file-loaded inventory that already differs from disk can publish HTTP rows through a detached policy projection without adding unsaved live disables; retained catalog writes still refuse stale inventory, and its load provenance follows the [config contract](config.md). diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 1608a832257..a55b27089f8 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -71,7 +71,7 @@ silence. It cannot observe the client's workspace or prove whether a successful repository state. Consequently, the active-turn and session-lane gates are concurrency admission limits, the translator budget follows [live retained-byte ownership](byte-accounting.md#stream-buffer-accounting), the response-state caps are cache retention limits, and the stall watchdog is a silence limit. None is a cumulative continuation or -semantic no-progress budget. The turn lease in `src/server/lifecycle.ts` subscribes to HTTP ingress cancellation, internal decision cancellation and every registered controller. Cancellation releases turn admission, session lanes, attached workflow leases and pending native-main selection ownership without waiting for upstream work. Live/Realtime acquisition binds ingress cancellation until WebSocket upgrade transfers ownership. Normal completion detaches the listeners; late finalizers remain idempotent. +semantic no-progress budget. The turn lease in `src/server/lifecycle.ts` subscribes to HTTP ingress cancellation, internal decision cancellation and every registered controller. Cancellation aborts every bound controller but retains turn admission, session lanes and attached ownership until the corresponding owner finalizes; pending native-main selection cannot claim an account after cancellation. Live/Realtime acquisition binds ingress cancellation until WebSocket upgrade transfers ownership. Normal completion detaches the listeners; late finalizers remain idempotent. > Decision record: [ADR-0031](../decisions/ADR-0031-responses-http-sse.md) > Decision record: [ADR-0032](../decisions/ADR-0032-responses-http-sse.md) @@ -590,8 +590,8 @@ keeps only allowlisted transport verdict codes and the explicit local `spend_led deserialize them as required fields. -Active-turn admission owns workflow admission, so both remain held until a streaming body finishes or -is cancelled. +Active-turn admission owns workflow admission, so both remain held until their owner finalizes. +Cancellation signals upstream work immediately; signalling alone does not release admission. The shared endpoint path also preserves the cyber-policy stop when a malformed UTF-8 5xx body is replacement-decoded; other malformed-body usage, quota, reset evidence and classification retain the diff --git a/tests/codex-integration/active-turn-disconnect.test.ts b/tests/codex-integration/active-turn-disconnect.test.ts index 12eeff30283..199ef48d97e 100644 --- a/tests/codex-integration/active-turn-disconnect.test.ts +++ b/tests/codex-integration/active-turn-disconnect.test.ts @@ -114,17 +114,34 @@ async function disconnectedTurn(protocol: Protocol, phase: Phase) { await until(() => getActiveTurnCount() === before.turns + 1, "HTTP admitted"); client.destroy(); } - await until(() => getActiveTurnCount() === before.turns && sessionLaneMetrics().active === before.lanes, `${protocol}/${phase} released`); + if (phase === "selection") { + await until(() => capturedSignal?.aborted === true, `${protocol}/${phase} cancelled`); + expect(getActiveTurnCount()).toBe(before.turns + 1); + expect(sessionLaneMetrics().active).toBe(before.lanes + 1); + if (protocol === "responses-http") expect(workflowBudgetSnapshot(root)?.active ?? 0).toBeGreaterThan(0); + expect(upstreamRequests).toBe(0); + } else { + await until(() => getActiveTurnCount() === before.turns && sessionLaneMetrics().active === before.lanes, `${protocol}/${phase} released`); + } console.log(JSON.stringify({ protocol, phase, before, after: { turns: getActiveTurnCount(), lanes: sessionLaneMetrics().active }, upstreamRequests, clientSignalAborted: capturedSignal?.aborted })); - expect(getActiveTurnCount()).toBe(before.turns); - expect(sessionLaneMetrics().active).toBe(before.lanes); - if (protocol !== "responses-ws") expect(workflowBudgetSnapshot(root)?.active ?? 0).toBe(0); + if (phase === "selection") { + expect(getActiveTurnCount()).toBe(before.turns + 1); + expect(sessionLaneMetrics().active).toBe(before.lanes + 1); + if (protocol === "responses-http") expect(workflowBudgetSnapshot(root)?.active ?? 0).toBeGreaterThan(0); + } else { + expect(getActiveTurnCount()).toBe(before.turns); + expect(sessionLaneMetrics().active).toBe(before.lanes); + if (protocol !== "responses-ws") expect(workflowBudgetSnapshot(root)?.active ?? 0).toBe(0); + } } catch (error) { console.log(JSON.stringify({ protocol, phase, before, after: { turns: getActiveTurnCount(), lanes: sessionLaneMetrics().active }, upstreamRequests, clientSignalAborted: capturedSignal?.aborted })); throw error; } finally { closeClient?.(); unblock.resolve(); + if (phase === "selection") { + await until(() => getActiveTurnCount() === before.turns && sessionLaneMetrics().active === before.lanes, `${protocol}/${phase} finalized`); + } prepare.mockRestore(); abortAndReleaseAllTurns(new Error("fixture teardown")); await server.stop(true); @@ -140,11 +157,11 @@ for (const protocol of ["responses-http", "responses-ws", "messages"] as const) } } for (const protocol of ["responses-http", "responses-ws"] as const) { - test(`${protocol} releases while account selection stays pending`, () => disconnectedTurn(protocol, "selection"), 15_000); + test(`${protocol} retains admission while account selection stays pending`, () => disconnectedTurn(protocol, "selection"), 15_000); } test("compact releases its lane after disconnect before upstream headers", () => disconnectedTurn("compact", "silent"), 15_000); -test("an aborted registered controller releases once while its work stays pending", () => { +test("an aborted registered controller retains ownership while its work stays pending", () => { const before = activeRegistryMetrics().activeTurns; const laneBefore = sessionLaneMetrics().active; const lease = tryAdmitTurn("fixture-controller")!; @@ -154,23 +171,26 @@ test("an aborted registered controller releases once while its work stays pendin lease.bindAbortController(ac); try { ac.abort(new Error("fixture client closed")); - expect(getActiveTurnCount()).toBe(before.active); - expect(sessionLaneMetrics().active).toBe(laneBefore); - expect(released).toBe(1); + expect(getActiveTurnCount()).toBe(before.active + 1); + expect(sessionLaneMetrics().active).toBe(laneBefore + 1); + expect(released).toBe(0); unregisterTurn(ac); lease.release(); + expect(getActiveTurnCount()).toBe(before.active); + expect(sessionLaneMetrics().active).toBe(laneBefore); expect(released).toBe(1); expect(activeRegistryMetrics().activeTurns.releaseMisses).toBe(before.releaseMisses); } finally { lease.release(); } }); -test("ingress cancellation releases pending selection and all bound controllers", () => { +test("ingress cancellation retains admission until pending selection unwinds", () => { const before = getActiveTurnCount(); const nativeBefore = getNativeMainProfileRequestCount(); const client = new AbortController(); const lease = tryAdmitTurn("fixture-selection", client.signal)!; const selection = lease.beginCodexAccountSelection(); + const duringSelection = getNativeMainProfileRequestCount(); const first = new AbortController(); const second = new AbortController(); lease.bindAbortController(first); @@ -178,8 +198,8 @@ test("ingress cancellation releases pending selection and all bound controllers" lease.bindAbortController(first); const reason = new Error("fixture ingress closed"); client.abort(reason); - expect(getActiveTurnCount()).toBe(before); - expect(getNativeMainProfileRequestCount()).toBe(nativeBefore); + expect(getActiveTurnCount()).toBe(before + 1); + expect(getNativeMainProfileRequestCount()).toBe(duringSelection); expect(first.signal.reason).toBe(reason); expect(second.signal.reason).toBe(reason); expect(selection.claimMainProfile()).toBe(false); @@ -194,18 +214,15 @@ test("ingress cancellation releases pending selection and all bound controllers" expect(released).toBe(1); }); -test("already aborted ingress cannot retain admission or selection", () => { +test("already aborted ingress cannot acquire admission or selection", () => { const before = getActiveTurnCount(); const nativeBefore = getNativeMainProfileRequestCount(); const client = new AbortController(); client.abort(new Error("fixture already closed")); - const lease = tryAdmitTurn("fixture-pre-aborted", client.signal)!; + const lease = tryAdmitTurn("fixture-pre-aborted", client.signal); expect(getActiveTurnCount()).toBe(before); - const selection = lease.beginCodexAccountSelection(); - expect(selection.claimMainProfile()).toBe(false); + expect(lease).toBeNull(); expect(getNativeMainProfileRequestCount()).toBe(nativeBefore); - selection.release(); - lease.release(); }); test("normal completion detaches cancellation listeners", () => { @@ -222,7 +239,7 @@ test("normal completion detaches cancellation listeners", () => { }); -test("internal decision cancellation releases admission while dispatch remains pending", async () => { +test("internal decision cancellation retains admission while dispatch remains pending", async () => { const before = getActiveTurnCount(); const entered = deferred(); const unblock = deferred(); @@ -239,7 +256,7 @@ test("internal decision cancellation releases admission while dispatch remains p expect(getActiveTurnCount()).toBe(before + 1); try { client.abort(new Error("fixture decision closed")); - expect(getActiveTurnCount()).toBe(before); + expect(getActiveTurnCount()).toBe(before + 1); } finally { unblock.resolve(new Response(null, { status: 499 })); await settled; @@ -247,7 +264,7 @@ test("internal decision cancellation releases admission while dispatch remains p }); -test("Live upgrade disconnect releases admission while resolution remains pending", async () => { +test("Live upgrade disconnect retains admission while resolution remains pending", async () => { const previousHome = process.env.OPENCODEX_HOME; const home = mkdtempSync(join(tmpdir(), "ocx-upgrade-disconnect-")); process.env.OPENCODEX_HOME = home; @@ -269,7 +286,9 @@ test("Live upgrade disconnect releases admission while resolution remains pendin await entered.promise; expect(getActiveTurnCount()).toBe(before + 1); socket.close(); - await until(() => getActiveTurnCount() === before, "Live upgrade admission released"); + expect(getActiveTurnCount()).toBe(before + 1); + unblock.resolve(new Response(null, { status: 499 })); + await until(() => getActiveTurnCount() === before, "Live upgrade admission finalized"); } finally { unblock.resolve(new Response(null, { status: 499 })); resolve.mockRestore(); @@ -295,5 +314,61 @@ test("upgrade transfer detaches ingress cancellation and keeps connection owners expect(getActiveTurnCount()).toBe(before + 1); expect(connection.signal.aborted).toBe(false); connection.abort(new Error("fixture socket closed")); + expect(getActiveTurnCount()).toBe(before + 1); + lease.release(); expect(getActiveTurnCount()).toBe(before); }); + +test("retains saturated admission until stalled work settles without aborting sibling work", () => { + const leases = []; + try { + for (let index = 0; index < 512; index += 1) { + const lease = tryAdmitTurn(); + if (!lease) break; + leases.push(lease); + } + expect(leases.length).toBeGreaterThan(1); + const cancelled = leases[0]!; + const sibling = leases[1]!; + const cancellation = new AbortController(); + const work = new AbortController(); + const siblingWork = new AbortController(); + cancelled.bindAbortSignal(cancellation.signal); + cancelled.bindAbortController(work); + sibling.bindAbortController(siblingWork); + + cancellation.abort(new Error("client disconnected")); + + expect(work.signal.aborted).toBe(true); + expect(siblingWork.signal.aborted).toBe(false); + expect(getActiveTurnCount()).toBe(leases.length); + expect(tryAdmitTurn()).toBeNull(); + + const lateWork = new AbortController(); + cancelled.bindAbortController(lateWork); + expect(lateWork.signal.aborted).toBe(true); + const selection = cancelled.beginCodexAccountSelection(); + expect(selection.claimMainProfile()).toBe(false); + selection.release(); + } finally { + for (const lease of leases) lease.release(); + } +}); + +test("cancellation aborts all controllers when an abort listener releases the lease", () => { + const before = getActiveTurnCount(); + const cancellation = new AbortController(); + const lease = tryAdmitTurn("fixture-reentrant-release", cancellation.signal)!; + const first = new AbortController(); + const second = new AbortController(); + const reason = new Error("fixture reentrant cancellation"); + lease.bindAbortController(first); + lease.bindAbortController(second); + first.signal.addEventListener("abort", () => lease.release(), { once: true }); + try { + cancellation.abort(reason); + expect(first.signal.reason).toBe(reason); + expect(second.signal.reason).toBe(reason); + expect(getActiveTurnCount()).toBe(before); + } finally { lease.release(); } +}); From df7d1e2ea926c7b90205a5da4165f39d5e7dfbc9 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:14:07 +0900 Subject: [PATCH 2/2] fix(server): keep late cancelled controllers from releasing admission Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/server/lifecycle.ts | 6 +-- structure/runtime.md | 2 +- .../active-turn-disconnect.test.ts | 54 ++++++++++++++++--- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/server/lifecycle.ts b/src/server/lifecycle.ts index 90679c65ffe..de3bd32969e 100644 --- a/src/server/lifecycle.ts +++ b/src/server/lifecycle.ts @@ -230,13 +230,13 @@ export function tryAdmitTurn(sessionLaneId?: string, clientSignal?: AbortSignal) ac.abort(cancellationReason ?? new Error("turn already settled")); return; } - transferred = true; - controllers.add(ac); - activeTurns.set(ac, lease); if (cancelled) { ac.abort(cancellationReason ?? new Error("turn cancelled")); return; } + transferred = true; + controllers.add(ac); + activeTurns.set(ac, lease); lease.bindAbortSignal(ac.signal); }, beginCodexAccountSelection() { diff --git a/structure/runtime.md b/structure/runtime.md index 3669f58c40c..229e2fafc2c 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -26,7 +26,7 @@ Native result continuations and function-result injection follow [the mode-speci Native steering follows [the shared WebSocket contract](transports/streaming-health.md#experimental-native-mid-turn-steering); this surface's defaults remain unchanged. -Responses admission and finalization are composed through the [core module ownership](transports/responses.md#core-module-ownership). Kiro's optional account-load admission is process-local and request-owned; its slot ends with the response body or cancellation. Other providers retain their admission path. Cancellation after a slot grant and before the request resumes releases the slot and returns cancellation without dispatch. In `src/server/lifecycle.ts`, cancellation promptly aborts bound controllers but retains active-turn, session-lane and workflow/native ownership until the owner finalizes; late controllers abort immediately, cancelled turns cannot select native accounts, and shutdown may release turns while admission is fenced. +Responses admission and finalization are composed through the [core module ownership](transports/responses.md#core-module-ownership). Kiro's optional account-load admission is process-local and request-owned; its slot ends with the response body or cancellation. Other providers retain their admission path. Cancellation after a slot grant and before the request resumes releases the slot and returns cancellation without dispatch. In `src/server/lifecycle.ts`, cancellation promptly aborts bound controllers but retains active-turn, session-lane and workflow/native ownership until the owner finalizes; late controllers abort immediately without entering the active registry or transferring ownership, cancelled turns cannot select native accounts, and shutdown may release turns while admission is fenced. Catalog HTTP acquisition follows the [proxy-routing contract](catalog.md#remote-catalog-http-proxy-routing). CLI model lists expose shared catalog estimates as `price`; `models price` retains the saved override in `cost` and adds `effectiveCost`. Explicit zero overrides win; unknown prices remain null, and automatic defaults are never persisted to `modelCosts`. Covered by `tests/cli/cli-models.test.ts` and `tests/cli/cli-models-price.test.ts`. Ordinary model discovery, startup sync, and explicit sync apply the [new-arrival catalog contract](catalog.md#shared-catalog) before publishing rows. The scoped persisted decision also updates the live config merge baseline, so subsequent visibility writes preserve manual re-enablement. A file-loaded inventory that already differs from disk can publish HTTP rows through a detached policy projection without adding unsaved live disables; retained catalog writes still refuse stale inventory, and its load provenance follows the [config contract](config.md). diff --git a/tests/codex-integration/active-turn-disconnect.test.ts b/tests/codex-integration/active-turn-disconnect.test.ts index 199ef48d97e..c95b4b5d7dd 100644 --- a/tests/codex-integration/active-turn-disconnect.test.ts +++ b/tests/codex-integration/active-turn-disconnect.test.ts @@ -139,15 +139,18 @@ async function disconnectedTurn(protocol: Protocol, phase: Phase) { } finally { closeClient?.(); unblock.resolve(); - if (phase === "selection") { - await until(() => getActiveTurnCount() === before.turns && sessionLaneMetrics().active === before.lanes, `${protocol}/${phase} finalized`); + try { + if (phase === "selection") { + await until(() => getActiveTurnCount() === before.turns && sessionLaneMetrics().active === before.lanes, `${protocol}/${phase} finalized`); + } + } finally { + prepare.mockRestore(); + abortAndReleaseAllTurns(new Error("fixture teardown")); + await server.stop(true); + await upstream.stop(true); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(home); } - prepare.mockRestore(); - abortAndReleaseAllTurns(new Error("fixture teardown")); - await server.stop(true); - await upstream.stop(true); - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; - removeTreeWithRetry(home); } } @@ -372,3 +375,38 @@ test("cancellation aborts all controllers when an abort listener releases the le expect(getActiveTurnCount()).toBe(before); } finally { lease.release(); } }); + +test.each([false, true])("late cancelled controller unregister preserves its owner's admission (transferred=%s)", transferred => { + const before = activeRegistryMetrics().activeTurns; + const lanes = sessionLaneMetrics().active; + const nativeBefore = getNativeMainProfileRequestCount(); + const ingress = new AbortController(); + const lease = tryAdmitTurn(`fixture-late-controller-${transferred}`, ingress.signal)!; + const selection = lease.beginCodexAccountSelection(); + const duringSelection = getNativeMainProfileRequestCount(); + let released = 0; + lease.attach({ release() { released += 1; } }); + if (transferred) lease.bindAbortController(new AbortController()); + const reason = new Error("fixture cancelled before late binding"); + ingress.abort(reason); + const late = new AbortController(); + late.signal.addEventListener("abort", () => unregisterTurn(late), { once: true }); + try { + lease.bindAbortController(late); + expect(late.signal.reason).toBe(reason); + expect(getActiveTurnCount()).toBe(before.active + 1); + expect(sessionLaneMetrics().active).toBe(lanes + 1); + expect(getNativeMainProfileRequestCount()).toBe(duringSelection); + expect(selection.claimMainProfile()).toBe(false); + expect(released).toBe(0); + expect(lease.isTransferred()).toBe(transferred); + unregisterTurn(late); + expect(getActiveTurnCount()).toBe(before.active + 1); + expect(activeRegistryMetrics().activeTurns.releaseMisses).toBe(before.releaseMisses); + lease.release(); + expect(getActiveTurnCount()).toBe(before.active); + expect(sessionLaneMetrics().active).toBe(lanes); + expect(getNativeMainProfileRequestCount()).toBe(nativeBefore); + expect(released).toBe(1); + } finally { lease.release(); } +});