From 00c09ec342781ba2b165e3812ee0dd8a044fec24 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 19:02:55 +0000 Subject: [PATCH] reactor: a connection its handler still holds at the stop is freed when it lets go, and its socket closed if its peer closed first (#606) The stop kept a held connection's memory, as its handler may still write to it, but nothing freed it once that handler let go. One still in the table kept the reactor's ref, so the release took the count to 1. One the reactor had already let go of (peer EOF, a recv error, a receive-queue overflow) reached 0 and went on the recycle queue, which only the loop drains, so its descriptor stayed open as well. Now the stop lets go of the reactor's ref on a held connection it drops from the table, and marks the socket it closed for it with ClientFd -1. A recycle on a stopped reactor, or one still queued when the loop ended, is done by the thread that let go, or by the teardown's last step: the descriptor is closed unless the stop closed that number, which may be another socket's by then, and the memory freed, overflow segments included. A connection whose send the kernel may still hold keeps its slab, as before. In incremental mode the stop unregisters every connection ring still registered while the ring is open: those the table can't reach, whose peer closed first, included. --- src/ioxide/Connection/Tcp/TcpConnection.cs | 2 + src/ioxide/Reactor/Loop/Reactor.Drainers.cs | 23 +++ .../Reactor/Loop/Reactor.Loop.Incremental.cs | 25 +++ src/ioxide/Reactor/Reactor.Runner.cs | 10 +- .../Reactor/Transport/Tcp/Reactor.Tcp.cs | 12 +- .../Ioxide.Tests.E2E/Core/StopParkedTests.cs | 163 +++++++++++++++++- .../Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs | 4 +- 7 files changed, 229 insertions(+), 10 deletions(-) diff --git a/src/ioxide/Connection/Tcp/TcpConnection.cs b/src/ioxide/Connection/Tcp/TcpConnection.cs index 1be10e7d..36006e0a 100644 --- a/src/ioxide/Connection/Tcp/TcpConnection.cs +++ b/src/ioxide/Connection/Tcp/TcpConnection.cs @@ -9,6 +9,7 @@ public sealed unsafe partial class TcpConnection internal Reactor Reactor => _reactor; + /// The connection's socket, or -1 once its reactor's stop has closed it. public int ClientFd { get; private set; } /// The listener port this connection was accepted on; set per accept. @@ -316,6 +317,7 @@ public void Dispose() NativeMemory.AlignedFree(_msg); _msg = null; } + FreeKeptSegments(); // a no-op after Clear(); a stopped reactor frees a connection without one DisposeIncremental(); } } diff --git a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs index 1c75a425..47408fb9 100644 --- a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs +++ b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs @@ -117,6 +117,11 @@ internal void EnqueueRecycle(TcpConnection conn) } _recycleQ.Enqueue(conn); Wake(); + + if (_postsClosed) + { + DrainStoppedRecycles(); // the reactor is gone: nothing else will take it + } } private void DrainRecycleQ() @@ -127,6 +132,24 @@ private void DrainRecycleQ() } } + // A stopped reactor's recycle, on the thread that let go: with the ring gone only the descriptor and the memory are + // left, and the descriptor only when the stop has not closed it (ClientFd -1). + [MethodImpl(MethodImplOptions.NoInlining)] + private void DrainStoppedRecycles() + { + // As in DrainStoppedPosts: a release that saw the queue open must not be stranded. + Interlocked.MemoryBarrier(); + + while (_recycleQ.TryDequeue(out TcpConnection? conn)) + { + if (conn.ClientFd >= 0) + { + close(conn.ClientFd); + } + conn.Dispose(); + } + } + private void Recycle(TcpConnection conn, int fd) { conn.MarkClosed(); diff --git a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs index 2e15c390..07e59236 100644 --- a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs +++ b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs @@ -101,6 +101,31 @@ private void TeardownConnectionBufRing(TcpConnection conn) // Allocations stay for pool reuse. } + // At the stop, while the ring is open: a ring still registered is one the stop could not return, which a release + // after it frees, and a recv still armed on it could fill it after that. Every gid off the free stack is one. + private void UnregisterHeldConnectionBufRings() + { + if (_freeGids is null) + { + return; + } + + var free = new bool[_maxConnections + 2]; + foreach (ushort gid in _freeGids) + { + free[gid] = true; + } + + for (int gid = 2; gid < free.Length; gid++) + { + if (!free[gid]) + { + var reg = new io_uring_buf_reg { bgid = (ushort)gid }; + io_uring_register(_ring.Fd, IORING_UNREGISTER_PBUF_RING, ®, 1); + } + } + } + // Re-add a fully-consumed buffer to its connection's ring. Reactor-thread-only. private void ReturnConnectionBuffer(TcpConnection conn, ushort bid) { diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs index 400ef75c..2ffe9ecb 100644 --- a/src/ioxide/Reactor/Reactor.Runner.cs +++ b/src/ioxide/Reactor/Reactor.Runner.cs @@ -170,8 +170,9 @@ private void Teardown(bool startFailed) } // Before the fd goes: unregistering is synchronous, closing is not (io_uring_release // defers to a workqueue), so without this the frees below could hand the kernel's pages - // back to the allocator while it still holds them. + // back to the allocator while it still holds them, and so could a held connection's release. UnregisterSharedBufRings(); + UnregisterHeldConnectionBufRings(); // The ring goes either way, its memory with it. When the start failed its number does not: releasing that // mid-run kills an unrelated live connection - #242, whose bisect found the damage is the number being handed @@ -187,7 +188,8 @@ private void Teardown(bool startFailed) } // Pooled connections are past their last send (#250) and their incremental ring is unregistered, - // so their slabs can go, the ones the stop returned included. One a handler still holds stays (#223). + // so their slabs can go, the ones the stop returned included. One a handler still holds stays + // until it lets go (#223). while (_pool.TryPop(out TcpConnection? pooled)) { pooled.Dispose(); @@ -206,9 +208,11 @@ private void Teardown(bool startFailed) } FreeUdpMemory(); - // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever. + // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever, and a + // connection let go of is freed past the ring. _postsClosed = true; DrainStoppedPosts(); + DrainStoppedRecycles(); DrainStoppedOps(); // After all that could still run here, since until then a handshake may read a TLS service's contexts (#543). diff --git a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs index cf94b53d..bb223bf5 100644 --- a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs +++ b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs @@ -433,9 +433,10 @@ private void CloseAcceptedTcpSockets() { for (int fd = 0; fd < _connections.Length; fd++) { - if (_connections[fd] != null) + if (_connections[fd] is { } conn) { close(fd); + conn.SetFd(-1); // the number may be another socket's by the time a handler lets go (#242) } } } @@ -459,8 +460,9 @@ private void ReleaseAcceptedTcpConnections(bool canCancel) } // Once every wake is done, while the ring is open: a connection its handler let go of goes back to the pool as a - // recycle would, and the teardown's last step frees it. One a handler still holds, or whose send the kernel may - // still be reading, keeps its memory, as a write into it would land in freed memory (#223). + // recycle would, and the teardown's last step frees it. One a handler still holds keeps its memory until that + // handler lets go, as a write into it would land in freed memory (#223); one whose send the kernel may still be + // reading keeps it for good. private void ReturnReleasedTcpConnections() { for (int fd = 0; fd < _connections.Length; fd++) @@ -481,6 +483,10 @@ private void ReturnReleasedTcpConnections() conn.Clear(); ReturnToPool(conn); } + else if (!conn.FlushSubmitted) + { + conn.ReleaseReactorRef(); // the handler's release then frees it, past the ring + } } } diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs index 008675b3..3da4e958 100644 --- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs @@ -223,7 +223,7 @@ public static void Register(Runner runner) { string mode = incremental ? "incremental" : "shared ring"; - runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it ({mode})", () => + runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it until it lets go ({mode})", () => { var conns = new System.Collections.Concurrent.ConcurrentDictionary(); using var parked = new CountdownEvent(Roles.Length); @@ -273,13 +273,16 @@ public static void Register(Runner runner) await held.Task; // busy outside ioxide when the stop comes conn.Write("late"u8); await conn.FlushAsync(); - heldFinished.TrySetResult(); break; } } finally { conn.DecRef(); + if (role == 'h') + { + heldFinished.TrySetResult(); + } } }, config); @@ -315,6 +318,12 @@ public static void Register(Runner runner) Assert.True(HasWriteBuffer(conns['h']), "the write slab of a connection its handler still holds was freed under it"); held.TrySetResult(); Assert.True(heldFinished.Task.Wait(ReturnsWithinMs), "the held handler never finished"); + + Assert.True(!HasWriteBuffer(conns['h']), "the write slab of the connection its handler let go of after the stop outlived that release"); + if (incremental) + { + Assert.True(!HasRing(conns['h']), "the buffer ring of the connection its handler let go of after the stop outlived that release"); + } } finally { @@ -325,6 +334,53 @@ public static void Register(Runner runner) } } }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); + + runner.Test($"stop: a connection whose peer closed first, let go of after the stop, has its socket closed and its memory freed ({mode})", () => + { + PeerClosedFirst handler = new(); + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental)); + + using Socket client = SendAndClose(port); + TcpConnection conn = handler.Wait(); + + StopAndJoin(reactor, thread); + Assert.True(HasWriteBuffer(conn), "the write slab of a connection its handler still holds was freed under it"); + + handler.Release(); + AssertClosedAndFreed(conn, client, incremental); + }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); + + runner.Test($"stop: a connection let go of just before the stop, its recycle still queued, has its socket closed and its memory freed ({mode})", () => + { + PeerClosedFirst handler = new(); + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental)); + + using Socket client = SendAndClose(port); + TcpConnection conn = handler.Wait(); + + // The loop takes recycles before posts, so one queued while it runs a post waits for its next pass, and a + // stop requested meanwhile means there is none. + using var running = new ManualResetEventSlim(); + using var busy = new ManualResetEventSlim(); + reactor.ScheduleOnReactor(_ => + { + running.Set(); + busy.Wait(ReturnsWithinMs); + }, null); + try + { + Assert.True(running.Wait(5_000), "the reactor never ran the post"); + handler.Release(); + reactor.Stop(); + } + finally + { + busy.Set(); + } + Assert.True(thread.Join(TimeSpan.FromSeconds(10)), "the reactor did not stop"); + + AssertClosedAndFreed(conn, client, incremental); + }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); } runner.Test("stop: a flush from another thread after the reactor stopped returns", () => @@ -472,6 +528,106 @@ private static async Task ReadRole(TcpConnection conn) } } + // More than the default write slab: shared ring mode spills it into overflow segments, incremental mode grows the + // slab. + private const int StagedBytes = 64 * 1024; + + private static ServerConfig PeerClosedConfig(bool incremental) => new() + { + Incremental = incremental ? new IncrementalOptions { MaxConnections = 16, RecvSlots = 4 } : null, + Tcp = new TcpOptions { WriteOverflow = WriteOverflowStrategy.Segmented }, + }; + + // A handler whose peer closes first, so the reactor has let go of the connection before the stop and the stop never + // sees it. It stages a write it never flushes and waits outside ioxide until the test releases it. + private sealed class PeerClosedFirst + { + private readonly TaskCompletionSource _closed = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task Handle(Reactor reactor, TcpConnection conn) + { + if (!await IsThisTestsConnection(conn)) + { + conn.DecRef(); + return; + } + + await UntilClosed(conn); // the peer's FIN: the reactor lets go of its ref + conn.Write(new byte[StagedBytes]); + _closed.TrySetResult(conn); + + await _release.Task; + conn.DecRef(); + _released.TrySetResult(); + } + + public TcpConnection Wait() + { + Assert.True(_closed.Task.Wait(5_000), "the handler never saw its peer close"); + return _closed.Task.Result; + } + + public void Release() + { + _release.TrySetResult(); + Assert.True(_released.Task.Wait(ReturnsWithinMs), "the handler never let go"); + } + } + + private static async Task UntilClosed(TcpConnection conn) + { + while (true) + { + RecvSnapshot snapshot = await conn.ReadAsync(); + while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) + { + conn.ReturnBuffer(in item); + } + } + + if (snapshot.IsClosed) + { + return; + } + } + } + + // Its request sent, the client closes its side, as one done sending does. + private static Socket SendAndClose(int port) + { + Socket client = Connect(port); + client.Send("hi"u8); + client.Shutdown(SocketShutdown.Send); + return client; + } + + private static void AssertClosedAndFreed(TcpConnection conn, Socket client, bool incremental) + { + Assert.True(!HasWriteBuffer(conn), "the write slab outlived its handler's release"); + Assert.True(KeptSegmentsOf(conn) == 0, $"{KeptSegmentsOf(conn)} overflow segments outlived its handler's release"); + if (incremental) + { + Assert.True(!HasRing(conn), "the buffer ring outlived its handler's release"); + } + + // Closed, the server's side sends the client its FIN, or a reset for what it never read. + client.ReceiveTimeout = ReturnsWithinMs; + string outcome; + try + { + outcome = client.Receive(new byte[16]) == 0 ? "closed" : "data"; + } + catch (SocketException e) + { + outcome = e.SocketErrorCode == SocketError.TimedOut ? "still open" : "closed"; + } + Assert.True(outcome == "closed", $"the server's socket, whose peer closed first, is {outcome} after its handler let go"); + } + private static unsafe bool HasWriteBuffer(TcpConnection connection) => WriteBufferOf(connection) != null; private static unsafe bool HasRing(TcpConnection connection) => BufRingOf(connection) != null && BufSlabOf(connection) != null; @@ -485,6 +641,9 @@ private static async Task ReadRole(TcpConnection conn) [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "BufSlab")] private static extern unsafe ref byte* BufSlabOf(TcpConnection connection); + [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_ovKept")] + private static extern ref int KeptSegmentsOf(TcpConnection connection); + private static async Task SwitchTo(Reactor reactor) => await new ReactorSwitch(reactor); private static void AfterTheStop(Func call, string what) diff --git a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs index 55a6886c..4e03288c 100644 --- a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs @@ -722,8 +722,8 @@ private static void StalledResponse(bool zeroCopy) private static bool IsOpen(int fd) => fcntl(fd, FGetFd) != -1; - // Teardown closes every fd without releasing the reactor's ref, so a later release still sees a - // live connection. Its FIN must not reach whichever socket took the number since. + // Teardown closes every fd, and a release after it frees the connection. Neither its FIN nor that + // free's close may reach whichever socket took the number since. private static void ReleaseAfterStop(Action release) { var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);