diff --git a/src/ioxide/Connection/Tcp/TcpConnection.cs b/src/ioxide/Connection/Tcp/TcpConnection.cs index 1be10e7d..36006e0a 100644 --- a/src/ioxide/Connection/Tcp/TcpConnection.cs +++ b/src/ioxide/Connection/Tcp/TcpConnection.cs @@ -9,6 +9,7 @@ public sealed unsafe partial class TcpConnection internal Reactor Reactor => _reactor; + /// The connection's socket, or -1 once its reactor's stop has closed it. public int ClientFd { get; private set; } /// The listener port this connection was accepted on; set per accept. @@ -316,6 +317,7 @@ public void Dispose() NativeMemory.AlignedFree(_msg); _msg = null; } + FreeKeptSegments(); // a no-op after Clear(); a stopped reactor frees a connection without one DisposeIncremental(); } } diff --git a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs index 1c75a425..47408fb9 100644 --- a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs +++ b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs @@ -117,6 +117,11 @@ internal void EnqueueRecycle(TcpConnection conn) } _recycleQ.Enqueue(conn); Wake(); + + if (_postsClosed) + { + DrainStoppedRecycles(); // the reactor is gone: nothing else will take it + } } private void DrainRecycleQ() @@ -127,6 +132,24 @@ private void DrainRecycleQ() } } + // A stopped reactor's recycle, on the thread that let go: with the ring gone only the descriptor and the memory are + // left, and the descriptor only when the stop has not closed it (ClientFd -1). + [MethodImpl(MethodImplOptions.NoInlining)] + private void DrainStoppedRecycles() + { + // As in DrainStoppedPosts: a release that saw the queue open must not be stranded. + Interlocked.MemoryBarrier(); + + while (_recycleQ.TryDequeue(out TcpConnection? conn)) + { + if (conn.ClientFd >= 0) + { + close(conn.ClientFd); + } + conn.Dispose(); + } + } + private void Recycle(TcpConnection conn, int fd) { conn.MarkClosed(); diff --git a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs index 2e15c390..07e59236 100644 --- a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs +++ b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs @@ -101,6 +101,31 @@ private void TeardownConnectionBufRing(TcpConnection conn) // Allocations stay for pool reuse. } + // At the stop, while the ring is open: a ring still registered is one the stop could not return, which a release + // after it frees, and a recv still armed on it could fill it after that. Every gid off the free stack is one. + private void UnregisterHeldConnectionBufRings() + { + if (_freeGids is null) + { + return; + } + + var free = new bool[_maxConnections + 2]; + foreach (ushort gid in _freeGids) + { + free[gid] = true; + } + + for (int gid = 2; gid < free.Length; gid++) + { + if (!free[gid]) + { + var reg = new io_uring_buf_reg { bgid = (ushort)gid }; + io_uring_register(_ring.Fd, IORING_UNREGISTER_PBUF_RING, ®, 1); + } + } + } + // Re-add a fully-consumed buffer to its connection's ring. Reactor-thread-only. private void ReturnConnectionBuffer(TcpConnection conn, ushort bid) { diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs index 400ef75c..2ffe9ecb 100644 --- a/src/ioxide/Reactor/Reactor.Runner.cs +++ b/src/ioxide/Reactor/Reactor.Runner.cs @@ -170,8 +170,9 @@ private void Teardown(bool startFailed) } // Before the fd goes: unregistering is synchronous, closing is not (io_uring_release // defers to a workqueue), so without this the frees below could hand the kernel's pages - // back to the allocator while it still holds them. + // back to the allocator while it still holds them, and so could a held connection's release. UnregisterSharedBufRings(); + UnregisterHeldConnectionBufRings(); // The ring goes either way, its memory with it. When the start failed its number does not: releasing that // mid-run kills an unrelated live connection - #242, whose bisect found the damage is the number being handed @@ -187,7 +188,8 @@ private void Teardown(bool startFailed) } // Pooled connections are past their last send (#250) and their incremental ring is unregistered, - // so their slabs can go, the ones the stop returned included. One a handler still holds stays (#223). + // so their slabs can go, the ones the stop returned included. One a handler still holds stays + // until it lets go (#223). while (_pool.TryPop(out TcpConnection? pooled)) { pooled.Dispose(); @@ -206,9 +208,11 @@ private void Teardown(bool startFailed) } FreeUdpMemory(); - // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever. + // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever, and a + // connection let go of is freed past the ring. _postsClosed = true; DrainStoppedPosts(); + DrainStoppedRecycles(); DrainStoppedOps(); // After all that could still run here, since until then a handshake may read a TLS service's contexts (#543). diff --git a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs index cf94b53d..bb223bf5 100644 --- a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs +++ b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs @@ -433,9 +433,10 @@ private void CloseAcceptedTcpSockets() { for (int fd = 0; fd < _connections.Length; fd++) { - if (_connections[fd] != null) + if (_connections[fd] is { } conn) { close(fd); + conn.SetFd(-1); // the number may be another socket's by the time a handler lets go (#242) } } } @@ -459,8 +460,9 @@ private void ReleaseAcceptedTcpConnections(bool canCancel) } // Once every wake is done, while the ring is open: a connection its handler let go of goes back to the pool as a - // recycle would, and the teardown's last step frees it. One a handler still holds, or whose send the kernel may - // still be reading, keeps its memory, as a write into it would land in freed memory (#223). + // recycle would, and the teardown's last step frees it. One a handler still holds keeps its memory until that + // handler lets go, as a write into it would land in freed memory (#223); one whose send the kernel may still be + // reading keeps it for good. private void ReturnReleasedTcpConnections() { for (int fd = 0; fd < _connections.Length; fd++) @@ -481,6 +483,10 @@ private void ReturnReleasedTcpConnections() conn.Clear(); ReturnToPool(conn); } + else if (!conn.FlushSubmitted) + { + conn.ReleaseReactorRef(); // the handler's release then frees it, past the ring + } } } diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs index 008675b3..3da4e958 100644 --- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs @@ -223,7 +223,7 @@ public static void Register(Runner runner) { string mode = incremental ? "incremental" : "shared ring"; - runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it ({mode})", () => + runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it until it lets go ({mode})", () => { var conns = new System.Collections.Concurrent.ConcurrentDictionary(); using var parked = new CountdownEvent(Roles.Length); @@ -273,13 +273,16 @@ public static void Register(Runner runner) await held.Task; // busy outside ioxide when the stop comes conn.Write("late"u8); await conn.FlushAsync(); - heldFinished.TrySetResult(); break; } } finally { conn.DecRef(); + if (role == 'h') + { + heldFinished.TrySetResult(); + } } }, config); @@ -315,6 +318,12 @@ public static void Register(Runner runner) Assert.True(HasWriteBuffer(conns['h']), "the write slab of a connection its handler still holds was freed under it"); held.TrySetResult(); Assert.True(heldFinished.Task.Wait(ReturnsWithinMs), "the held handler never finished"); + + Assert.True(!HasWriteBuffer(conns['h']), "the write slab of the connection its handler let go of after the stop outlived that release"); + if (incremental) + { + Assert.True(!HasRing(conns['h']), "the buffer ring of the connection its handler let go of after the stop outlived that release"); + } } finally { @@ -325,6 +334,53 @@ public static void Register(Runner runner) } } }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); + + runner.Test($"stop: a connection whose peer closed first, let go of after the stop, has its socket closed and its memory freed ({mode})", () => + { + PeerClosedFirst handler = new(); + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental)); + + using Socket client = SendAndClose(port); + TcpConnection conn = handler.Wait(); + + StopAndJoin(reactor, thread); + Assert.True(HasWriteBuffer(conn), "the write slab of a connection its handler still holds was freed under it"); + + handler.Release(); + AssertClosedAndFreed(conn, client, incremental); + }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); + + runner.Test($"stop: a connection let go of just before the stop, its recycle still queued, has its socket closed and its memory freed ({mode})", () => + { + PeerClosedFirst handler = new(); + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental)); + + using Socket client = SendAndClose(port); + TcpConnection conn = handler.Wait(); + + // The loop takes recycles before posts, so one queued while it runs a post waits for its next pass, and a + // stop requested meanwhile means there is none. + using var running = new ManualResetEventSlim(); + using var busy = new ManualResetEventSlim(); + reactor.ScheduleOnReactor(_ => + { + running.Set(); + busy.Wait(ReturnsWithinMs); + }, null); + try + { + Assert.True(running.Wait(5_000), "the reactor never ran the post"); + handler.Release(); + reactor.Stop(); + } + finally + { + busy.Set(); + } + Assert.True(thread.Join(TimeSpan.FromSeconds(10)), "the reactor did not stop"); + + AssertClosedAndFreed(conn, client, incremental); + }, skip: incremental && !TestServer.KernelAtLeast(6, 12)); } runner.Test("stop: a flush from another thread after the reactor stopped returns", () => @@ -472,6 +528,106 @@ private static async Task ReadRole(TcpConnection conn) } } + // More than the default write slab: shared ring mode spills it into overflow segments, incremental mode grows the + // slab. + private const int StagedBytes = 64 * 1024; + + private static ServerConfig PeerClosedConfig(bool incremental) => new() + { + Incremental = incremental ? new IncrementalOptions { MaxConnections = 16, RecvSlots = 4 } : null, + Tcp = new TcpOptions { WriteOverflow = WriteOverflowStrategy.Segmented }, + }; + + // A handler whose peer closes first, so the reactor has let go of the connection before the stop and the stop never + // sees it. It stages a write it never flushes and waits outside ioxide until the test releases it. + private sealed class PeerClosedFirst + { + private readonly TaskCompletionSource _closed = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task Handle(Reactor reactor, TcpConnection conn) + { + if (!await IsThisTestsConnection(conn)) + { + conn.DecRef(); + return; + } + + await UntilClosed(conn); // the peer's FIN: the reactor lets go of its ref + conn.Write(new byte[StagedBytes]); + _closed.TrySetResult(conn); + + await _release.Task; + conn.DecRef(); + _released.TrySetResult(); + } + + public TcpConnection Wait() + { + Assert.True(_closed.Task.Wait(5_000), "the handler never saw its peer close"); + return _closed.Task.Result; + } + + public void Release() + { + _release.TrySetResult(); + Assert.True(_released.Task.Wait(ReturnsWithinMs), "the handler never let go"); + } + } + + private static async Task UntilClosed(TcpConnection conn) + { + while (true) + { + RecvSnapshot snapshot = await conn.ReadAsync(); + while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) + { + conn.ReturnBuffer(in item); + } + } + + if (snapshot.IsClosed) + { + return; + } + } + } + + // Its request sent, the client closes its side, as one done sending does. + private static Socket SendAndClose(int port) + { + Socket client = Connect(port); + client.Send("hi"u8); + client.Shutdown(SocketShutdown.Send); + return client; + } + + private static void AssertClosedAndFreed(TcpConnection conn, Socket client, bool incremental) + { + Assert.True(!HasWriteBuffer(conn), "the write slab outlived its handler's release"); + Assert.True(KeptSegmentsOf(conn) == 0, $"{KeptSegmentsOf(conn)} overflow segments outlived its handler's release"); + if (incremental) + { + Assert.True(!HasRing(conn), "the buffer ring outlived its handler's release"); + } + + // Closed, the server's side sends the client its FIN, or a reset for what it never read. + client.ReceiveTimeout = ReturnsWithinMs; + string outcome; + try + { + outcome = client.Receive(new byte[16]) == 0 ? "closed" : "data"; + } + catch (SocketException e) + { + outcome = e.SocketErrorCode == SocketError.TimedOut ? "still open" : "closed"; + } + Assert.True(outcome == "closed", $"the server's socket, whose peer closed first, is {outcome} after its handler let go"); + } + private static unsafe bool HasWriteBuffer(TcpConnection connection) => WriteBufferOf(connection) != null; private static unsafe bool HasRing(TcpConnection connection) => BufRingOf(connection) != null && BufSlabOf(connection) != null; @@ -485,6 +641,9 @@ private static async Task ReadRole(TcpConnection conn) [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "BufSlab")] private static extern unsafe ref byte* BufSlabOf(TcpConnection connection); + [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_ovKept")] + private static extern ref int KeptSegmentsOf(TcpConnection connection); + private static async Task SwitchTo(Reactor reactor) => await new ReactorSwitch(reactor); private static void AfterTheStop(Func call, string what) diff --git a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs index 55a6886c..4e03288c 100644 --- a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs @@ -722,8 +722,8 @@ private static void StalledResponse(bool zeroCopy) private static bool IsOpen(int fd) => fcntl(fd, FGetFd) != -1; - // Teardown closes every fd without releasing the reactor's ref, so a later release still sees a - // live connection. Its FIN must not reach whichever socket took the number since. + // Teardown closes every fd, and a release after it frees the connection. Neither its FIN nor that + // free's close may reach whichever socket took the number since. private static void ReleaseAfterStop(Action release) { var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);