diff --git a/src/ioxide/Connection/Tcp/TcpConnection.cs b/src/ioxide/Connection/Tcp/TcpConnection.cs
index 1be10e7d..36006e0a 100644
--- a/src/ioxide/Connection/Tcp/TcpConnection.cs
+++ b/src/ioxide/Connection/Tcp/TcpConnection.cs
@@ -9,6 +9,7 @@ public sealed unsafe partial class TcpConnection
internal Reactor Reactor => _reactor;
+ /// The connection's socket, or -1 once its reactor's stop has closed it.
public int ClientFd { get; private set; }
/// The listener port this connection was accepted on; set per accept.
@@ -316,6 +317,7 @@ public void Dispose()
NativeMemory.AlignedFree(_msg);
_msg = null;
}
+ FreeKeptSegments(); // a no-op after Clear(); a stopped reactor frees a connection without one
DisposeIncremental();
}
}
diff --git a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs
index 1c75a425..47408fb9 100644
--- a/src/ioxide/Reactor/Loop/Reactor.Drainers.cs
+++ b/src/ioxide/Reactor/Loop/Reactor.Drainers.cs
@@ -117,6 +117,11 @@ internal void EnqueueRecycle(TcpConnection conn)
}
_recycleQ.Enqueue(conn);
Wake();
+
+ if (_postsClosed)
+ {
+ DrainStoppedRecycles(); // the reactor is gone: nothing else will take it
+ }
}
private void DrainRecycleQ()
@@ -127,6 +132,24 @@ private void DrainRecycleQ()
}
}
+ // A stopped reactor's recycle, on the thread that let go: with the ring gone only the descriptor and the memory are
+ // left, and the descriptor only when the stop has not closed it (ClientFd -1).
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private void DrainStoppedRecycles()
+ {
+ // As in DrainStoppedPosts: a release that saw the queue open must not be stranded.
+ Interlocked.MemoryBarrier();
+
+ while (_recycleQ.TryDequeue(out TcpConnection? conn))
+ {
+ if (conn.ClientFd >= 0)
+ {
+ close(conn.ClientFd);
+ }
+ conn.Dispose();
+ }
+ }
+
private void Recycle(TcpConnection conn, int fd)
{
conn.MarkClosed();
diff --git a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs
index 2e15c390..07e59236 100644
--- a/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs
+++ b/src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs
@@ -101,6 +101,31 @@ private void TeardownConnectionBufRing(TcpConnection conn)
// Allocations stay for pool reuse.
}
+ // At the stop, while the ring is open: a ring still registered is one the stop could not return, which a release
+ // after it frees, and a recv still armed on it could fill it after that. Every gid off the free stack is one.
+ private void UnregisterHeldConnectionBufRings()
+ {
+ if (_freeGids is null)
+ {
+ return;
+ }
+
+ var free = new bool[_maxConnections + 2];
+ foreach (ushort gid in _freeGids)
+ {
+ free[gid] = true;
+ }
+
+ for (int gid = 2; gid < free.Length; gid++)
+ {
+ if (!free[gid])
+ {
+ var reg = new io_uring_buf_reg { bgid = (ushort)gid };
+ io_uring_register(_ring.Fd, IORING_UNREGISTER_PBUF_RING, ®, 1);
+ }
+ }
+ }
+
// Re-add a fully-consumed buffer to its connection's ring. Reactor-thread-only.
private void ReturnConnectionBuffer(TcpConnection conn, ushort bid)
{
diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs
index 400ef75c..2ffe9ecb 100644
--- a/src/ioxide/Reactor/Reactor.Runner.cs
+++ b/src/ioxide/Reactor/Reactor.Runner.cs
@@ -170,8 +170,9 @@ private void Teardown(bool startFailed)
}
// Before the fd goes: unregistering is synchronous, closing is not (io_uring_release
// defers to a workqueue), so without this the frees below could hand the kernel's pages
- // back to the allocator while it still holds them.
+ // back to the allocator while it still holds them, and so could a held connection's release.
UnregisterSharedBufRings();
+ UnregisterHeldConnectionBufRings();
// The ring goes either way, its memory with it. When the start failed its number does not: releasing that
// mid-run kills an unrelated live connection - #242, whose bisect found the damage is the number being handed
@@ -187,7 +188,8 @@ private void Teardown(bool startFailed)
}
// Pooled connections are past their last send (#250) and their incremental ring is unregistered,
- // so their slabs can go, the ones the stop returned included. One a handler still holds stays (#223).
+ // so their slabs can go, the ones the stop returned included. One a handler still holds stays
+ // until it lets go (#223).
while (_pool.TryPop(out TcpConnection? pooled))
{
pooled.Dispose();
@@ -206,9 +208,11 @@ private void Teardown(bool startFailed)
}
FreeUdpMemory();
- // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever.
+ // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever, and a
+ // connection let go of is freed past the ring.
_postsClosed = true;
DrainStoppedPosts();
+ DrainStoppedRecycles();
DrainStoppedOps();
// After all that could still run here, since until then a handshake may read a TLS service's contexts (#543).
diff --git a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs
index cf94b53d..bb223bf5 100644
--- a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs
+++ b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs
@@ -433,9 +433,10 @@ private void CloseAcceptedTcpSockets()
{
for (int fd = 0; fd < _connections.Length; fd++)
{
- if (_connections[fd] != null)
+ if (_connections[fd] is { } conn)
{
close(fd);
+ conn.SetFd(-1); // the number may be another socket's by the time a handler lets go (#242)
}
}
}
@@ -459,8 +460,9 @@ private void ReleaseAcceptedTcpConnections(bool canCancel)
}
// Once every wake is done, while the ring is open: a connection its handler let go of goes back to the pool as a
- // recycle would, and the teardown's last step frees it. One a handler still holds, or whose send the kernel may
- // still be reading, keeps its memory, as a write into it would land in freed memory (#223).
+ // recycle would, and the teardown's last step frees it. One a handler still holds keeps its memory until that
+ // handler lets go, as a write into it would land in freed memory (#223); one whose send the kernel may still be
+ // reading keeps it for good.
private void ReturnReleasedTcpConnections()
{
for (int fd = 0; fd < _connections.Length; fd++)
@@ -481,6 +483,10 @@ private void ReturnReleasedTcpConnections()
conn.Clear();
ReturnToPool(conn);
}
+ else if (!conn.FlushSubmitted)
+ {
+ conn.ReleaseReactorRef(); // the handler's release then frees it, past the ring
+ }
}
}
diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs
index 008675b3..3da4e958 100644
--- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs
+++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs
@@ -223,7 +223,7 @@ public static void Register(Runner runner)
{
string mode = incremental ? "incremental" : "shared ring";
- runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it ({mode})", () =>
+ runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it until it lets go ({mode})", () =>
{
var conns = new System.Collections.Concurrent.ConcurrentDictionary();
using var parked = new CountdownEvent(Roles.Length);
@@ -273,13 +273,16 @@ public static void Register(Runner runner)
await held.Task; // busy outside ioxide when the stop comes
conn.Write("late"u8);
await conn.FlushAsync();
- heldFinished.TrySetResult();
break;
}
}
finally
{
conn.DecRef();
+ if (role == 'h')
+ {
+ heldFinished.TrySetResult();
+ }
}
}, config);
@@ -315,6 +318,12 @@ public static void Register(Runner runner)
Assert.True(HasWriteBuffer(conns['h']), "the write slab of a connection its handler still holds was freed under it");
held.TrySetResult();
Assert.True(heldFinished.Task.Wait(ReturnsWithinMs), "the held handler never finished");
+
+ Assert.True(!HasWriteBuffer(conns['h']), "the write slab of the connection its handler let go of after the stop outlived that release");
+ if (incremental)
+ {
+ Assert.True(!HasRing(conns['h']), "the buffer ring of the connection its handler let go of after the stop outlived that release");
+ }
}
finally
{
@@ -325,6 +334,53 @@ public static void Register(Runner runner)
}
}
}, skip: incremental && !TestServer.KernelAtLeast(6, 12));
+
+ runner.Test($"stop: a connection whose peer closed first, let go of after the stop, has its socket closed and its memory freed ({mode})", () =>
+ {
+ PeerClosedFirst handler = new();
+ (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental));
+
+ using Socket client = SendAndClose(port);
+ TcpConnection conn = handler.Wait();
+
+ StopAndJoin(reactor, thread);
+ Assert.True(HasWriteBuffer(conn), "the write slab of a connection its handler still holds was freed under it");
+
+ handler.Release();
+ AssertClosedAndFreed(conn, client, incremental);
+ }, skip: incremental && !TestServer.KernelAtLeast(6, 12));
+
+ runner.Test($"stop: a connection let go of just before the stop, its recycle still queued, has its socket closed and its memory freed ({mode})", () =>
+ {
+ PeerClosedFirst handler = new();
+ (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental));
+
+ using Socket client = SendAndClose(port);
+ TcpConnection conn = handler.Wait();
+
+ // The loop takes recycles before posts, so one queued while it runs a post waits for its next pass, and a
+ // stop requested meanwhile means there is none.
+ using var running = new ManualResetEventSlim();
+ using var busy = new ManualResetEventSlim();
+ reactor.ScheduleOnReactor(_ =>
+ {
+ running.Set();
+ busy.Wait(ReturnsWithinMs);
+ }, null);
+ try
+ {
+ Assert.True(running.Wait(5_000), "the reactor never ran the post");
+ handler.Release();
+ reactor.Stop();
+ }
+ finally
+ {
+ busy.Set();
+ }
+ Assert.True(thread.Join(TimeSpan.FromSeconds(10)), "the reactor did not stop");
+
+ AssertClosedAndFreed(conn, client, incremental);
+ }, skip: incremental && !TestServer.KernelAtLeast(6, 12));
}
runner.Test("stop: a flush from another thread after the reactor stopped returns", () =>
@@ -472,6 +528,106 @@ private static async Task ReadRole(TcpConnection conn)
}
}
+ // More than the default write slab: shared ring mode spills it into overflow segments, incremental mode grows the
+ // slab.
+ private const int StagedBytes = 64 * 1024;
+
+ private static ServerConfig PeerClosedConfig(bool incremental) => new()
+ {
+ Incremental = incremental ? new IncrementalOptions { MaxConnections = 16, RecvSlots = 4 } : null,
+ Tcp = new TcpOptions { WriteOverflow = WriteOverflowStrategy.Segmented },
+ };
+
+ // A handler whose peer closes first, so the reactor has let go of the connection before the stop and the stop never
+ // sees it. It stages a write it never flushes and waits outside ioxide until the test releases it.
+ private sealed class PeerClosedFirst
+ {
+ private readonly TaskCompletionSource _closed = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public async Task Handle(Reactor reactor, TcpConnection conn)
+ {
+ if (!await IsThisTestsConnection(conn))
+ {
+ conn.DecRef();
+ return;
+ }
+
+ await UntilClosed(conn); // the peer's FIN: the reactor lets go of its ref
+ conn.Write(new byte[StagedBytes]);
+ _closed.TrySetResult(conn);
+
+ await _release.Task;
+ conn.DecRef();
+ _released.TrySetResult();
+ }
+
+ public TcpConnection Wait()
+ {
+ Assert.True(_closed.Task.Wait(5_000), "the handler never saw its peer close");
+ return _closed.Task.Result;
+ }
+
+ public void Release()
+ {
+ _release.TrySetResult();
+ Assert.True(_released.Task.Wait(ReturnsWithinMs), "the handler never let go");
+ }
+ }
+
+ private static async Task UntilClosed(TcpConnection conn)
+ {
+ while (true)
+ {
+ RecvSnapshot snapshot = await conn.ReadAsync();
+ while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
+ {
+ if (item.HasBuffer)
+ {
+ conn.ReturnBuffer(in item);
+ }
+ }
+
+ if (snapshot.IsClosed)
+ {
+ return;
+ }
+ }
+ }
+
+ // Its request sent, the client closes its side, as one done sending does.
+ private static Socket SendAndClose(int port)
+ {
+ Socket client = Connect(port);
+ client.Send("hi"u8);
+ client.Shutdown(SocketShutdown.Send);
+ return client;
+ }
+
+ private static void AssertClosedAndFreed(TcpConnection conn, Socket client, bool incremental)
+ {
+ Assert.True(!HasWriteBuffer(conn), "the write slab outlived its handler's release");
+ Assert.True(KeptSegmentsOf(conn) == 0, $"{KeptSegmentsOf(conn)} overflow segments outlived its handler's release");
+ if (incremental)
+ {
+ Assert.True(!HasRing(conn), "the buffer ring outlived its handler's release");
+ }
+
+ // Closed, the server's side sends the client its FIN, or a reset for what it never read.
+ client.ReceiveTimeout = ReturnsWithinMs;
+ string outcome;
+ try
+ {
+ outcome = client.Receive(new byte[16]) == 0 ? "closed" : "data";
+ }
+ catch (SocketException e)
+ {
+ outcome = e.SocketErrorCode == SocketError.TimedOut ? "still open" : "closed";
+ }
+ Assert.True(outcome == "closed", $"the server's socket, whose peer closed first, is {outcome} after its handler let go");
+ }
+
private static unsafe bool HasWriteBuffer(TcpConnection connection) => WriteBufferOf(connection) != null;
private static unsafe bool HasRing(TcpConnection connection) => BufRingOf(connection) != null && BufSlabOf(connection) != null;
@@ -485,6 +641,9 @@ private static async Task ReadRole(TcpConnection conn)
[System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "BufSlab")]
private static extern unsafe ref byte* BufSlabOf(TcpConnection connection);
+ [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_ovKept")]
+ private static extern ref int KeptSegmentsOf(TcpConnection connection);
+
private static async Task SwitchTo(Reactor reactor) => await new ReactorSwitch(reactor);
private static void AfterTheStop(Func call, string what)
diff --git a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs
index 55a6886c..4e03288c 100644
--- a/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs
+++ b/tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs
@@ -722,8 +722,8 @@ private static void StalledResponse(bool zeroCopy)
private static bool IsOpen(int fd) => fcntl(fd, FGetFd) != -1;
- // Teardown closes every fd without releasing the reactor's ref, so a later release still sees a
- // live connection. Its FIN must not reach whichever socket took the number since.
+ // Teardown closes every fd, and a release after it frees the connection. Neither its FIN nor that
+ // free's close may reach whichever socket took the number since.
private static void ReleaseAfterStop(Action release)
{
var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);