Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/ioxide/Connection/Tcp/TcpConnection.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public sealed unsafe partial class TcpConnection

internal Reactor Reactor => _reactor;

/// <summary>The connection's socket, or -1 once its reactor's stop has closed it.</summary>
public int ClientFd { get; private set; }

/// <summary>The listener port this connection was accepted on; set per accept.</summary>
Expand Down Expand Up @@ -316,6 +317,7 @@ public void Dispose()
NativeMemory.AlignedFree(_msg);
_msg = null;
}
FreeKeptSegments(); // a no-op after Clear(); a stopped reactor frees a connection without one
DisposeIncremental();
}
}
23 changes: 23 additions & 0 deletions src/ioxide/Reactor/Loop/Reactor.Drainers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ internal void EnqueueRecycle(TcpConnection conn)
}
_recycleQ.Enqueue(conn);
Wake();

if (_postsClosed)
{
DrainStoppedRecycles(); // the reactor is gone: nothing else will take it
}
}

private void DrainRecycleQ()
Expand All @@ -127,6 +132,24 @@ private void DrainRecycleQ()
}
}

// A stopped reactor's recycle, on the thread that let go: with the ring gone only the descriptor and the memory are
// left, and the descriptor only when the stop has not closed it (ClientFd -1).
[MethodImpl(MethodImplOptions.NoInlining)]
private void DrainStoppedRecycles()
{
// As in DrainStoppedPosts: a release that saw the queue open must not be stranded.
Interlocked.MemoryBarrier();

while (_recycleQ.TryDequeue(out TcpConnection? conn))
{
if (conn.ClientFd >= 0)
{
close(conn.ClientFd);
}
conn.Dispose();
}
}

private void Recycle(TcpConnection conn, int fd)
{
conn.MarkClosed();
Expand Down
25 changes: 25 additions & 0 deletions src/ioxide/Reactor/Loop/Reactor.Loop.Incremental.cs
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,31 @@ private void TeardownConnectionBufRing(TcpConnection conn)
// Allocations stay for pool reuse.
}

// At the stop, while the ring is open: a ring still registered is one the stop could not return, which a release
// after it frees, and a recv still armed on it could fill it after that. Every gid off the free stack is one.
private void UnregisterHeldConnectionBufRings()
{
if (_freeGids is null)
{
return;
}

var free = new bool[_maxConnections + 2];
foreach (ushort gid in _freeGids)
{
free[gid] = true;
}

for (int gid = 2; gid < free.Length; gid++)
{
if (!free[gid])
{
var reg = new io_uring_buf_reg { bgid = (ushort)gid };
io_uring_register(_ring.Fd, IORING_UNREGISTER_PBUF_RING, &reg, 1);
}
}
}

// Re-add a fully-consumed buffer to its connection's ring. Reactor-thread-only.
private void ReturnConnectionBuffer(TcpConnection conn, ushort bid)
{
Expand Down
10 changes: 7 additions & 3 deletions src/ioxide/Reactor/Reactor.Runner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,9 @@ private void Teardown(bool startFailed)
}
// Before the fd goes: unregistering is synchronous, closing is not (io_uring_release
// defers to a workqueue), so without this the frees below could hand the kernel's pages
// back to the allocator while it still holds them.
// back to the allocator while it still holds them, and so could a held connection's release.
UnregisterSharedBufRings();
UnregisterHeldConnectionBufRings();

// The ring goes either way, its memory with it. When the start failed its number does not: releasing that
// mid-run kills an unrelated live connection - #242, whose bisect found the damage is the number being handed
Expand All @@ -187,7 +188,8 @@ private void Teardown(bool startFailed)
}

// Pooled connections are past their last send (#250) and their incremental ring is unregistered,
// so their slabs can go, the ones the stop returned included. One a handler still holds stays (#223).
// so their slabs can go, the ones the stop returned included. One a handler still holds stays
// until it lets go (#223).
while (_pool.TryPop(out TcpConnection? pooled))
{
pooled.Dispose();
Expand All @@ -206,9 +208,11 @@ private void Teardown(bool startFailed)
}
FreeUdpMemory();

// Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever.
// Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever, and a
// connection let go of is freed past the ring.
_postsClosed = true;
DrainStoppedPosts();
DrainStoppedRecycles();
DrainStoppedOps();

// After all that could still run here, since until then a handshake may read a TLS service's contexts (#543).
Expand Down
12 changes: 9 additions & 3 deletions src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs
Original file line number Diff line number Diff line change
Expand Up @@ -433,9 +433,10 @@ private void CloseAcceptedTcpSockets()
{
for (int fd = 0; fd < _connections.Length; fd++)
{
if (_connections[fd] != null)
if (_connections[fd] is { } conn)
{
close(fd);
conn.SetFd(-1); // the number may be another socket's by the time a handler lets go (#242)
}
}
}
Expand All @@ -459,8 +460,9 @@ private void ReleaseAcceptedTcpConnections(bool canCancel)
}

// Once every wake is done, while the ring is open: a connection its handler let go of goes back to the pool as a
// recycle would, and the teardown's last step frees it. One a handler still holds, or whose send the kernel may
// still be reading, keeps its memory, as a write into it would land in freed memory (#223).
// recycle would, and the teardown's last step frees it. One a handler still holds keeps its memory until that
// handler lets go, as a write into it would land in freed memory (#223); one whose send the kernel may still be
// reading keeps it for good.
private void ReturnReleasedTcpConnections()
{
for (int fd = 0; fd < _connections.Length; fd++)
Expand All @@ -481,6 +483,10 @@ private void ReturnReleasedTcpConnections()
conn.Clear();
ReturnToPool(conn);
}
else if (!conn.FlushSubmitted)
{
conn.ReleaseReactorRef(); // the handler's release then frees it, past the ring
}
}
}

Expand Down
163 changes: 161 additions & 2 deletions tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ public static void Register(Runner runner)
{
string mode = incremental ? "incremental" : "shared ring";

runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it ({mode})", () =>
runner.Test($"stop: a connection its handler let go of has its memory freed, one a handler still holds keeps it until it lets go ({mode})", () =>
{
var conns = new System.Collections.Concurrent.ConcurrentDictionary<char, TcpConnection>();
using var parked = new CountdownEvent(Roles.Length);
Expand Down Expand Up @@ -273,13 +273,16 @@ public static void Register(Runner runner)
await held.Task; // busy outside ioxide when the stop comes
conn.Write("late"u8);
await conn.FlushAsync();
heldFinished.TrySetResult();
break;
}
}
finally
{
conn.DecRef();
if (role == 'h')
{
heldFinished.TrySetResult();
}
}
}, config);

Expand Down Expand Up @@ -315,6 +318,12 @@ public static void Register(Runner runner)
Assert.True(HasWriteBuffer(conns['h']), "the write slab of a connection its handler still holds was freed under it");
held.TrySetResult();
Assert.True(heldFinished.Task.Wait(ReturnsWithinMs), "the held handler never finished");

Assert.True(!HasWriteBuffer(conns['h']), "the write slab of the connection its handler let go of after the stop outlived that release");
if (incremental)
{
Assert.True(!HasRing(conns['h']), "the buffer ring of the connection its handler let go of after the stop outlived that release");
}
}
finally
{
Expand All @@ -325,6 +334,53 @@ public static void Register(Runner runner)
}
}
}, skip: incremental && !TestServer.KernelAtLeast(6, 12));

runner.Test($"stop: a connection whose peer closed first, let go of after the stop, has its socket closed and its memory freed ({mode})", () =>
{
PeerClosedFirst handler = new();
(int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental));

using Socket client = SendAndClose(port);
TcpConnection conn = handler.Wait();

StopAndJoin(reactor, thread);
Assert.True(HasWriteBuffer(conn), "the write slab of a connection its handler still holds was freed under it");

handler.Release();
AssertClosedAndFreed(conn, client, incremental);
}, skip: incremental && !TestServer.KernelAtLeast(6, 12));

runner.Test($"stop: a connection let go of just before the stop, its recycle still queued, has its socket closed and its memory freed ({mode})", () =>
{
PeerClosedFirst handler = new();
(int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(handler.Handle, PeerClosedConfig(incremental));

using Socket client = SendAndClose(port);
TcpConnection conn = handler.Wait();

// The loop takes recycles before posts, so one queued while it runs a post waits for its next pass, and a
// stop requested meanwhile means there is none.
using var running = new ManualResetEventSlim();
using var busy = new ManualResetEventSlim();
reactor.ScheduleOnReactor(_ =>
{
running.Set();
busy.Wait(ReturnsWithinMs);
}, null);
try
{
Assert.True(running.Wait(5_000), "the reactor never ran the post");
handler.Release();
reactor.Stop();
}
finally
{
busy.Set();
}
Assert.True(thread.Join(TimeSpan.FromSeconds(10)), "the reactor did not stop");

AssertClosedAndFreed(conn, client, incremental);
}, skip: incremental && !TestServer.KernelAtLeast(6, 12));
}

runner.Test("stop: a flush from another thread after the reactor stopped returns", () =>
Expand Down Expand Up @@ -472,6 +528,106 @@ private static async Task<int> ReadRole(TcpConnection conn)
}
}

// More than the default write slab: shared ring mode spills it into overflow segments, incremental mode grows the
// slab.
private const int StagedBytes = 64 * 1024;

private static ServerConfig PeerClosedConfig(bool incremental) => new()
{
Incremental = incremental ? new IncrementalOptions { MaxConnections = 16, RecvSlots = 4 } : null,
Tcp = new TcpOptions { WriteOverflow = WriteOverflowStrategy.Segmented },
};

// A handler whose peer closes first, so the reactor has let go of the connection before the stop and the stop never
// sees it. It stages a write it never flushes and waits outside ioxide until the test releases it.
private sealed class PeerClosedFirst
{
private readonly TaskCompletionSource<TcpConnection> _closed = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);

public async Task Handle(Reactor reactor, TcpConnection conn)
{
if (!await IsThisTestsConnection(conn))
{
conn.DecRef();
return;
}

await UntilClosed(conn); // the peer's FIN: the reactor lets go of its ref
conn.Write(new byte[StagedBytes]);
_closed.TrySetResult(conn);

await _release.Task;
conn.DecRef();
_released.TrySetResult();
}

public TcpConnection Wait()
{
Assert.True(_closed.Task.Wait(5_000), "the handler never saw its peer close");
return _closed.Task.Result;
}

public void Release()
{
_release.TrySetResult();
Assert.True(_released.Task.Wait(ReturnsWithinMs), "the handler never let go");
}
}

private static async Task UntilClosed(TcpConnection conn)
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
conn.ReturnBuffer(in item);
}
}

if (snapshot.IsClosed)
{
return;
}
}
}

// Its request sent, the client closes its side, as one done sending does.
private static Socket SendAndClose(int port)
{
Socket client = Connect(port);
client.Send("hi"u8);
client.Shutdown(SocketShutdown.Send);
return client;
}

private static void AssertClosedAndFreed(TcpConnection conn, Socket client, bool incremental)
{
Assert.True(!HasWriteBuffer(conn), "the write slab outlived its handler's release");
Assert.True(KeptSegmentsOf(conn) == 0, $"{KeptSegmentsOf(conn)} overflow segments outlived its handler's release");
if (incremental)
{
Assert.True(!HasRing(conn), "the buffer ring outlived its handler's release");
}

// Closed, the server's side sends the client its FIN, or a reset for what it never read.
client.ReceiveTimeout = ReturnsWithinMs;
string outcome;
try
{
outcome = client.Receive(new byte[16]) == 0 ? "closed" : "data";
}
catch (SocketException e)
{
outcome = e.SocketErrorCode == SocketError.TimedOut ? "still open" : "closed";
}
Assert.True(outcome == "closed", $"the server's socket, whose peer closed first, is {outcome} after its handler let go");
}

private static unsafe bool HasWriteBuffer(TcpConnection connection) => WriteBufferOf(connection) != null;

private static unsafe bool HasRing(TcpConnection connection) => BufRingOf(connection) != null && BufSlabOf(connection) != null;
Expand All @@ -485,6 +641,9 @@ private static async Task<int> ReadRole(TcpConnection conn)
[System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "BufSlab")]
private static extern unsafe ref byte* BufSlabOf(TcpConnection connection);

[System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_ovKept")]
private static extern ref int KeptSegmentsOf(TcpConnection connection);

private static async Task SwitchTo(Reactor reactor) => await new ReactorSwitch(reactor);

private static void AfterTheStop(Func<TcpConnection, Reactor, Task> call, string what)
Expand Down
4 changes: 2 additions & 2 deletions tests/Ioxide.Tests.E2E/Core/TcpTimeoutTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -722,8 +722,8 @@ private static void StalledResponse(bool zeroCopy)

private static bool IsOpen(int fd) => fcntl(fd, FGetFd) != -1;

// Teardown closes every fd without releasing the reactor's ref, so a later release still sees a
// live connection. Its FIN must not reach whichever socket took the number since.
// Teardown closes every fd, and a release after it frees the connection. Neither its FIN nor that
// free's close may reach whichever socket took the number since.
private static void ReleaseAfterStop(Action<TcpConnection> release)
{
var captured = new TaskCompletionSource<TcpConnection>(TaskCreationOptions.RunContinuationsAsynchronously);
Expand Down
Loading