From d86c131dc40d697911e209e00a66f22e450cced9 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 19:48:28 +0000 Subject: [PATCH] httpclient, reactor: a reactor's stop disposes the pools and clients started on it, so the requests waiting on them end, and a connection handed over as it stops is closed (#610) Release hands a connection straight to the first waiting request, which yields to the reactor's post queue before it takes it. A stop in that same pass ended the loop before the yield ran, and the teardown's last drain resumed it on the thread pool, where the yield threw ObjectDisposedException. The connection, in neither the idle list nor the waiter queue and never returned to SendAsync, was never closed, and the requests queued behind it waited forever: only a Release, a Discard, a failed open, Dispose or the sweep wakes them, and the sweep is a ticker a stopped reactor no longer runs. The stop also left every pool's idle connections open and RingHttpClient's TLS context allocated, since only their Dispose frees them, and a Dispose called after the stop is a post the stopped reactor drops. - reactor: AddStopper registers a callback that the teardown runs first, on the reactor thread, before the connections close and before posts close. Each runs guarded, so one that throws is logged and the rest of the teardown still runs. - httpclient: HttpClientPool.Start and RingHttpClient.Start register their Dispose, so the stop closes the idle connections, frees the TLS context and wakes the waiting requests, which end with ObjectDisposedException on the thread pool. Run first, a pool also refuses a request from a handler the stop's closes resume, which the last submit would otherwise send to the origin while its caller was told it failed. - httpclient: a request whose yield throws disposes the connection it was handed. Tests: Http's HttpClientStopTests (4) and E2E's StopParkedTests stopper test, each failing when its part of the fix is reverted. --- .../ioxide.httpclient/Http1/HttpClientPool.cs | 14 +- .../ioxide.httpclient/Http1/RingHttpClient.cs | 4 +- src/ioxide/Reactor/Reactor.RingHost.cs | 10 + src/ioxide/Reactor/Reactor.Runner.cs | 20 ++ .../Ioxide.Tests.E2E/Core/StopParkedTests.cs | 40 +++ .../Ioxide.Tests.Http/HttpClientStopTests.cs | 276 ++++++++++++++++++ tests/Ioxide.Tests.Http/Program.cs | 1 + 7 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 tests/Ioxide.Tests.Http/HttpClientStopTests.cs diff --git a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs index b4d801b8..83ebaed7 100644 --- a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs +++ b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs @@ -62,6 +62,7 @@ public static HttpClientPool Start(Reactor reactor, HttpClientOptions options) } reactor.AddService(pool); reactor.AddTicker(pool.Sweep); + reactor.AddStopper(pool.Dispose); return pool; } @@ -181,7 +182,15 @@ private async ValueTask AcquireAsync() _waiters.Enqueue((waiter, deadlineMs)); HttpClientConnection? handed = await waiter.Task; - await ReactorSwitch.Yield(_host); // out of the completer's frame, still on the reactor + try + { + await ReactorSwitch.Yield(_host); // out of the completer's frame, still on the reactor + } + catch (ObjectDisposedException) when (handed is not null) + { + handed.Dispose(); // stopped before this request took it: nothing else holds it, and closing needs no ring + throw; + } if (handed is not null) { return handed; // Release hands over nothing broken @@ -333,7 +342,8 @@ private string AcquireTimeoutMessage() /// Close the idle connections and stop replenishing. Connections currently checked out by an /// in-flight request are not touched - the request owns them, and they are dropped by /// when it finishes, because a disposed pool takes nothing back. Called - /// from another thread, it is queued to the reactor and returns. + /// from another thread, it is queued to the reactor and returns. A pool from + /// is disposed by its reactor's stop too, which ends the requests still waiting on it. /// public void Dispose() { diff --git a/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs b/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs index 03a0fcd9..eb19a00d 100644 --- a/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs +++ b/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs @@ -41,6 +41,7 @@ public static RingHttpClient Start(Reactor reactor, RingHttpClientOptions? optio var client = new RingHttpClient(reactor, options ?? new RingHttpClientOptions()); reactor.AddService(client); reactor.AddTicker(client.Sweep); + reactor.AddStopper(client.Dispose); return client; } @@ -161,7 +162,8 @@ private void Sweep() } /// Close every origin's idle connections; requests in flight finish first, as with the pool. - /// Called from another thread, it is queued to the reactor and returns. + /// Called from another thread, it is queued to the reactor and returns. The reactor's stop disposes + /// the client too. public void Dispose() { // The origins and their pools are the reactor's. diff --git a/src/ioxide/Reactor/Reactor.RingHost.cs b/src/ioxide/Reactor/Reactor.RingHost.cs index a537f70c..c252d7c3 100644 --- a/src/ioxide/Reactor/Reactor.RingHost.cs +++ b/src/ioxide/Reactor/Reactor.RingHost.cs @@ -53,6 +53,16 @@ public T GetService() where T : class internal void OwnTlsService(TlsService service) => (_tlsServices ??= []).Add(service); + // What the clients started on this reactor run when it stops: Teardown runs them first. + private List? _stoppers; + + /// + /// Register a callback run once on the reactor thread when it stops, before its connections close. Call from + /// OnStart. A client closes what it holds there and wakes the requests waiting on it, which a Dispose + /// called after the stop can no longer do: it would be posted to a reactor that runs nothing more. + /// + public void AddStopper(Action stopper) => (_stoppers ??= []).Add(stopper); + /// /// Runs on the reactor thread before the loop starts - open ring-native clients here. /// diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs index 400ef75c..4981d5af 100644 --- a/src/ioxide/Reactor/Reactor.Runner.cs +++ b/src/ioxide/Reactor/Reactor.Runner.cs @@ -129,6 +129,10 @@ private void Teardown(bool startFailed) // A faulted loop never set it: UdpSendTo sends directly from here, as the ring may never be submitted again. _stopRequested = true; + // First: a client stopped here takes no new request from a handler the closes below resume, and what it wakes + // posts its switch in time for the last drain, which ends it on the thread pool. + RunStoppers(); + // Every connection closed to its handler first, then the queue sent while each socket it names is still its // own: left queued, a later submit would send it to whatever took the number (#531). MarkAcceptedTcpConnectionsClosed(); @@ -218,6 +222,22 @@ private void Teardown(bool startFailed) } } + // One that throws must not leave the rest of the teardown undone: the ring, the sockets and the memory. + private void RunStoppers() + { + for (int i = 0; i < (_stoppers?.Count ?? 0); i++) + { + try + { + _stoppers![i](); + } + catch (Exception e) + { + Console.Error.WriteLine($"[r{_id}] stopper faulted: {e.Message}"); + } + } + } + // True once the kernel has taken everything queued, a few tries allowed: a cancel can't find what never reached it. private bool SubmitQueued() { diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs index 008675b3..2898640e 100644 --- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs @@ -353,6 +353,46 @@ public static void Register(Runner runner) Assert.True(switched!.Exception?.InnerException is ObjectDisposedException, $"it ended {switched.Status}: {switched.Exception?.InnerException?.GetType().Name}"); }); + + runner.Test("stop: the stoppers run on the reactor thread before its connections close, and one that throws leaves the teardown to finish", () => + { + var parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bool closed = false; + var ran = new List(); + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + if (!await IsThisTestsConnection(conn)) + { + conn.DecRef(); + return; + } + + try + { + parked.TrySetResult(); + await conn.ReadAsync(); + closed = true; + } + finally + { + conn.DecRef(); + } + }, new ServerConfig { Tcp = new TcpOptions() }, r => + { + r.AddStopper(() => ran.Add($"on the reactor {Reactor.Current == r}, its connection closed {closed}")); + r.AddStopper(() => throw new InvalidOperationException("a stopper that throws")); + r.AddStopper(() => ran.Add("the next one")); + }); + + using Socket client = Connect(port); + client.Send("hi"u8); + Assert.True(parked.Task.Wait(5_000), "the handler never parked"); + + StopAndJoin(reactor, thread); + Assert.Equal("on the reactor True, its connection closed False; the next one", string.Join("; ", ran)); + Assert.True(closed, "the stop never closed the connection to its handler"); + }); } private static readonly byte[] Queued = "QUEUED-FOR-THE-CLOSED-SOCKET"u8.ToArray(); diff --git a/tests/Ioxide.Tests.Http/HttpClientStopTests.cs b/tests/Ioxide.Tests.Http/HttpClientStopTests.cs new file mode 100644 index 00000000..af14d8f3 --- /dev/null +++ b/tests/Ioxide.Tests.Http/HttpClientStopTests.cs @@ -0,0 +1,276 @@ +using System.Net.Sockets; +using System.Runtime.CompilerServices; +using System.Text; +using ioxide; +using ioxide.httpclient; +using ioxide.utils; + +namespace Ioxide.Tests; + +/// +/// The clients as their reactor stops. The stop closes what they hold and wakes the requests waiting on them, which +/// end on the thread pool: a Dispose called after the stop would only be posted, to a reactor that runs nothing more. +/// +internal static class HttpClientStopTests +{ + // Generous against a close or an end that takes no time at all: the assertion is that it comes. + private const int WithinMs = 5_000; + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_tls")] + private static extern ref TlsClientContext? TlsOf(RingHttpClient client); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_disposed")] + private static extern ref bool DisposedOf(TlsClientContext context); + + public static void Register(Runner runner) + { + runner.Test("httpclient h1: a connection handed to a waiting request as its reactor stops is closed, and the requests queued behind it end", () => + { + using var origin = new CountingOrigin(); + HttpClientPool pool = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => pool = HttpClientPool.Start(r, + new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 1, AcquireTimeoutMs = 30_000 })); + + // The first answer hands the one connection to the second request, whose continuation is posted, and the + // stop in that same pass ends the loop before the post runs. + (Task second, Task third) = RunOnReactor(reactor, async () => + { + ValueTask first = pool.GetAsync("/"); + Task second = EndOf(pool.GetAsync("/")); + Task third = EndOf(pool.GetAsync("/")); + (await first).Dispose(); + reactor.Stop(); + return (second, third); + }); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("second: ObjectDisposedException, third: ObjectDisposedException, origin connections closed: 1 of 1", + $"second: {Ended(second)}, third: {Ended(third)}, origin connections closed: {origin.ClosedOf(1)}"); + }); + + runner.Test("httpclient h1: a pool's idle connections are closed by its reactor's stop", () => + { + using var origin = new CountingOrigin(); + HttpClientPool pool = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => pool = HttpClientPool.Start(r, + new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 2 })); + + int idle = RunOnReactor(reactor, async () => + { + ValueTask a = pool.GetAsync("/"), b = pool.GetAsync("/"); + (await a).Dispose(); + (await b).Dispose(); + return pool.IdleCount; + }); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("idle 2, origin connections closed: 2 of 2", $"idle {idle}, origin connections closed: {origin.ClosedOf(2)}"); + }); + + runner.Test("httpclient h1: a request a handler makes as its reactor stops is refused, and never reaches the origin", () => + { + // The stop resumes a handler by closing its connection, before the ring's last submit: a request the pool + // still took then would be sent by that submit, and its caller told it failed. + using var origin = new CountingOrigin(); + var parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task? asked = null; + HttpClientPool pool = null!; + (int port, Reactor reactor, Thread thread) = StartReactor( + r => pool = HttpClientPool.Start(r, new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 1 }), + async (_, connection) => + { + try + { + if (await SentSomething(connection)) + { + parked.TrySetResult(); + await connection.ReadAsync(); // until the stop closes it + asked = FailureOf(pool); + } + } + finally + { + connection.DecRef(); + } + }); + + RunOnReactor(reactor, async () => + { + (await pool.GetAsync("/")).Dispose(); // the pool's one connection, idle + return 0; + }); + using var client = new TcpClient("127.0.0.1", port); + client.GetStream().Write("hi"u8); + Assert.True(parked.Task.Wait(WithinMs), "the handler never parked"); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + string closed = origin.ClosedOf(1); + Assert.Equal($"pool for 127.0.0.1:{origin.Port} is disposed; origin requests 1, connections closed: 1 of 1", + $"{(asked is null ? "never asked" : Ended(asked))}; origin requests {origin.Requests}, connections closed: {closed}"); + }); + + runner.Test("ring client: its reactor's stop closes its origins' idle connections and frees its TLS context", () => + { + using var origin = new CountingOrigin(); + using TlsTestOrigin secure = TlsTestOrigin.Start("http/1.1"); + (string certPath, _) = TestCert.Ensure(); + RingHttpClient http = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => http = RingHttpClient.Start(r, new RingHttpClientOptions + { + Tls = new TlsClientOptions { CaFile = certPath, AlpnProtocols = ["http/1.1"] }, + })); + + string answered = RunOnReactor(reactor, async () => + { + using HttpClientResponse plain = await http.GetAsync($"http://127.0.0.1:{origin.Port}/"); + using HttpClientResponse tls = await http.GetAsync($"https://localhost:{secure.Port}/"); + return $"{plain.Status} {tls.Status}, origins {http.OriginCount}"; + }, timeoutMs: 20_000); + TlsClientContext context = TlsOf(http)!; + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("200 200, origins 2; origin connections closed: 1 of 1, TLS context freed: True", + $"{answered}; origin connections closed: {origin.ClosedOf(1)}, TLS context freed: {DisposedOf(context)}"); + }); + } + + private static (int Port, Reactor Reactor, Thread Thread) StartReactor(Action onStart, + Func? handle = null) + => TestServer.StartConfigured( + handle ?? ((_, connection) => { connection.DecRef(); return Task.CompletedTask; }), + new ServerConfig + { + RecvBufferSize = 4096, + RecvSlots = 64, + Tcp = new TcpOptions { WriteSlabSize = 16 * 1024, PoolMax = 8, RecvQueueEntries = 64 }, + }, + onStart); + + // From the reactor's post queue rather than OnStart, so a body that stops the reactor cannot do it before the + // harness has seen it listen. + private static T RunOnReactor(Reactor reactor, Func> body, int timeoutMs = 10_000) + { + var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + reactor.ScheduleOnReactor(_ => _ = RunAsync(), null); + Assert.True(outcome.Task.Wait(timeoutMs), $"the code on the reactor did not finish within {timeoutMs} ms"); + return outcome.Task.Result; + + async Task RunAsync() + { + try + { + outcome.TrySetResult(await body()); + } + catch (Exception e) + { + outcome.TrySetException(e); + } + } + } + + // How a request ended: the status it was answered with, or the type of what it threw. + private static async Task EndOf(ValueTask request) + { + try + { + using HttpClientResponse response = await request; + return $"answered {response.Status}"; + } + catch (Exception e) + { + return e.GetType().Name; + } + } + + private static string Ended(Task request) => request.Wait(WithinMs) ? request.Result : "never ended"; + + // Why a request made now fails, or the status it was answered with. + private static async Task FailureOf(HttpClientPool pool) + { + try + { + using HttpClientResponse response = await pool.GetAsync("/"); + return $"answered {response.Status}"; + } + catch (Exception e) + { + return e.Message; + } + } + + // The harness's listen probe connects and closes without sending anything. + private static async Task SentSomething(TcpConnection connection) + { + while (true) + { + RecvSnapshot snapshot = await connection.ReadAsync(); + + bool received = false; + while (connection.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) + { + received = true; + connection.ReturnBuffer(in item); + } + } + + if (received || snapshot.IsClosed) + { + return received; + } + } + } + + // Answers every request with 200, and counts them and the connections its client closed. + private sealed class CountingOrigin : IDisposable + { + private readonly FakeServer _server; + private int _requests; + private int _closed; + + public CountingOrigin() => _server = new FakeServer((_, socket) => + { + try + { + var buffer = new byte[4096]; + var received = new StringBuilder(); + int n; + while ((n = socket.Receive(buffer)) > 0) + { + received.Append(Encoding.Latin1.GetString(buffer, 0, n)); + for (int end; (end = received.ToString().IndexOf("\r\n\r\n", StringComparison.Ordinal)) >= 0;) + { + received.Remove(0, end + 4); + Interlocked.Increment(ref _requests); + socket.Send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok"u8); + } + } + } + finally + { + Interlocked.Increment(ref _closed); + } + }); + + public int Port => _server.Port; + + public int Requests => Volatile.Read(ref _requests); + + // How many of `expected` connections it saw closed, given until WithinMs to see them all. + public string ClosedOf(int expected) + { + long deadline = Environment.TickCount64 + WithinMs; + while (Volatile.Read(ref _closed) < expected && Environment.TickCount64 < deadline) + { + Thread.Sleep(20); + } + return $"{Volatile.Read(ref _closed)} of {expected}"; + } + + public void Dispose() => _server.Dispose(); + } +} diff --git a/tests/Ioxide.Tests.Http/Program.cs b/tests/Ioxide.Tests.Http/Program.cs index a44780b0..75936e03 100644 --- a/tests/Ioxide.Tests.Http/Program.cs +++ b/tests/Ioxide.Tests.Http/Program.cs @@ -20,6 +20,7 @@ private static int Main() HttpClientRequestHeadTests.Register(runner); HttpClientConnectTests.Register(runner); RingHttpClientTests.Register(runner); + HttpClientStopTests.Register(runner); HttpClientChunkedTests.Register(runner); HttpClientResponseLimitTests.Register(runner); HttpClientHttp10Tests.Register(runner);