diff --git a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs index b4d801b8..83ebaed7 100644 --- a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs +++ b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs @@ -62,6 +62,7 @@ public static HttpClientPool Start(Reactor reactor, HttpClientOptions options) } reactor.AddService(pool); reactor.AddTicker(pool.Sweep); + reactor.AddStopper(pool.Dispose); return pool; } @@ -181,7 +182,15 @@ private async ValueTask AcquireAsync() _waiters.Enqueue((waiter, deadlineMs)); HttpClientConnection? handed = await waiter.Task; - await ReactorSwitch.Yield(_host); // out of the completer's frame, still on the reactor + try + { + await ReactorSwitch.Yield(_host); // out of the completer's frame, still on the reactor + } + catch (ObjectDisposedException) when (handed is not null) + { + handed.Dispose(); // stopped before this request took it: nothing else holds it, and closing needs no ring + throw; + } if (handed is not null) { return handed; // Release hands over nothing broken @@ -333,7 +342,8 @@ private string AcquireTimeoutMessage() /// Close the idle connections and stop replenishing. Connections currently checked out by an /// in-flight request are not touched - the request owns them, and they are dropped by /// when it finishes, because a disposed pool takes nothing back. Called - /// from another thread, it is queued to the reactor and returns. + /// from another thread, it is queued to the reactor and returns. A pool from + /// is disposed by its reactor's stop too, which ends the requests still waiting on it. /// public void Dispose() { diff --git a/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs b/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs index 03a0fcd9..eb19a00d 100644 --- a/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs +++ b/src/clients/ioxide.httpclient/Http1/RingHttpClient.cs @@ -41,6 +41,7 @@ public static RingHttpClient Start(Reactor reactor, RingHttpClientOptions? optio var client = new RingHttpClient(reactor, options ?? new RingHttpClientOptions()); reactor.AddService(client); reactor.AddTicker(client.Sweep); + reactor.AddStopper(client.Dispose); return client; } @@ -161,7 +162,8 @@ private void Sweep() } /// Close every origin's idle connections; requests in flight finish first, as with the pool. - /// Called from another thread, it is queued to the reactor and returns. + /// Called from another thread, it is queued to the reactor and returns. The reactor's stop disposes + /// the client too. public void Dispose() { // The origins and their pools are the reactor's. diff --git a/src/ioxide/Reactor/Reactor.RingHost.cs b/src/ioxide/Reactor/Reactor.RingHost.cs index a537f70c..c252d7c3 100644 --- a/src/ioxide/Reactor/Reactor.RingHost.cs +++ b/src/ioxide/Reactor/Reactor.RingHost.cs @@ -53,6 +53,16 @@ public T GetService() where T : class internal void OwnTlsService(TlsService service) => (_tlsServices ??= []).Add(service); + // What the clients started on this reactor run when it stops: Teardown runs them first. + private List? _stoppers; + + /// + /// Register a callback run once on the reactor thread when it stops, before its connections close. Call from + /// OnStart. A client closes what it holds there and wakes the requests waiting on it, which a Dispose + /// called after the stop can no longer do: it would be posted to a reactor that runs nothing more. + /// + public void AddStopper(Action stopper) => (_stoppers ??= []).Add(stopper); + /// /// Runs on the reactor thread before the loop starts - open ring-native clients here. /// diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs index 400ef75c..4981d5af 100644 --- a/src/ioxide/Reactor/Reactor.Runner.cs +++ b/src/ioxide/Reactor/Reactor.Runner.cs @@ -129,6 +129,10 @@ private void Teardown(bool startFailed) // A faulted loop never set it: UdpSendTo sends directly from here, as the ring may never be submitted again. _stopRequested = true; + // First: a client stopped here takes no new request from a handler the closes below resume, and what it wakes + // posts its switch in time for the last drain, which ends it on the thread pool. + RunStoppers(); + // Every connection closed to its handler first, then the queue sent while each socket it names is still its // own: left queued, a later submit would send it to whatever took the number (#531). MarkAcceptedTcpConnectionsClosed(); @@ -218,6 +222,22 @@ private void Teardown(bool startFailed) } } + // One that throws must not leave the rest of the teardown undone: the ring, the sockets and the memory. + private void RunStoppers() + { + for (int i = 0; i < (_stoppers?.Count ?? 0); i++) + { + try + { + _stoppers![i](); + } + catch (Exception e) + { + Console.Error.WriteLine($"[r{_id}] stopper faulted: {e.Message}"); + } + } + } + // True once the kernel has taken everything queued, a few tries allowed: a cancel can't find what never reached it. private bool SubmitQueued() { diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs index 008675b3..2898640e 100644 --- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs @@ -353,6 +353,46 @@ public static void Register(Runner runner) Assert.True(switched!.Exception?.InnerException is ObjectDisposedException, $"it ended {switched.Status}: {switched.Exception?.InnerException?.GetType().Name}"); }); + + runner.Test("stop: the stoppers run on the reactor thread before its connections close, and one that throws leaves the teardown to finish", () => + { + var parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bool closed = false; + var ran = new List(); + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + if (!await IsThisTestsConnection(conn)) + { + conn.DecRef(); + return; + } + + try + { + parked.TrySetResult(); + await conn.ReadAsync(); + closed = true; + } + finally + { + conn.DecRef(); + } + }, new ServerConfig { Tcp = new TcpOptions() }, r => + { + r.AddStopper(() => ran.Add($"on the reactor {Reactor.Current == r}, its connection closed {closed}")); + r.AddStopper(() => throw new InvalidOperationException("a stopper that throws")); + r.AddStopper(() => ran.Add("the next one")); + }); + + using Socket client = Connect(port); + client.Send("hi"u8); + Assert.True(parked.Task.Wait(5_000), "the handler never parked"); + + StopAndJoin(reactor, thread); + Assert.Equal("on the reactor True, its connection closed False; the next one", string.Join("; ", ran)); + Assert.True(closed, "the stop never closed the connection to its handler"); + }); } private static readonly byte[] Queued = "QUEUED-FOR-THE-CLOSED-SOCKET"u8.ToArray(); diff --git a/tests/Ioxide.Tests.Http/HttpClientStopTests.cs b/tests/Ioxide.Tests.Http/HttpClientStopTests.cs new file mode 100644 index 00000000..af14d8f3 --- /dev/null +++ b/tests/Ioxide.Tests.Http/HttpClientStopTests.cs @@ -0,0 +1,276 @@ +using System.Net.Sockets; +using System.Runtime.CompilerServices; +using System.Text; +using ioxide; +using ioxide.httpclient; +using ioxide.utils; + +namespace Ioxide.Tests; + +/// +/// The clients as their reactor stops. The stop closes what they hold and wakes the requests waiting on them, which +/// end on the thread pool: a Dispose called after the stop would only be posted, to a reactor that runs nothing more. +/// +internal static class HttpClientStopTests +{ + // Generous against a close or an end that takes no time at all: the assertion is that it comes. + private const int WithinMs = 5_000; + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_tls")] + private static extern ref TlsClientContext? TlsOf(RingHttpClient client); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_disposed")] + private static extern ref bool DisposedOf(TlsClientContext context); + + public static void Register(Runner runner) + { + runner.Test("httpclient h1: a connection handed to a waiting request as its reactor stops is closed, and the requests queued behind it end", () => + { + using var origin = new CountingOrigin(); + HttpClientPool pool = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => pool = HttpClientPool.Start(r, + new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 1, AcquireTimeoutMs = 30_000 })); + + // The first answer hands the one connection to the second request, whose continuation is posted, and the + // stop in that same pass ends the loop before the post runs. + (Task second, Task third) = RunOnReactor(reactor, async () => + { + ValueTask first = pool.GetAsync("/"); + Task second = EndOf(pool.GetAsync("/")); + Task third = EndOf(pool.GetAsync("/")); + (await first).Dispose(); + reactor.Stop(); + return (second, third); + }); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("second: ObjectDisposedException, third: ObjectDisposedException, origin connections closed: 1 of 1", + $"second: {Ended(second)}, third: {Ended(third)}, origin connections closed: {origin.ClosedOf(1)}"); + }); + + runner.Test("httpclient h1: a pool's idle connections are closed by its reactor's stop", () => + { + using var origin = new CountingOrigin(); + HttpClientPool pool = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => pool = HttpClientPool.Start(r, + new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 2 })); + + int idle = RunOnReactor(reactor, async () => + { + ValueTask a = pool.GetAsync("/"), b = pool.GetAsync("/"); + (await a).Dispose(); + (await b).Dispose(); + return pool.IdleCount; + }); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("idle 2, origin connections closed: 2 of 2", $"idle {idle}, origin connections closed: {origin.ClosedOf(2)}"); + }); + + runner.Test("httpclient h1: a request a handler makes as its reactor stops is refused, and never reaches the origin", () => + { + // The stop resumes a handler by closing its connection, before the ring's last submit: a request the pool + // still took then would be sent by that submit, and its caller told it failed. + using var origin = new CountingOrigin(); + var parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task? asked = null; + HttpClientPool pool = null!; + (int port, Reactor reactor, Thread thread) = StartReactor( + r => pool = HttpClientPool.Start(r, new HttpClientOptions { Port = (ushort)origin.Port, PoolSize = 1 }), + async (_, connection) => + { + try + { + if (await SentSomething(connection)) + { + parked.TrySetResult(); + await connection.ReadAsync(); // until the stop closes it + asked = FailureOf(pool); + } + } + finally + { + connection.DecRef(); + } + }); + + RunOnReactor(reactor, async () => + { + (await pool.GetAsync("/")).Dispose(); // the pool's one connection, idle + return 0; + }); + using var client = new TcpClient("127.0.0.1", port); + client.GetStream().Write("hi"u8); + Assert.True(parked.Task.Wait(WithinMs), "the handler never parked"); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + string closed = origin.ClosedOf(1); + Assert.Equal($"pool for 127.0.0.1:{origin.Port} is disposed; origin requests 1, connections closed: 1 of 1", + $"{(asked is null ? "never asked" : Ended(asked))}; origin requests {origin.Requests}, connections closed: {closed}"); + }); + + runner.Test("ring client: its reactor's stop closes its origins' idle connections and frees its TLS context", () => + { + using var origin = new CountingOrigin(); + using TlsTestOrigin secure = TlsTestOrigin.Start("http/1.1"); + (string certPath, _) = TestCert.Ensure(); + RingHttpClient http = null!; + (_, Reactor reactor, Thread thread) = StartReactor(r => http = RingHttpClient.Start(r, new RingHttpClientOptions + { + Tls = new TlsClientOptions { CaFile = certPath, AlpnProtocols = ["http/1.1"] }, + })); + + string answered = RunOnReactor(reactor, async () => + { + using HttpClientResponse plain = await http.GetAsync($"http://127.0.0.1:{origin.Port}/"); + using HttpClientResponse tls = await http.GetAsync($"https://localhost:{secure.Port}/"); + return $"{plain.Status} {tls.Status}, origins {http.OriginCount}"; + }, timeoutMs: 20_000); + TlsClientContext context = TlsOf(http)!; + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Assert.Equal("200 200, origins 2; origin connections closed: 1 of 1, TLS context freed: True", + $"{answered}; origin connections closed: {origin.ClosedOf(1)}, TLS context freed: {DisposedOf(context)}"); + }); + } + + private static (int Port, Reactor Reactor, Thread Thread) StartReactor(Action onStart, + Func? handle = null) + => TestServer.StartConfigured( + handle ?? ((_, connection) => { connection.DecRef(); return Task.CompletedTask; }), + new ServerConfig + { + RecvBufferSize = 4096, + RecvSlots = 64, + Tcp = new TcpOptions { WriteSlabSize = 16 * 1024, PoolMax = 8, RecvQueueEntries = 64 }, + }, + onStart); + + // From the reactor's post queue rather than OnStart, so a body that stops the reactor cannot do it before the + // harness has seen it listen. + private static T RunOnReactor(Reactor reactor, Func> body, int timeoutMs = 10_000) + { + var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + reactor.ScheduleOnReactor(_ => _ = RunAsync(), null); + Assert.True(outcome.Task.Wait(timeoutMs), $"the code on the reactor did not finish within {timeoutMs} ms"); + return outcome.Task.Result; + + async Task RunAsync() + { + try + { + outcome.TrySetResult(await body()); + } + catch (Exception e) + { + outcome.TrySetException(e); + } + } + } + + // How a request ended: the status it was answered with, or the type of what it threw. + private static async Task EndOf(ValueTask request) + { + try + { + using HttpClientResponse response = await request; + return $"answered {response.Status}"; + } + catch (Exception e) + { + return e.GetType().Name; + } + } + + private static string Ended(Task request) => request.Wait(WithinMs) ? request.Result : "never ended"; + + // Why a request made now fails, or the status it was answered with. + private static async Task FailureOf(HttpClientPool pool) + { + try + { + using HttpClientResponse response = await pool.GetAsync("/"); + return $"answered {response.Status}"; + } + catch (Exception e) + { + return e.Message; + } + } + + // The harness's listen probe connects and closes without sending anything. + private static async Task SentSomething(TcpConnection connection) + { + while (true) + { + RecvSnapshot snapshot = await connection.ReadAsync(); + + bool received = false; + while (connection.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) + { + received = true; + connection.ReturnBuffer(in item); + } + } + + if (received || snapshot.IsClosed) + { + return received; + } + } + } + + // Answers every request with 200, and counts them and the connections its client closed. + private sealed class CountingOrigin : IDisposable + { + private readonly FakeServer _server; + private int _requests; + private int _closed; + + public CountingOrigin() => _server = new FakeServer((_, socket) => + { + try + { + var buffer = new byte[4096]; + var received = new StringBuilder(); + int n; + while ((n = socket.Receive(buffer)) > 0) + { + received.Append(Encoding.Latin1.GetString(buffer, 0, n)); + for (int end; (end = received.ToString().IndexOf("\r\n\r\n", StringComparison.Ordinal)) >= 0;) + { + received.Remove(0, end + 4); + Interlocked.Increment(ref _requests); + socket.Send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok"u8); + } + } + } + finally + { + Interlocked.Increment(ref _closed); + } + }); + + public int Port => _server.Port; + + public int Requests => Volatile.Read(ref _requests); + + // How many of `expected` connections it saw closed, given until WithinMs to see them all. + public string ClosedOf(int expected) + { + long deadline = Environment.TickCount64 + WithinMs; + while (Volatile.Read(ref _closed) < expected && Environment.TickCount64 < deadline) + { + Thread.Sleep(20); + } + return $"{Volatile.Read(ref _closed)} of {expected}"; + } + + public void Dispose() => _server.Dispose(); + } +} diff --git a/tests/Ioxide.Tests.Http/Program.cs b/tests/Ioxide.Tests.Http/Program.cs index a44780b0..75936e03 100644 --- a/tests/Ioxide.Tests.Http/Program.cs +++ b/tests/Ioxide.Tests.Http/Program.cs @@ -20,6 +20,7 @@ private static int Main() HttpClientRequestHeadTests.Register(runner); HttpClientConnectTests.Register(runner); RingHttpClientTests.Register(runner); + HttpClientStopTests.Register(runner); HttpClientChunkedTests.Register(runner); HttpClientResponseLimitTests.Register(runner); HttpClientHttp10Tests.Register(runner);