From 338c6a5a719095c306bd7398e69c99b1d128f017 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 24 Oct 2026 22:39:53 +0100 Subject: [PATCH 01/18] Update ioxide to 0.20.396 - Rename IdleTimeoutMs to ReadTimeoutMs on the TCP and QUIC options, and document what the read timeout and the send timeout bound now (#929, #917) - Drop the raw shutdown(2) calls: DecRef sends the FIN itself, after any send in flight (#929) - Take the FlushResult the HTTP/2 and HTTP/3 writers return, and end a streamed body with ConnectionClosedException once the peer is gone, so a server-sent event source notices the client left (#932) - Rethrow a fault that happens after the headers are out, so ioxide resets the stream instead of ending a truncated body cleanly (#930) - TLS on the TCP transports is terminated in BoringSSL now; update the docs and samples --- API/Ioxide/GenHTTP.Api.Ioxide.csproj | 2 +- Engine/Ioxide/EngineOptions.cs | 37 ++- Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj | 8 +- Engine/Ioxide/Infrastructure/Server.Quic.cs | 2 +- .../Ioxide/Infrastructure/Server.Tcp.Tls.cs | 13 +- Engine/Ioxide/Infrastructure/Server.Tcp.cs | 2 +- .../Protocol/Drivers/Quic/Http3Driver.cs | 25 +- .../Protocol/Drivers/Tcp/Http2Driver.cs | 23 +- .../Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs | 14 +- .../Protocol/Responses/StreamedResponder.cs | 3 +- Engine/Ioxide/Protocol/Sinks/StreamedSink.cs | 26 +- Modules/Files/GenHTTP.Modules.Files.csproj | 2 +- Playground/Samples/Ioxide/KernelTlsSample.cs | 2 +- Playground/Samples/Ioxide/README.md | 2 +- Playground/Samples/Ioxide/ShowcaseSample.cs | 2 +- Playground/Samples/Ioxide/TlsSample.cs | 6 +- .../Engine/Ioxide/IdleTimeoutTests.cs | 117 +++++++++ .../Engine/Ioxide/StreamedResponseEndTests.cs | 229 ++++++++++++++++++ .../ClientCertificateAuthenticationTests.cs | 2 +- 19 files changed, 459 insertions(+), 58 deletions(-) create mode 100644 Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs create mode 100644 Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs diff --git a/API/Ioxide/GenHTTP.Api.Ioxide.csproj b/API/Ioxide/GenHTTP.Api.Ioxide.csproj index 19393b847..7d0170c27 100644 --- a/API/Ioxide/GenHTTP.Api.Ioxide.csproj +++ b/API/Ioxide/GenHTTP.Api.Ioxide.csproj @@ -9,7 +9,7 @@ - + diff --git a/Engine/Ioxide/EngineOptions.cs b/Engine/Ioxide/EngineOptions.cs index b9bc4ca61..794e8c470 100644 --- a/Engine/Ioxide/EngineOptions.cs +++ b/Engine/Ioxide/EngineOptions.cs @@ -56,33 +56,42 @@ public sealed record TcpTransportOptions public int RecvQueueEntries { get; init; } = 64; /// - /// Close a connection that has neither received nor sent anything for this long. 0 (the - /// default) disables it. + /// Close a connection that has waited this long on its client: a read parked, nothing + /// buffered and nothing being sent. 0 (the default) disables it. /// /// + /// The clock only runs while the server waits on the client. Bytes in either direction + /// restart it, a send in flight holds it, and the time a handler spends producing a response + /// does not count - so a slow handler keeps its connection, while an idle keep-alive + /// connection or a silent websocket does not. HTTP/2 is the exception: its frame loop keeps a + /// read parked while handlers work, so a response slower than this, to a client that sends + /// nothing in the meantime, is cut. + /// /// Off by default because this listener also carries upgraded connections, and an idle /// websocket is a healthy one. ioxide's own default is 60s, which is right for HTTP keep-alive /// and wrong for a websocket whose ping interval is longer than that - the transport cannot /// tell the two apart, so the framework does not guess on your behalf. /// /// Turn it on if you serve no upgrades, or set it comfortably above your websocket ping - /// interval so pings keep the connection marked alive. + /// interval - a ping from either side restarts the clock. /// public int IdleTimeoutMs { get; init; } /// - /// Close a connection whose response has been outstanding for this long. 0 (the default) - /// disables it. + /// Close a connection whose send has been in flight for this long. 0 (the default) disables it. /// /// /// This is what bounds a client that stops reading: its window shuts, the send never completes, /// and without a clock the connection is pinned for as long as the peer cares to hold it. TCP - /// will not end it on its own. + /// will not end it on its own, and neither will : a client that + /// keeps sending restarts that clock while its own window stays shut. A websocket written from + /// a background task is the case that needs this one. /// - /// Off by default because it is a deadline for the WHOLE response, not for progress within it, - /// so a large download to a slow client counts against it. Size it against your slowest - /// legitimate response rather than against a stall - for an API that serves small payloads, - /// something like 30s is a straightforward win. + /// Off by default because it is a deadline for a whole flush, not for progress within it, and + /// content that does not flush as it writes goes out as one flush - so a large download to a + /// slow client counts against it. Size it against your slowest legitimate response rather than + /// against a stall - for an API that serves small payloads, something like 30s is a + /// straightforward win. /// public int SendTimeoutMs { get; init; } } @@ -92,6 +101,14 @@ public sealed record QuicTransportOptions { public int HandshakeTimeoutMs { get; init; } = 10_000; + /// + /// Close a connection whose client has sent nothing for this long. 0 disables it. + /// + /// + /// While a response is still owed, the engine keeps the client answering with keep-alive + /// PINGs, so a slow handler is not timed out - only a client that went quiet with nothing + /// pending is. + /// public int IdleTimeoutMs { get; init; } = 60_000; public QuicRouting Routing { get; init; } = QuicRouting.Forward; diff --git a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj index df36c082a..4762c690b 100644 --- a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj +++ b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj @@ -16,11 +16,11 @@ - + - - - + + + diff --git a/Engine/Ioxide/Infrastructure/Server.Quic.cs b/Engine/Ioxide/Infrastructure/Server.Quic.cs index 072a78968..dd71d11d9 100644 --- a/Engine/Ioxide/Infrastructure/Server.Quic.cs +++ b/Engine/Ioxide/Infrastructure/Server.Quic.cs @@ -85,7 +85,7 @@ private ServerConfig WithQuic(ServerConfig serverConfig) Port = quicEndPoint.Port, ConnectionFactory = _quicEngine.CreateFactory(), - IdleTimeoutMs = _engineOptions.Quic.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Quic.IdleTimeoutMs, Routing = _engineOptions.Quic.Routing, PinMigratedPeers = _engineOptions.Quic.PinMigratedPeers, }, diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs index 1eacb5021..f20b53652 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs @@ -11,7 +11,7 @@ namespace GenHTTP.Engine.Ioxide.Infrastructure; -/// TLS termination on the TCP transports, in OpenSSL. +/// TLS termination on the TCP transports, in BoringSSL. public sealed partial class Server { // The TLS options for every secure TCP port that can produce a certificate; the rest refuse their handshakes. @@ -119,7 +119,7 @@ private void StartTlsServices(Reactor reactor, int index, IReadOnlyList 0 ? byHost : null; } - // The TLS floor for one endpoint, widening rather than throwing where SslProtocols asks for something OpenSSL cannot express. + // The TLS floor for one endpoint, widening rather than throwing where SslProtocols asks for something BoringSSL cannot express. private TlsProtocolVersion ResolveMinProtocolVersion(SecureEndPoint endPoint) { var protocols = endPoint.SecurityConfiguration.Protocols; @@ -130,7 +130,7 @@ private TlsProtocolVersion ResolveMinProtocolVersion(SecureEndPoint endPoint) if ((protocols & ~(SslProtocols.Tls12 | SslProtocols.Tls13)) != 0) { _logger.LogWarning( - "Port {Port} was bound asking for a TLS version below 1.2. This engine terminates TLS in OpenSSL, whose floor is 1.2, " + "Port {Port} was bound asking for a TLS version below 1.2. This engine terminates TLS in BoringSSL, whose floor is 1.2, " + "so those versions are not served and never were; the endpoint serves {Served}.", endPoint.Port, tls13 && !tls12 ? "TLS 1.3" : "TLS 1.2 and above"); } @@ -164,7 +164,7 @@ private void WarnAboutValidatorGaps() if (validator.RevocationCheck != X509RevocationMode.NoCheck) { _logger.LogWarning( - "The validator on port {Port} asks for {Mode} revocation checking, which this engine does not perform - neither OpenSSL " + "The validator on port {Port} asks for {Mode} revocation checking, which this engine does not perform - neither BoringSSL " + "nor ngtcp2 is given a CRL or an OCSP responder here, so a revoked client certificate is accepted until it expires. " + "Use short-lived certificates, or check the peer in ICertificateValidator.Validate against your own source of truth.", endPoint.Port, validator.RevocationCheck); @@ -173,9 +173,8 @@ private void WarnAboutValidatorGaps() if (endPoint.Protocols.HasFlag(HttpProtocols.Http3)) { _logger.LogWarning( - "Port {Port} serves HTTP/3 with a certificate validator. Validate runs on the TCP transports, where OpenSSL hands over the " - + "peer certificate, but NOT over QUIC: ngtcp2 exposes only the peer's subject and common name, so there is no certificate " - + "to hand it. An HTTP/3 client is admitted on the chain check and RequireCertificate alone. Keep the two transports on " + "Port {Port} serves HTTP/3 with a certificate validator. Validate runs on the TCP transports, but NOT over QUIC: " + + "an HTTP/3 client is admitted on the chain check and RequireCertificate alone. Keep the two transports on " + "separate ports if that difference matters.", endPoint.Port); } diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.cs index 5666c37cf..4ca71d12d 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.cs @@ -32,7 +32,7 @@ private ServerConfig WithTcp(ServerConfig serverConfig) => serverConfig with // Set explicitly, because ioxide defaults both to 60s and this listener carries // upgraded connections too - inheriting that would start closing idle websockets. - IdleTimeoutMs = _engineOptions.Tcp.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Tcp.IdleTimeoutMs, SendTimeoutMs = _engineOptions.Tcp.SendTimeoutMs, }, }; diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index 39b248e79..b79217907 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -20,7 +20,8 @@ internal static Task RunAsync(IServer server, IEndPoint endPoint, QuicConnection => new Nghttp3Connection(connection, options) .RunStreamedResponseAsync((request, writer) => DispatchAsync(server, endPoint, request, writer)); - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream. + // One stream through the handler chain; a fault becomes a 500 rather than a dead stream, or a + // reset once the headers are out. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Nghttp3Request request, Nghttp3ResponseWriter writer) { @@ -68,16 +69,32 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, headersWritten = true; await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + + await writer.CompleteAsync(); + } + catch (ConnectionClosedException) + { + // The client stopped the stream or went away mid-response: nobody is left to answer, + // and that is no fault. } catch (Exception e) { server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Quic.Http3Driver") .LogError(e, "Failed to handle HTTP/3 request"); - if (!headersWritten && !writer.IsCompleted) + if (headersWritten) + { + // Too late for a 500, and a FIN would pass the truncated body off as whole. + // Rethrown, ioxide resets the stream (H3_INTERNAL_ERROR), so the client sees it fail. + throw; + } + + if (!writer.IsCompleted) { writer.WriteHeaders(new Nghttp3Response { Status = 500 }); } + + await writer.CompleteAsync(); } finally { @@ -85,9 +102,7 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, { StreamedRequest.Return(mapped); } - - await writer.CompleteAsync(); } } - + } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index a7c09f4dd..74cf23e83 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -24,7 +24,8 @@ internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pi => new Http2Connection(pipe, Options) .RunAsync((request, writer) => DispatchAsync(server, endPoint, request, writer, remoteAddress, secure)); - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream. + // One stream through the handler chain; a fault becomes a 500 rather than a dead stream, or a + // reset once the headers are out. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Http2Request request, Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) { @@ -70,16 +71,32 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, headersWritten = true; await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + + await writer.CompleteAsync(); + } + catch (ConnectionClosedException) + { + // The client reset the stream or went away mid-response: nobody is left to answer, + // and that is no fault. } catch (Exception e) { server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Tcp.Http2Driver") .LogError(e, "Failed to handle HTTP/2 request"); - if (!headersWritten && !writer.IsCompleted) + if (headersWritten) + { + // Too late for a 500, and END_STREAM would pass the truncated body off as whole. + // Rethrown, ioxide resets the stream (INTERNAL_ERROR), so the client sees it fail. + throw; + } + + if (!writer.IsCompleted) { writer.WriteHeaders(new Http2Response { Status = 500 }); } + + await writer.CompleteAsync(); } finally { @@ -87,8 +104,6 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, { StreamedRequest.Return(mapped); } - - await writer.CompleteAsync(); } } } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs index 1d2ffd5e5..6c49702b8 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs @@ -23,12 +23,6 @@ namespace GenHTTP.Engine.Ioxide.Protocol.Drivers.Tcp; /// Takes an accepted TCP connection through TLS and up to whichever protocol it turns out to speak. internal static partial class TcpDriver { - private const int ShutWrite = 1; - - // Half-closes the socket, which is what puts a FIN on the wire. - [LibraryImport("libc", EntryPoint = "shutdown")] - private static partial int Shutdown(int sockfd, int how); - // The peer's sockaddr, which ioxide does not surface itself. [LibraryImport("libc", EntryPoint = "getpeername")] private static partial int GetPeerName(int sockfd, [Out] byte[] addr, ref int addrlen); @@ -49,7 +43,6 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon { if (!IoxideReactor.Current.GetService().TryFor(conn.ListenerPort, out var service)) { - _ = Shutdown(conn.ClientFd, ShutWrite); conn.DecRef(); return; } @@ -118,7 +111,7 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon return (new TlsConnectionDualPipe(conn, session), session.NegotiatedAlpn); } - // Asks the endpoint's validator about the peer certificate OpenSSL ended up with. + // Asks the endpoint's validator about the peer certificate BoringSSL ended up with. private static bool Accepts(ICertificateValidator validator, IoxideTls.TlsSession session) { var der = session.PeerCertificateDer; @@ -167,7 +160,9 @@ private static async ValueTask StartsWithPrefaceAsync(PipeReader reader) } } - // Completes both halves of the pipe, then FINs the socket and drops the connection's reference. + // Completes both halves of the pipe, then drops the connection's reference. The release is what + // FINs the socket - once any send still in flight is out, and never on a descriptor the reactor + // has already closed. internal static async ValueTask CloseAsync(IDuplexPipe pipe, IoConnection conn) { await pipe.Input.CompleteAsync(); @@ -178,7 +173,6 @@ internal static async ValueTask CloseAsync(IDuplexPipe pipe, IoConnection conn) await disposable.DisposeAsync(); // tears down a TLS transport (stops the decrypt pump, close_notify) } - Shutdown(conn.ClientFd, ShutWrite); conn.DecRef(); } diff --git a/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs b/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs index 90c06b7f3..a8581088a 100644 --- a/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs +++ b/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.IO.Pipelines; using GenHTTP.Api.Protocol; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Sinks; @@ -89,7 +90,7 @@ internal static StreamedResponseData BuildHeaders(IResponse response, StreamedRe } // Writes the content, or skips it for a HEAD while still disposing it. - internal static async ValueTask WriteBodyAsync(IResponse response, IBufferWriter writer, Func flush, bool headRequest) + internal static async ValueTask WriteBodyAsync(IResponse response, IBufferWriter writer, Func> flush, bool headRequest) { var content = response.Content; diff --git a/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs b/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs index f387ca122..37c42bbdb 100644 --- a/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs +++ b/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs @@ -1,5 +1,7 @@ using System.Buffers; +using System.IO.Pipelines; +using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; namespace GenHTTP.Engine.Ioxide.Protocol.Sinks; @@ -9,12 +11,12 @@ internal sealed class StreamedSink : IResponseSink { private readonly IBufferWriter _writer; - private readonly Func _flush; + private readonly Func> _flush; private Stream? _stream; // The response sink for one stream: write into the frame buffer, flush to pace it. - internal StreamedSink(IBufferWriter writer, Func flush) + internal StreamedSink(IBufferWriter writer, Func> flush) { _writer = writer; _flush = flush; @@ -29,12 +31,12 @@ private sealed class FlushingStream : Stream { private readonly IBufferWriter _target; - private readonly Func _flush; + private readonly Func> _flush; private long _written; // A write-only stream over the frame buffer that flushes as it goes. - internal FlushingStream(IBufferWriter target, Func flush) + internal FlushingStream(IBufferWriter target, Func> flush) { _target = target; _flush = flush; @@ -70,7 +72,7 @@ public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTo _target.Write(buffer.Span); _written += buffer.Length; - return _flush(); + return FlushOrThrowAsync(); } // The array overload, over the memory one. @@ -81,7 +83,19 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati public override void Flush() { } // Pushes the staged frames out to the peer. - public override Task FlushAsync(CancellationToken cancellationToken) => _flush().AsTask(); + public override Task FlushAsync(CancellationToken cancellationToken) => FlushOrThrowAsync().AsTask(); + + // ioxide reports a peer that reset the stream or left as a completed flush, never as an + // exception. A Stream has no other way to say so than to throw - so a writer that loops + // until a write fails (a server-sent event source, a long download) stops instead of + // producing for nobody forever, as it does over HTTP/1.1. + private async ValueTask FlushOrThrowAsync() + { + if ((await _flush()).IsCompleted) + { + throw new ConnectionClosedException(); + } + } // Write-only, so reading is a mistake worth reporting. public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); diff --git a/Modules/Files/GenHTTP.Modules.Files.csproj b/Modules/Files/GenHTTP.Modules.Files.csproj index cb0b5b150..901840567 100644 --- a/Modules/Files/GenHTTP.Modules.Files.csproj +++ b/Modules/Files/GenHTTP.Modules.Files.csproj @@ -22,7 +22,7 @@ - + diff --git a/Playground/Samples/Ioxide/KernelTlsSample.cs b/Playground/Samples/Ioxide/KernelTlsSample.cs index 8201fbeeb..808389819 100644 --- a/Playground/Samples/Ioxide/KernelTlsSample.cs +++ b/Playground/Samples/Ioxide/KernelTlsSample.cs @@ -35,7 +35,7 @@ public static IServerHost Create() * TLS connection exactly as it does on a cleartext one. * * It is not a free win, and it is off by default for that reason: on loopback it trails - * OpenSSL on large writes and costs more CPU per request. Turn it on for what it is for + * userspace TLS on large writes and costs more CPU per request. Turn it on for what it is for * - sendfile, and NICs that offload TLS - and measure your own traffic. * * curl -k --http1.1 https://localhost:8443/ok diff --git a/Playground/Samples/Ioxide/README.md b/Playground/Samples/Ioxide/README.md index c076b9c8d..ec0c1b71a 100644 --- a/Playground/Samples/Ioxide/README.md +++ b/Playground/Samples/Ioxide/README.md @@ -19,7 +19,7 @@ A deployment points the providers at the files its ACME client already writes in | `Http1Sample` | `http://localhost:8080` | the smallest binding there is - no certificate, no negotiation | | `Http2Sample` | `http://localhost:8081`, `:8082` | h2c: HTTP/2 alone, and sharing one socket with HTTP/1.1 | | `Http3Sample` | `https://localhost:8443` | QUIC only, and why it needs the certificate as files | -| `TlsSample` | `https://localhost:8443` | TLS in OpenSSL, with ALPN choosing h2 or http/1.1 | +| `TlsSample` | `https://localhost:8443` | TLS in BoringSSL, with ALPN choosing h2 or http/1.1 | | `KernelTlsSample` | `https://localhost:8443` | the record layer in the kernel, and what it pins | | `SniSample` | `https://localhost:8443` | a certificate per host name, chosen by the client | | `CertificateRotationSample` | `https://localhost:8443` | replacing certificates on a running server (`SIGHUP`) | diff --git a/Playground/Samples/Ioxide/ShowcaseSample.cs b/Playground/Samples/Ioxide/ShowcaseSample.cs index 3ff3f3bdc..205f93830 100644 --- a/Playground/Samples/Ioxide/ShowcaseSample.cs +++ b/Playground/Samples/Ioxide/ShowcaseSample.cs @@ -184,7 +184,7 @@ public static IServerHost Create() /// different issuer. /// /// -/// OpenSSL (or ngtcp2 on HTTP/3) validates the chain against that CA before the handshake +/// BoringSSL (or ngtcp2 on HTTP/3) validates the chain against that CA before the handshake /// completes, so a bad one is refused before any request exists. Validate still runs afterwards on /// the TCP transports, for whatever the application wants to decide for itself. /// diff --git a/Playground/Samples/Ioxide/TlsSample.cs b/Playground/Samples/Ioxide/TlsSample.cs index d1f39a162..d7d0ef26e 100644 --- a/Playground/Samples/Ioxide/TlsSample.cs +++ b/Playground/Samples/Ioxide/TlsSample.cs @@ -18,17 +18,17 @@ public static IServerHost Create() { /* * - * TLS is terminated in OpenSSL, and the handshake rides the same io_uring reads and + * TLS is terminated in BoringSSL, and the handshake rides the same io_uring reads and * writes as everything else. Where a port serves both protocols, ALPN decides: the * server offers h2 first, so a client that speaks it gets HTTP/2 and everyone else * falls back to HTTP/1.1. * * The certificate can come as files or as an X509Certificate2 here - unlike HTTP/3, - * OpenSSL takes the certificate as data. Files are still preferred: OpenSSL reads a + * BoringSSL takes the certificate as data. Files are still preferred: BoringSSL reads a * chain file whole, so intermediates come from the file rather than the machine store, * and the private key never enters managed memory. * - * SslProtocols names the floor. OpenSSL takes a minimum version and has no maximum, so + * SslProtocols names the floor. ioxide takes a minimum version and has no maximum, so * Tls12 | Tls13 means "1.2 and up" and Tls13 alone means 1.3-only. * * curl -k --http1.1 https://localhost:8443/ok diff --git a/Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs b/Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs new file mode 100644 index 000000000..f864e52ba --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.Functional; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// The TCP idle timeout runs only while the server waits on its client, so an idle keep-alive +/// connection is closed, while a handler that takes longer than the timeout keeps its connection. +/// +[TestClass] +public sealed class IdleTimeoutTests +{ + + private static readonly EngineOptions Options = new() + { + Reactor = new ReactorOptions { ReactorCount = 2 }, + Tcp = new TcpTransportOptions { IdleTimeoutMs = 500 }, + }; + + [IoxideTestMethod] + public async Task TestAnIdleConnectionIsClosed() + { + var app = Inline.Create().Get(() => "ok"); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create(options: Options).Handler(app).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var client = new TcpClient(); + + await client.ConnectAsync(IPAddress.Loopback, port); + + var stream = client.GetStream(); + + await stream.WriteAsync("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"u8.ToArray()); + + var response = await ReadResponseAsync(stream); + + StringAssert.StartsWith(response, "HTTP/1.1 200"); + + // nothing more is sent, so the connection waits on the client from here on + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(0, await stream.ReadAsync(new byte[1], timeout.Token), "The idle connection was not closed"); + } + finally + { + await server.StopAsync(); + } + } + + [IoxideTestMethod] + public async Task TestASlowHandlerIsNotCut() + { + var app = Inline.Create().Get(async () => + { + await Task.Delay(TimeSpan.FromSeconds(2)); + return "slow"; + }); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create(options: Options).Handler(app).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(); + + Assert.AreEqual("slow", await client.GetStringAsync($"http://localhost:{port}/")); + } + finally + { + await server.StopAsync(); + } + } + + // Reads one response with a known length off the connection, leaving it open. + private static async Task ReadResponseAsync(NetworkStream stream) + { + var buffer = new byte[4096]; + var received = new StringBuilder(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + while (true) + { + var read = await stream.ReadAsync(buffer, timeout.Token); + + Assert.AreNotEqual(0, read, "The connection closed before the response arrived"); + + received.Append(Encoding.ASCII.GetString(buffer, 0, read)); + + var text = received.ToString(); + + var end = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + + if (end >= 0 && text.EndsWith("ok", StringComparison.Ordinal)) + { + return text; + } + } + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs new file mode 100644 index 000000000..42c8388e1 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs @@ -0,0 +1,229 @@ +using System.Net; +using System.Net.Quic; +using System.Net.Security; +using System.Text; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.ServerSentEvents; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// How a streamed response ends over HTTP/2 and HTTP/3, where the end of the stream is all that tells +/// the client a response is complete: a client that leaves is noticed by the content still writing, +/// and a content that fails after the headers resets the stream rather than ending it cleanly. +/// +[TestClass] +public sealed class StreamedResponseEndTests +{ + + [IoxideTestMethod] + public async Task TestEventSourceNoticesTheClientLeftOverHttp2() + { + await AssertGeneratorEndsAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestEventSourceNoticesTheClientLeftOverHttp3() + { + await AssertGeneratorEndsAsync(HttpVersion.Version30); + } + + [IoxideTestMethod] + public async Task TestFailureAfterTheHeadersResetsTheHttp2Stream() + { + await AssertResponseFailsAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestFailureAfterTheHeadersResetsTheHttp3Stream() + { + await AssertResponseFailsAsync(HttpVersion.Version30); + } + + [IoxideTestMethod] + public async Task TestFailureBeforeTheHeadersIsStillA500OverHttp2() + { + // The pre-header path is unchanged: there is still time for an honest status. + await using var server = await ServerAsync(new FailingHandler(), HttpVersion.Version20); + + using var client = server.CreateClient(); + + using var response = await client.GetAsync(server.Url); + + Assert.AreEqual(HttpStatusCode.InternalServerError, response.StatusCode); + Assert.AreEqual(HttpVersion.Version20, response.Version); + } + + #region Infrastructure + + // A source that sends until a send fails, which only happens once the engine reports the client gone. + private static async Task AssertGeneratorEndsAsync(Version version) + { + var left = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var source = EventSource.Create() + .Generator(async c => + { + while (await c.CommentAsync("ping")) + { + await Task.Delay(50); + } + + Assert.IsFalse(c.Connected); + left.TrySetResult(); + }); + + await using var server = await ServerAsync(source.Build(), version); + + using (var client = server.CreateClient()) + { + using var response = await client.GetAsync(server.Url, HttpCompletionOption.ResponseHeadersRead); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + // wait for the first event, so the generator is running when the client leaves + await using var stream = await response.Content.ReadAsStreamAsync(); + + var buffer = new byte[1024]; + var received = ""; + + while (!received.Contains(": ping")) + { + var read = await stream.ReadAsync(buffer); + Assert.AreNotEqual(0, read); + received += Encoding.ASCII.GetString(buffer, 0, read); + } + } + + var finished = await Task.WhenAny(left.Task, Task.Delay(TimeSpan.FromSeconds(10))); + + Assert.AreSame(left.Task, finished, $"The generator kept sending after the HTTP/{version} client left"); + } + + // The headers and the first part of the body are out when the content fails. The response has to + // fail rather than end, since there is no content-length to tell the client the body is short. + // Where it fails depends on the client: a reset may overtake the headers it has not read yet. + private static async Task AssertResponseFailsAsync(Version version) + { + await using var server = await ServerAsync(new FailingMidBodyHandler(), version); + + using var client = server.CreateClient(); + + var failure = await Assert.ThrowsAsync(async () => + { + using var response = await client.GetAsync(server.Url, HttpCompletionOption.ResponseHeadersRead); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + await using var stream = await response.Content.ReadAsStreamAsync(); + + await stream.CopyToAsync(Stream.Null); + }); + + Assert.IsTrue(failure is HttpRequestException or IOException, $"The response did not fail as a transfer: {failure}"); + } + + private static async Task ServerAsync(IHandler handler, Version version) + { + var port = (ushort)TestHost.NextPort(); + + IServerHost host; + + if (version == HttpVersion.Version30) + { + if (!QuicConnection.IsSupported) + { + Assert.Inconclusive("QUIC is not supported on this machine (msquic missing)."); + } + + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + host = Host.Create().Handler(handler) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), httpProtocols: HttpProtocols.Http3); + } + else + { + host = Host.Create().Handler(handler) + .Bind(IPAddress.Loopback, port, HttpProtocols.Http2); + } + + await host.StartAsync(); + + return new RunningServer(host, port, version); + } + + private sealed class RunningServer(IServerHost host, ushort port, Version version) : IAsyncDisposable + { + public string Url => version == HttpVersion.Version30 ? $"https://localhost:{port}/" : $"http://localhost:{port}/"; + + public HttpClient CreateClient() + { + var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + return new HttpClient(handler) + { + DefaultRequestVersion = version, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(10), + }; + } + + public async ValueTask DisposeAsync() => await host.StopAsync(); + } + + /// Answers with a body that fails once its first kilobyte is on the wire. + private sealed class FailingMidBodyHandler : IHandler + { + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) + => new(request.Respond() + .Status(ResponseStatus.Ok) + .Content(new FailingMidBodyContent()) + .Build()); + } + + /// Writes and flushes part of an unknown-length body, then fails. + private sealed class FailingMidBodyContent : IResponseContent + { + public ulong? Length => null; + + public ContentType? Type => ContentType.ApplicationOctetStream; + + public ReadOnlyMemory? Encoding => null; + + public ValueTask CalculateChecksumAsync() => new((ulong?)null); + + public async ValueTask WriteAsync(IResponseSink sink) + { + await sink.Stream.WriteAsync(new byte[1024]); + await sink.Stream.FlushAsync(); + + throw new InvalidOperationException("The content failed after its headers were sent"); + } + } + + /// Fails before it has a response at all. + private sealed class FailingHandler : IHandler + { + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) => throw new InvalidOperationException("The handler failed"); + } + + #endregion + +} diff --git a/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs b/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs index f02098975..f064f8775 100644 --- a/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs +++ b/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs @@ -121,7 +121,7 @@ private static async ValueTask TestAsync(ClientCertificateA { if (engine == ServerEngine.Ioxide && validator.RequireCertificate) { - // ioxide validates the client chain in OpenSSL, which wants the trust anchors before the + // ioxide validates the client chain in BoringSSL, which wants the trust anchors before the // handshake begins - so requiring a certificate there means naming what it is validated // against. ConfigurableValidator is a plain ICertificateValidator and names none, which // the engine refuses when the server starts rather than admitting a client it never From df566431e6a43be021a37d11b36786eb1a23f3ea Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 24 Oct 2026 22:40:01 +0100 Subject: [PATCH 02/18] Fail the ioxide server start when a reactor cannot start Set Reactor.OnFault, so a reactor that cannot start - a ring the kernel refuses, a port already taken, a throw from OnStart - fails StartAsync with its reason instead of ending the process from the reactor's thread. The other reactors are stopped again, rather than serving one reactor short. A reactor that faults after it started is logged as critical (#933). --- Engine/Ioxide/Infrastructure/Server.cs | 118 +++++++++++++++--- .../Engine/Ioxide/ReactorStartupTests.cs | 117 +++++++++++++++++ 2 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs diff --git a/Engine/Ioxide/Infrastructure/Server.cs b/Engine/Ioxide/Infrastructure/Server.cs index b938bec8b..3de2e57e9 100644 --- a/Engine/Ioxide/Infrastructure/Server.cs +++ b/Engine/Ioxide/Infrastructure/Server.cs @@ -80,7 +80,8 @@ internal Server( #endregion - // Starts a reactor per thread and waits until every one of them is listening. + // Starts a reactor per thread and waits until every one of them is listening. A reactor that + // cannot start fails the start with its reason, once the others are stopped again. public async ValueTask StartAsync() { await PrepareHandlerAsync(); @@ -95,18 +96,20 @@ public async ValueTask StartAsync() WarnAboutValidatorGaps(); - _threads = new Thread[serverConfig.ReactorCount]; - _reactors = new Reactor[serverConfig.ReactorCount]; + var reactors = new Reactor[serverConfig.ReactorCount]; + _tcpTlsRegistries = secureTcp ? new TcpTlsRegistry?[serverConfig.ReactorCount] : null; - CountdownEvent listening = new CountdownEvent(serverConfig.ReactorCount); + var startup = new ReactorStartup(serverConfig.ReactorCount); - for (var i = 0; i < _threads.Length; i++) + // Every reactor is built before any runs: the constructor refuses a configuration ioxide + // cannot run with, and that should happen before a single thread has started. + for (var i = 0; i < reactors.Length; i++) { // OnStart runs on the reactor's thread, long after the loop variable has moved on. int capturedIndex = i; - Reactor reactor = new Reactor(i, serverConfig) + reactors[i] = new Reactor(i, serverConfig) { OnStart = _reactor_ => { @@ -119,7 +122,31 @@ public async ValueTask StartAsync() _onReactorStart?.Invoke(_reactor_); - listening.Signal(); + startup.Started(capturedIndex); + }, + // Set so a fault reaches the host rather than ending the process from the reactor's + // thread - a kernel refusing the ring, a port already taken, a throw from OnStart. + OnFault = (_, e) => + { + if (startup.TryFail(capturedIndex, e)) + { + return; + } + + if (startup.WasStarted(capturedIndex)) + { + _logger.LogCritical(e, "Reactor {Reactor} stopped after a fault and accepts no more connections; those it served were closed", capturedIndex); + } + else if (startup.Failed < 0) + { + // The start was given up on, so nothing else reports this one. + _logger.LogError(e, "Reactor {Reactor} could not start; the server runs without it", capturedIndex); + } + else + { + // The start fails with another reactor's fault already, most likely this same one. + _logger.LogDebug(e, "Reactor {Reactor} could not start either", capturedIndex); + } }, TcpHandle = (_, tcpConnection) => { @@ -132,10 +159,14 @@ public async ValueTask StartAsync() ? (_, quicConnection) => Http3Driver.RunAsync(this, _quicEndPoint!, quicConnection, _h3Options!) : null }; + } - _reactors[i] = reactor; + _reactors = reactors; + _threads = new Thread[reactors.Length]; - _threads[i] = new Thread(reactor.Run) + for (var i = 0; i < reactors.Length; i++) + { + _threads[i] = new Thread(reactors[i].Run) { Name = $"ioxide-genhttp-{i}", IsBackground = true, @@ -144,14 +175,23 @@ public async ValueTask StartAsync() _threads[i].Start(); } - if (await Task.Run(() => listening.Wait(TimeSpan.FromSeconds(10)))) + try { - listening.Dispose(); + await startup.Listening.WaitAsync(TimeSpan.FromSeconds(10)); } - else + catch (TimeoutException) { + startup.GiveUp(); + _logger.LogWarning("Not all reactors reported listening within 10s; the server may not be fully accepting yet."); } + catch (Exception e) + { + // Rather than a server serving on fewer reactors than it was asked for. + await DisposeAsync(); + + throw new InvalidOperationException($"The ioxide engine failed to start reactor {startup.Failed}: {e.Message}", e); + } if (_logger.IsEnabled(LogLevel.Information)) { @@ -209,8 +249,8 @@ private ServerConfig BuildServerConfig() DualStack = _dualStack, - // Explicitly null: ioxide's own default is a live listener on 8080, which an - // HTTP/3-only server would otherwise inherit. + // No TCP listener unless a binding asks for one; WithTcp adds it. An HTTP/3-only + // server binds none. Tcp = null, }; @@ -308,6 +348,8 @@ public async ValueTask DisposeAsync() if (reactors is null || threads is null) { + // A start refused before any reactor ran may still have built the QUIC engine. + DisposeQuic(); return; } @@ -333,3 +375,51 @@ await Task.Run(() => _logger.LogInformation("Stopped ioxide reactors"); } } + +/// One start of the reactors: done once every one is listening, failed by the first that cannot. +internal sealed class ReactorStartup(int reactorCount) +{ + private readonly TaskCompletionSource _listening = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private readonly bool[] _started = new bool[reactorCount]; + + private int _pending = reactorCount; + + private int _failed = -1; + + // Completes once every reactor is listening, or faults with the first that could not start. + public Task Listening => _listening.Task; + + // The reactor whose fault failed the start, or -1. + public int Failed => Volatile.Read(ref _failed); + + // A reactor got through OnStart, so it is listening. + public void Started(int reactor) + { + Volatile.Write(ref _started[reactor], true); + + if (Interlocked.Decrement(ref _pending) == 0) + { + _listening.TrySetResult(); + } + } + + // Fails the start with a fault that ended a reactor before it was listening. False for a reactor + // that was serving already, or once the start has failed or been given up on - that fault is + // left to the host to report. + public bool TryFail(int reactor, Exception e) + { + if (WasStarted(reactor) || _listening.Task.IsCompleted || Interlocked.CompareExchange(ref _failed, reactor, -1) != -1) + { + return false; + } + + return _listening.TrySetException(e); + } + + // Whether the reactor got through OnStart before it faulted. + public bool WasStarted(int reactor) => Volatile.Read(ref _started[reactor]); + + // Stops waiting: a reactor failing from here on no longer fails a start nobody waits for. + public void GiveUp() => _listening.TrySetCanceled(); +} diff --git a/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs b/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs new file mode 100644 index 000000000..7104ecd0d --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Sockets; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.IO; +using GenHTTP.Modules.Layouting; + +using ioxide; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A reactor that cannot start fails the server start with its reason, instead of ending the +/// process from the reactor's thread or leaving a server that listens one reactor short. +/// +[TestClass] +public sealed class ReactorStartupTests +{ + + [IoxideTestMethod] + public async Task TestARingTheKernelRefusesFailsTheStart() + { + // Every kernel refuses a ring this large, so this fails the same way an old kernel does. + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2, RingEntries = 65_536 } }; + + var server = Build(options: options); + + var e = await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + // ioxide names the cause, and the start passes it on + StringAssert.Contains(e.Message, "32768"); + Assert.IsNotNull(e.InnerException); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestAThrowFromOnReactorStartFailsTheStart() + { + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var server = Build(_ => throw new InvalidOperationException("The reactor callback failed"), options); + + var e = await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + StringAssert.Contains(e.Message, "The reactor callback failed"); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestAPortTakenAlreadyFailsTheStart() + { + var port = (ushort)TestHost.NextPort(); + + // a listener without SO_REUSEPORT, which the reactors' listeners cannot share the port with + using var taken = new TcpListener(IPAddress.Any, port); + + taken.Start(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var server = Build(options: options, port: port); + + await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestTheReactorsServeOnceStarted() + { + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var port = (ushort)TestHost.NextPort(); + + var server = Build(options: options, port: port); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(); + + Assert.AreEqual("ok", await client.GetStringAsync($"http://localhost:{port}/ok")); + } + finally + { + await server.StopAsync(); + } + } + + [IoxideTestMethod] + public async Task TestASizeTheReactorCannotRunWithIsRefusedBeforeItRuns() + { + // ioxide refuses this in the reactor's constructor, which the engine reaches before starting + // any thread - so nothing is left half started to stop afterwards. + var options = new EngineOptions { Tcp = new TcpTransportOptions { RecvQueueEntries = 100 } }; + + var server = Build(options: options); + + await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + await server.StopAsync(); + } + + private static IServerHost Build(Action? onReactorStart = null, EngineOptions? options = null, ushort? port = null) + => Host.Create(onReactorStart, options) + .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))) + .Bind(IPAddress.Loopback, port ?? (ushort)TestHost.NextPort()); + +} From 47d43808a08dbc15649e8bd41be1e1ccef1dd362 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 24 Oct 2026 22:40:01 +0100 Subject: [PATCH 03/18] Let websocket connections serialize their writes SynchronizeWrites() on the builder makes the connection serialize every write and its flush, including the pong and close frames the default OnPing and OnClose handlers send from the message pump - which an application's own write lock never covers. Off by default; the docs on OnPing, OnClose and ISocketConnection now say which writes need serializing (#914). --- .../Functional/FunctionalWebsocketBuilder.cs | 20 +++++ Modules/Websockets/IReactiveHandler.cs | 10 +++ Modules/Websockets/ISocketConnection.cs | 5 ++ .../Websockets/Provider/ConnectionSettings.cs | 5 +- .../Websockets/Provider/WebsocketBuilder.cs | 20 ++++- .../Provider/WebsocketConnection.cs | 75 ++++++++++++++-- .../Websockets/SynchronizedWriteTests.cs | 88 +++++++++++++++++++ 7 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs diff --git a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs index 3a7cce6c7..a9bdda148 100644 --- a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs +++ b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs @@ -51,6 +51,17 @@ public FunctionalWebsocketBuilder DoNotAllocateFrameData() return this; } + /// + /// Serializes every write on a connection, including the pong and close frames the default + /// and handlers send, so frames can be sent from + /// several threads at once. + /// + public FunctionalWebsocketBuilder SynchronizeWrites() + { + _builder.SynchronizeWrites(); + return this; + } + /// /// Invoked when a new client has connected. /// @@ -106,6 +117,11 @@ public FunctionalWebsocketBuilder OnContinue(Func /// The connection to the client /// The incoming ping frame + /// + /// The default answers with a pong, written from the message pump. If you write to the + /// connection from other threads and serialize those writes with a lock of your own, replace + /// it with one that goes through that lock as well, or use . + /// public FunctionalWebsocketBuilder OnPing(Func handler) { _onPing = handler; @@ -128,6 +144,10 @@ public FunctionalWebsocketBuilder OnPong(Func /// The connection to the client /// The frame sent by the client + /// + /// The default answers with a close frame, written from the message pump - see + /// on serializing writes made from other threads. + /// public FunctionalWebsocketBuilder OnClose(Func handler) { _onClose = handler; diff --git a/Modules/Websockets/IReactiveHandler.cs b/Modules/Websockets/IReactiveHandler.cs index ac59a07a1..72d230bb9 100644 --- a/Modules/Websockets/IReactiveHandler.cs +++ b/Modules/Websockets/IReactiveHandler.cs @@ -36,6 +36,12 @@ public interface IReactiveHandler /// /// The connection to the client /// The incoming ping frame + /// + /// The default answers with a pong, written from the message pump. If you write to the + /// connection from other threads and serialize those writes with a lock of your own, override + /// this to go through that lock as well, or let the connection serialize every write + /// (SynchronizeWrites() on the builder). + /// ValueTask OnPing(IReactiveConnection connection, IWebsocketFrame message) => connection.PongAsync(message.Data); /// @@ -50,6 +56,10 @@ public interface IReactiveHandler /// /// The connection to the client /// The frame sent by the client + /// + /// The default answers with a close frame, written from the message pump - see + /// on serializing writes made from other threads. + /// ValueTask OnClose(IReactiveConnection connection, IWebsocketFrame message) => connection.CloseAsync(); /// diff --git a/Modules/Websockets/ISocketConnection.cs b/Modules/Websockets/ISocketConnection.cs index 16355de59..cbf7e69e1 100644 --- a/Modules/Websockets/ISocketConnection.cs +++ b/Modules/Websockets/ISocketConnection.cs @@ -9,6 +9,11 @@ namespace GenHTTP.Modules.Websockets; /// A connection which can be used to write websocket messages /// to the underlying stream. /// +/// +/// A connection is not thread safe unless it was built with SynchronizeWrites(). +/// Without that, writes from several threads have to be serialized by the caller - including +/// the pong and close frames the default OnPing and OnClose handlers write. +/// public interface ISocketConnection { diff --git a/Modules/Websockets/Provider/ConnectionSettings.cs b/Modules/Websockets/Provider/ConnectionSettings.cs index 95f61978d..133f4df14 100644 --- a/Modules/Websockets/Provider/ConnectionSettings.cs +++ b/Modules/Websockets/Provider/ConnectionSettings.cs @@ -10,6 +10,7 @@ namespace GenHTTP.Modules.Websockets.Provider; /// The serialization handler to send and receive structured data /// Whether the API user is expected to handle continuation frames manually /// Whether data will automatically be allocated so it can still be accessed after the next message has been read +/// Whether every write to the connection is serialized, so frames can be sent from several threads at once public record ConnectionSettings( FormatterRegistry Formatters, @@ -18,6 +19,8 @@ public record ConnectionSettings( bool HandleContinuationFramesManually, - bool AllocateFrameData + bool AllocateFrameData, + + bool SynchronizeWrites = false ); diff --git a/Modules/Websockets/Provider/WebsocketBuilder.cs b/Modules/Websockets/Provider/WebsocketBuilder.cs index 56f6115bc..f455386a7 100644 --- a/Modules/Websockets/Provider/WebsocketBuilder.cs +++ b/Modules/Websockets/Provider/WebsocketBuilder.cs @@ -11,7 +11,7 @@ public abstract class WebsocketBuilder : IHandlerBuilder where T : Websock { protected readonly List _concerns = []; - protected bool _handleContinuationFramesManually, _allocateFrameData = true; + protected bool _handleContinuationFramesManually, _allocateFrameData = true, _synchronizeWrites; protected int _maxRxBufferSize = BufferSize.Read; @@ -71,7 +71,23 @@ public T DoNotAllocateFrameData() return (T)this; } - protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData); + /// + /// By default, a connection is not thread safe, and writes from several threads need to be + /// serialized by the caller. With this option, the connection serializes every write itself - + /// including the pong and close frames sent by the default OnPing and OnClose + /// handlers, which run on the message pump rather than behind the caller's own lock. + /// + /// + /// Each call is atomic on its own. A message split into several frames (fin: false) + /// still needs the caller's lock across all of them, so another message cannot get in between. + /// + public T SynchronizeWrites() + { + _synchronizeWrites = true; + return (T)this; + } + + protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData, _synchronizeWrites); public abstract IHandler Build(); diff --git a/Modules/Websockets/Provider/WebsocketConnection.cs b/Modules/Websockets/Provider/WebsocketConnection.cs index e568744db..c42678d87 100644 --- a/Modules/Websockets/Provider/WebsocketConnection.cs +++ b/Modules/Websockets/Provider/WebsocketConnection.cs @@ -19,6 +19,9 @@ public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnec private readonly Stream _stream; private readonly PipeReader _pipeReader; + // Taken across each write and its flush, if the settings ask for writes to be serialized. + private readonly SemaphoreSlim? _writeLock; + private SequencePosition _consumed; private SequencePosition _examined; @@ -52,6 +55,8 @@ public WebsocketConnection(IRequest request, IResponseSink sink, ConnectionSetti _writer = sink.Writer; _stream = sink.Stream; + _writeLock = settings.SynchronizeWrites ? new SemaphoreSlim(1, 1) : null; + _pipeReader = request.Upgrade(); } @@ -61,20 +66,47 @@ public WebsocketConnection(IRequest request, IResponseSink sink, ConnectionSetti public async ValueTask WriteAsync(ReadOnlyMemory payload, FrameType opcode = FrameType.Text, bool fin = true, bool flush = true, CancellationToken token = default) { - Frame.Write(_writer, payload, opcode: (byte)opcode, fin); - if (flush) await _stream.FlushAsync(token); + await LockAsync(token); + + try + { + Frame.Write(_writer, payload, opcode: (byte)opcode, fin); + if (flush) await _stream.FlushAsync(token); + } + finally + { + Unlock(); + } } public async ValueTask PingAsync(bool flush = true, CancellationToken token = default) { - Frame.WritePing(_writer); - if (flush) await _stream.FlushAsync(token); + await LockAsync(token); + + try + { + Frame.WritePing(_writer); + if (flush) await _stream.FlushAsync(token); + } + finally + { + Unlock(); + } } public async ValueTask PongAsync(ReadOnlyMemory payload, bool flush = true, CancellationToken token = default) { - Frame.WritePong(_writer, payload); - if (flush) await _stream.FlushAsync(token); + await LockAsync(token); + + try + { + Frame.WritePong(_writer, payload); + if (flush) await _stream.FlushAsync(token); + } + finally + { + Unlock(); + } } public async ValueTask PongAsync(string payload, bool flush = true, CancellationToken token = default) @@ -89,11 +121,36 @@ public async ValueTask PongAsync(bool flush = true, CancellationToken token = de public async ValueTask CloseAsync(string? reason = null, ushort statusCode = 1000, bool flush = true, CancellationToken token = default) { - Frame.WriteClose(_writer, reason, statusCode); - if (flush) await _stream.FlushAsync(token); + await LockAsync(token); + + try + { + Frame.WriteClose(_writer, reason, statusCode); + if (flush) await _stream.FlushAsync(token); + } + finally + { + Unlock(); + } + } + + public async ValueTask FlushAsync(CancellationToken token = default) + { + await LockAsync(token); + + try + { + await _stream.FlushAsync(token); + } + finally + { + Unlock(); + } } - public ValueTask FlushAsync(CancellationToken token = default) => new(_stream.FlushAsync(token)); + private ValueTask LockAsync(CancellationToken token) => _writeLock is null ? ValueTask.CompletedTask : new(_writeLock.WaitAsync(token)); + + private void Unlock() => _writeLock?.Release(); public async ValueTask ReadFrameAsync(CancellationToken token = default) { diff --git a/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs new file mode 100644 index 000000000..29b1baa9b --- /dev/null +++ b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs @@ -0,0 +1,88 @@ +using System.Collections.Concurrent; +using System.Net.WebSockets; +using System.Text; + +using GenHTTP.Api.Infrastructure; + +namespace GenHTTP.Testing.Acceptance.Modules.Websockets; + +/// +/// A connection built with SynchronizeWrites() serializes every write, the framework's own +/// pongs included, so several threads can write while the client keeps pinging. +/// +[TestClass] +public sealed class SynchronizedWriteTests +{ + private const int Writers = 4; + + private const int Messages = 250; + + [TestMethod] + [MultiEngineTest] + public async Task TestSeveralThreadsWriteWhileTheClientPings(ServerEngine engine) + { + var faults = new ConcurrentQueue(); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .SynchronizeWrites() + .OnConnected(connection => + { + for (var w = 0; w < Writers; w++) + { + var writer = w; + + _ = Task.Run(async () => + { + try + { + for (var m = 0; m < Messages; m++) + { + await connection.WriteAsync(Encoding.UTF8.GetBytes($"{writer}:{m}")); + } + } + catch (Exception e) + { + faults.Enqueue(e); + } + }); + } + + return ValueTask.CompletedTask; + }); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + using var client = new ClientWebSocket(); + + // pings the server keeps answering from its message pump, while the writers write + client.Options.KeepAliveInterval = TimeSpan.FromMilliseconds(5); + client.Options.KeepAliveTimeout = TimeSpan.FromSeconds(10); + + await client.ConnectAsync(new Uri($"ws://localhost:{host.Port}"), cts.Token); + + var next = new int[Writers]; + var buffer = new byte[256]; + + for (var i = 0; i < Writers * Messages; i++) + { + var result = await client.ReceiveAsync(buffer, cts.Token); + + Assert.AreEqual(WebSocketMessageType.Text, result.MessageType); + Assert.IsTrue(result.EndOfMessage); + + var parts = Encoding.UTF8.GetString(buffer, 0, result.Count).Split(':'); + + var writer = int.Parse(parts[0]); + + // each writer's messages arrive whole and in the order it sent them + Assert.AreEqual(next[writer]++, int.Parse(parts[1])); + } + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + + Assert.IsTrue(faults.IsEmpty, string.Join(Environment.NewLine, faults)); + } + +} From 4ed6c18d2c2a9aa5cc469341775b5ace6bfe4f9a Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 24 Oct 2026 23:00:52 +0100 Subject: [PATCH 04/18] Share one stream dispatch between the HTTP/2 and HTTP/3 drivers Both drivers ran the handler, wrote the head and the body and handled a fault the same way, line for line, apart from the ioxide writer they held. That part is now StreamedDriver.DispatchAsync, driven through a small IStreamedWriter adapter per protocol; each driver keeps only the mapping of its own request. --- .../Protocol/Drivers/Quic/Http3Driver.cs | 77 ++++++--------- .../Ioxide/Protocol/Drivers/StreamedDriver.cs | 94 +++++++++++++++++++ .../Protocol/Drivers/Tcp/Http2Driver.cs | 75 +++++---------- 3 files changed, 148 insertions(+), 98 deletions(-) create mode 100644 Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index b79217907..4aca23d2c 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -1,10 +1,12 @@ +using System.Buffers; +using System.IO.Pipelines; + using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; using ioxide; using ioxide.nghttp3; -using Microsoft.Extensions.Logging; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Responses; @@ -20,20 +22,12 @@ internal static Task RunAsync(IServer server, IEndPoint endPoint, QuicConnection => new Nghttp3Connection(connection, options) .RunStreamedResponseAsync((request, writer) => DispatchAsync(server, endPoint, request, writer)); - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream, or a - // reset once the headers are out. + // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Nghttp3Request request, Nghttp3ResponseWriter writer) { StreamedRequest? mapped = null; - // Whether the head is already on the wire. IsCompleted cannot answer this - it means - // END_STREAM has been sent - so a fault AFTER the headers went out used to take the error - // path anyway, where WriteHeaders throws "headers have already been written". That second - // exception replaced the first, so the only thing ever reported was the masking error and - // the real fault was never logged at all. - var headersWritten = false; - try { var headRequest = request.Method.Span.SequenceEqual(Head.Span); @@ -53,56 +47,41 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, reader is null ? null : reader.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); - var response = await server.Handler.HandleAsync(mapped) - ?? throw new InvalidOperationException("The root request handler did not return a response"); - - var data = StreamedResponder.BuildHeaders(response, mapped); - - var head = new Nghttp3Response { Status = data.Status }; - - foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in data.Headers) + await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/3"); + } + finally + { + if (mapped is not null) { - head.Headers.Add(name, value); + StreamedRequest.Return(mapped); } + } + } - writer.WriteHeaders(head); - headersWritten = true; + /// ioxide's HTTP/3 response writer, as the shared dispatch drives it. + private readonly struct Writer(Nghttp3ResponseWriter writer) : IStreamedWriter + { + public IBufferWriter Body => writer; - await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + public Func> Flush => writer.FlushAsync; - await writer.CompleteAsync(); - } - catch (ConnectionClosedException) - { - // The client stopped the stream or went away mid-response: nobody is left to answer, - // and that is no fault. - } - catch (Exception e) - { - server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Quic.Http3Driver") - .LogError(e, "Failed to handle HTTP/3 request"); + public bool IsCompleted => writer.IsCompleted; - if (headersWritten) - { - // Too late for a 500, and a FIN would pass the truncated body off as whole. - // Rethrown, ioxide resets the stream (H3_INTERNAL_ERROR), so the client sees it fail. - throw; - } + public void WriteHead(StreamedResponseData head) + { + var response = new Nghttp3Response { Status = head.Status }; - if (!writer.IsCompleted) + foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in head.Headers) { - writer.WriteHeaders(new Nghttp3Response { Status = 500 }); + response.Headers.Add(name, value); } - await writer.CompleteAsync(); - } - finally - { - if (mapped is not null) - { - StreamedRequest.Return(mapped); - } + writer.WriteHeaders(response); } + + public void WriteHead(int status) => writer.WriteHeaders(new Nghttp3Response { Status = status }); + + public ValueTask CompleteAsync() => writer.CompleteAsync(); } } diff --git a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs new file mode 100644 index 000000000..4da352b9a --- /dev/null +++ b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs @@ -0,0 +1,94 @@ +using System.Buffers; +using System.IO.Pipelines; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide.Protocol.Requests; +using GenHTTP.Engine.Ioxide.Protocol.Responses; + +using Microsoft.Extensions.Logging; + +namespace GenHTTP.Engine.Ioxide.Protocol.Drivers; + +/// +/// ioxide's HTTP/2 or HTTP/3 response writer, as drives it. The two share +/// no type of their own, so each driver wraps its writer in one of these. +/// +internal interface IStreamedWriter +{ + + /// The protocol's frame buffer, which the body is written into. + IBufferWriter Body { get; } + + /// Pushes the staged frames out, and reports a peer that has gone. + Func> Flush { get; } + + /// Whether the stream has been ended already. + bool IsCompleted { get; } + + /// Writes the response head. + void WriteHead(StreamedResponseData head); + + /// Writes a head that carries a status and nothing else. + void WriteHead(int status); + + /// Ends the stream. + ValueTask CompleteAsync(); + +} + +/// What the HTTP/2 and HTTP/3 drivers have in common: one stream through the handler chain. +internal static class StreamedDriver +{ + + // A fault becomes a 500 while the head is still unwritten, and a reset once it is out. A client + // that left mid-response is no fault at all. + internal static async ValueTask DispatchAsync(IServer server, StreamedRequest request, TWriter writer, + bool headRequest, string protocol) where TWriter : IStreamedWriter + { + // Whether the head is already on the wire. IsCompleted cannot answer this - it means the + // stream has ended - so a fault AFTER the headers went out used to take the error path + // anyway, where writing a second head throws. That second exception replaced the first, so + // the only thing ever reported was the masking error and the real fault was never logged. + var headersWritten = false; + + try + { + var response = await server.Handler.HandleAsync(request) + ?? throw new InvalidOperationException("The root request handler did not return a response"); + + writer.WriteHead(StreamedResponder.BuildHeaders(response, request)); + headersWritten = true; + + await StreamedResponder.WriteBodyAsync(response, writer.Body, writer.Flush, headRequest); + + await writer.CompleteAsync(); + } + catch (ConnectionClosedException) + { + // The client reset or stopped the stream, or went away mid-response: nobody is left to + // answer, and that is no fault. + } + catch (Exception e) + { + server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.StreamedDriver") + .LogError(e, "Failed to handle {Protocol} request", protocol); + + if (headersWritten) + { + // Too late for a 500, and ending the stream would pass the truncated body off as + // whole. Rethrown, ioxide resets the stream (INTERNAL_ERROR on HTTP/2, + // H3_INTERNAL_ERROR on HTTP/3), so the client sees it fail. + throw; + } + + if (!writer.IsCompleted) + { + writer.WriteHead(500); + } + + await writer.CompleteAsync(); + } + } + +} diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index 74cf23e83..713411dc0 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.IO.Pipelines; using System.Net; @@ -6,7 +7,6 @@ using ioxide.http2; -using Microsoft.Extensions.Logging; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Responses; @@ -24,20 +24,12 @@ internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pi => new Http2Connection(pipe, Options) .RunAsync((request, writer) => DispatchAsync(server, endPoint, request, writer, remoteAddress, secure)); - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream, or a - // reset once the headers are out. + // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Http2Request request, Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) { StreamedRequest? mapped = null; - // Whether the head is already on the wire. IsCompleted cannot answer this - it means - // END_STREAM has been sent - so a fault AFTER the headers went out used to take the error - // path anyway, where WriteHeaders throws "headers have already been written". That second - // exception replaced the first, so the only thing ever reported was the masking error and - // the real fault was never logged at all. - var headersWritten = false; - try { var headRequest = request.Method.Span.SequenceEqual(Head.Span); @@ -55,55 +47,40 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, reader is null ? null : reader.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); - var response = await server.Handler.HandleAsync(mapped) - ?? throw new InvalidOperationException("The root request handler did not return a response"); - - var data = StreamedResponder.BuildHeaders(response, mapped); - - var head = new Http2Response { Status = data.Status }; - - foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in data.Headers) + await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/2"); + } + finally + { + if (mapped is not null) { - head.Headers.Add(name, value); + StreamedRequest.Return(mapped); } + } + } - writer.WriteHeaders(head); - headersWritten = true; + /// ioxide's HTTP/2 response writer, as the shared dispatch drives it. + private readonly struct Writer(Http2ResponseWriter writer) : IStreamedWriter + { + public IBufferWriter Body => writer; - await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + public Func> Flush => writer.FlushAsync; - await writer.CompleteAsync(); - } - catch (ConnectionClosedException) - { - // The client reset the stream or went away mid-response: nobody is left to answer, - // and that is no fault. - } - catch (Exception e) - { - server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Tcp.Http2Driver") - .LogError(e, "Failed to handle HTTP/2 request"); + public bool IsCompleted => writer.IsCompleted; - if (headersWritten) - { - // Too late for a 500, and END_STREAM would pass the truncated body off as whole. - // Rethrown, ioxide resets the stream (INTERNAL_ERROR), so the client sees it fail. - throw; - } + public void WriteHead(StreamedResponseData head) + { + var response = new Http2Response { Status = head.Status }; - if (!writer.IsCompleted) + foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in head.Headers) { - writer.WriteHeaders(new Http2Response { Status = 500 }); + response.Headers.Add(name, value); } - await writer.CompleteAsync(); - } - finally - { - if (mapped is not null) - { - StreamedRequest.Return(mapped); - } + writer.WriteHeaders(response); } + + public void WriteHead(int status) => writer.WriteHeaders(new Http2Response { Status = status }); + + public ValueTask CompleteAsync() => writer.CompleteAsync(); } } From 1df1def69c0055b2bccbe945a0b29c1d50559bf4 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 24 Oct 2026 23:00:53 +0100 Subject: [PATCH 05/18] Skip the debug message for a reactor that could not start either unless debug logging is on --- Engine/Ioxide/Infrastructure/Server.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Engine/Ioxide/Infrastructure/Server.cs b/Engine/Ioxide/Infrastructure/Server.cs index 3de2e57e9..5dae9250a 100644 --- a/Engine/Ioxide/Infrastructure/Server.cs +++ b/Engine/Ioxide/Infrastructure/Server.cs @@ -142,7 +142,7 @@ public async ValueTask StartAsync() // The start was given up on, so nothing else reports this one. _logger.LogError(e, "Reactor {Reactor} could not start; the server runs without it", capturedIndex); } - else + else if (_logger.IsEnabled(LogLevel.Debug)) { // The start fails with another reactor's fault already, most likely this same one. _logger.LogDebug(e, "Reactor {Reactor} could not start either", capturedIndex); From 4dec9690d28623688b4cbd488e2334eb5b25d242 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 26 Oct 2026 22:04:34 +0000 Subject: [PATCH 06/18] Rename the engine's IdleTimeoutMs options to ReadTimeoutMs, after ioxide IdleTimeoutMs stays as an obsolete alias on both, so code that sets it keeps compiling. --- Engine/Ioxide/EngineOptions.cs | 12 +++++++++--- Engine/Ioxide/Infrastructure/Server.Quic.cs | 2 +- Engine/Ioxide/Infrastructure/Server.Tcp.cs | 2 +- Playground/Samples/Ioxide/Http3Sample.cs | 2 +- Playground/Samples/Ioxide/ShowcaseSample.cs | 2 +- .../{IdleTimeoutTests.cs => ReadTimeoutTests.cs} | 6 +++--- 6 files changed, 16 insertions(+), 10 deletions(-) rename Testing/Acceptance/Engine/Ioxide/{IdleTimeoutTests.cs => ReadTimeoutTests.cs} (95%) diff --git a/Engine/Ioxide/EngineOptions.cs b/Engine/Ioxide/EngineOptions.cs index 794e8c470..178246b83 100644 --- a/Engine/Ioxide/EngineOptions.cs +++ b/Engine/Ioxide/EngineOptions.cs @@ -75,7 +75,10 @@ public sealed record TcpTransportOptions /// Turn it on if you serve no upgrades, or set it comfortably above your websocket ping /// interval - a ping from either side restarts the clock. /// - public int IdleTimeoutMs { get; init; } + public int ReadTimeoutMs { get; init; } + + [Obsolete("Please use 'ReadTimeoutMs' instead")] + public int IdleTimeoutMs { get => ReadTimeoutMs; init => ReadTimeoutMs = value; } /// /// Close a connection whose send has been in flight for this long. 0 (the default) disables it. @@ -83,7 +86,7 @@ public sealed record TcpTransportOptions /// /// This is what bounds a client that stops reading: its window shuts, the send never completes, /// and without a clock the connection is pinned for as long as the peer cares to hold it. TCP - /// will not end it on its own, and neither will : a client that + /// will not end it on its own, and neither will : a client that /// keeps sending restarts that clock while its own window stays shut. A websocket written from /// a background task is the case that needs this one. /// @@ -109,7 +112,10 @@ public sealed record QuicTransportOptions /// PINGs, so a slow handler is not timed out - only a client that went quiet with nothing /// pending is. /// - public int IdleTimeoutMs { get; init; } = 60_000; + public int ReadTimeoutMs { get; init; } = 60_000; + + [Obsolete("Please use 'ReadTimeoutMs' instead")] + public int IdleTimeoutMs { get => ReadTimeoutMs; init => ReadTimeoutMs = value; } public QuicRouting Routing { get; init; } = QuicRouting.Forward; diff --git a/Engine/Ioxide/Infrastructure/Server.Quic.cs b/Engine/Ioxide/Infrastructure/Server.Quic.cs index dd71d11d9..8241bab9f 100644 --- a/Engine/Ioxide/Infrastructure/Server.Quic.cs +++ b/Engine/Ioxide/Infrastructure/Server.Quic.cs @@ -85,7 +85,7 @@ private ServerConfig WithQuic(ServerConfig serverConfig) Port = quicEndPoint.Port, ConnectionFactory = _quicEngine.CreateFactory(), - ReadTimeoutMs = _engineOptions.Quic.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Quic.ReadTimeoutMs, Routing = _engineOptions.Quic.Routing, PinMigratedPeers = _engineOptions.Quic.PinMigratedPeers, }, diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.cs index 4ca71d12d..bb733fe59 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.cs @@ -32,7 +32,7 @@ private ServerConfig WithTcp(ServerConfig serverConfig) => serverConfig with // Set explicitly, because ioxide defaults both to 60s and this listener carries // upgraded connections too - inheriting that would start closing idle websockets. - ReadTimeoutMs = _engineOptions.Tcp.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Tcp.ReadTimeoutMs, SendTimeoutMs = _engineOptions.Tcp.SendTimeoutMs, }, }; diff --git a/Playground/Samples/Ioxide/Http3Sample.cs b/Playground/Samples/Ioxide/Http3Sample.cs index a079b4ab1..22f45da68 100644 --- a/Playground/Samples/Ioxide/Http3Sample.cs +++ b/Playground/Samples/Ioxide/Http3Sample.cs @@ -44,7 +44,7 @@ public static IServerHost Create() Quic = new QuicTransportOptions { HandshakeTimeoutMs = 10_000, - IdleTimeoutMs = 60_000, + ReadTimeoutMs = 60_000, }, }) .Handler(app) diff --git a/Playground/Samples/Ioxide/ShowcaseSample.cs b/Playground/Samples/Ioxide/ShowcaseSample.cs index 205f93830..ec040710a 100644 --- a/Playground/Samples/Ioxide/ShowcaseSample.cs +++ b/Playground/Samples/Ioxide/ShowcaseSample.cs @@ -132,7 +132,7 @@ public static IServerHost Create() Quic = new QuicTransportOptions { HandshakeTimeoutMs = 10_000, - IdleTimeoutMs = 60_000, + ReadTimeoutMs = 60_000, Routing = QuicRouting.Forward, PinMigratedPeers = true, // Clamped by net.core.rmem_max, and worth measuring rather than diff --git a/Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs b/Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs similarity index 95% rename from Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs rename to Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs index f864e52ba..ee1067a39 100644 --- a/Testing/Acceptance/Engine/Ioxide/IdleTimeoutTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs @@ -11,17 +11,17 @@ namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; /// -/// The TCP idle timeout runs only while the server waits on its client, so an idle keep-alive +/// The TCP read timeout runs only while the server waits on its client, so an idle keep-alive /// connection is closed, while a handler that takes longer than the timeout keeps its connection. /// [TestClass] -public sealed class IdleTimeoutTests +public sealed class ReadTimeoutTests { private static readonly EngineOptions Options = new() { Reactor = new ReactorOptions { ReactorCount = 2 }, - Tcp = new TcpTransportOptions { IdleTimeoutMs = 500 }, + Tcp = new TcpTransportOptions { ReadTimeoutMs = 500 }, }; [IoxideTestMethod] From 8e0dfc1946db1ac2679d95b9a0ed26a865694882 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:30:28 +0000 Subject: [PATCH 07/18] Update ioxide to 0.26.563, return to the reactor after the handler, and serve static files straight into the write buffer - ioxide 0.26.563, from 0.20.396: HTTP/2 and HTTP/3 run through the one RunAsync whose handler takes the body reader, so request bodies stream. Nothing else the engine uses changed. - #971: since ioxide 0.24 a reactor has no SynchronizationContext, so a handler that awaits something other than ioxide comes back on the thread that completed it. Both drivers take the reactor when they start and await a ReactorSwitch right after the handler, so the response, and the thread-statics it reads, are on the reactor again. - #975: IIoxideFileSink in GenHTTP.Api.Ioxide. HTTP/1.1 offers it where the writer is the connection's own write buffer, in cleartext or with kernel TLS: a static file is read off the ring straight into that buffer after the headers, 12 KiB per flush, with no AssetReader and no copy. TLS in BoringSSL, HTTP/2 and HTTP/3 keep the AssetReader path. - Certificate rotation: with ioxide 0.26 the reactors started with one TlsOptions share its contexts, so ReloadCertificates rotates each port through one service. One call per reactor only built the same set again. - Docs: SendTimeoutMs is also the listener's TCP_USER_TIMEOUT, and TxKernelTls fails the start without the kernel's tls module. --- API/Ioxide/GenHTTP.Api.Ioxide.csproj | 2 +- API/Ioxide/IIoxideFileSink.cs | 21 +++++ Engine/Ioxide/EngineOptions.cs | 15 +++ Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj | 8 +- .../Infrastructure/Server.Certificates.cs | 38 ++++---- .../Protocol/Drivers/Quic/Http3Driver.cs | 11 +-- .../Ioxide/Protocol/Drivers/StreamedDriver.cs | 8 ++ .../Protocol/Drivers/Tcp/Http1Driver.cs | 19 +++- .../Protocol/Drivers/Tcp/Http2Driver.cs | 15 ++- .../Protocol/Requests/StreamedRequest.cs | 6 +- .../Protocol/Responses/Http1Responder.cs | 10 +- Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs | 50 ++++++++++ Modules/Files/GenHTTP.Modules.Files.csproj | 2 +- .../Files/Multi/Ioxide/IoxideAssetContent.cs | 13 ++- .../Engine/Ioxide/CertificateTests.cs | 40 ++++++++ .../Acceptance/Engine/Ioxide/Http3Tests.cs | 72 ++++++++++++++ .../Engine/Ioxide/ReactorReturnTests.cs | 93 +++++++++++++++++++ 17 files changed, 369 insertions(+), 54 deletions(-) create mode 100644 API/Ioxide/IIoxideFileSink.cs create mode 100644 Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs create mode 100644 Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs diff --git a/API/Ioxide/GenHTTP.Api.Ioxide.csproj b/API/Ioxide/GenHTTP.Api.Ioxide.csproj index 7d0170c27..03a0cab60 100644 --- a/API/Ioxide/GenHTTP.Api.Ioxide.csproj +++ b/API/Ioxide/GenHTTP.Api.Ioxide.csproj @@ -9,7 +9,7 @@ - + diff --git a/API/Ioxide/IIoxideFileSink.cs b/API/Ioxide/IIoxideFileSink.cs new file mode 100644 index 000000000..033c8c6f4 --- /dev/null +++ b/API/Ioxide/IIoxideFileSink.cs @@ -0,0 +1,21 @@ +namespace GenHTTP.Api.Ioxide; + +/// +/// A response sink that appends part of a file to the body itself, reading it off the ring straight into the +/// connection's write buffer, after the headers already there. +/// +/// +/// Offered where that buffer is what the response is written to: HTTP/1.1 in cleartext, or with kernel TLS, where +/// the kernel encrypts on send. Content that serves a file checks the sink for this first, and copies the file +/// through a buffer of its own otherwise. +/// +public interface IIoxideFileSink +{ + + /// + /// Appends bytes of the file , from , to the + /// body. + /// + ValueTask WriteFileAsync(int fd, long offset, long length); + +} diff --git a/Engine/Ioxide/EngineOptions.cs b/Engine/Ioxide/EngineOptions.cs index 178246b83..af3fadc41 100644 --- a/Engine/Ioxide/EngineOptions.cs +++ b/Engine/Ioxide/EngineOptions.cs @@ -39,6 +39,15 @@ public sealed record TcpTransportOptions public string? CipherList { get; init; } + /// + /// Hands each TLS connection's sending to the kernel after its handshake (kTLS), so a response is + /// encrypted on send, and a static file served over HTTP/1.1 is read straight into the connection's + /// write buffer. + /// + /// + /// Needs the kernel's tls module: without it the server's start fails, rather than every + /// connection after its handshake. + /// public bool TxKernelTls { get; init; } public bool RxKernelTls { get; init; } @@ -95,6 +104,12 @@ public sealed record TcpTransportOptions /// slow client counts against it. Size it against your slowest legitimate response rather than /// against a stall - for an API that serves small payloads, something like 30s is a /// straightforward win. + /// + /// The listener also carries it as TCP_USER_TIMEOUT, so the kernel aborts a connection whose + /// client left sent data unacknowledged, or kept its window shut with data waiting, for that long. + /// That covers a client that stopped reading or vanished behind a stream of small flushes, which + /// each complete into the send buffer. A client whose network drops for longer than this, with data + /// in flight, is cut too. /// public int SendTimeoutMs { get; init; } } diff --git a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj index 4762c690b..cd42d6dc1 100644 --- a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj +++ b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj @@ -16,11 +16,11 @@ - + - - - + + + diff --git a/Engine/Ioxide/Infrastructure/Server.Certificates.cs b/Engine/Ioxide/Infrastructure/Server.Certificates.cs index 0afc25dc2..f139fddb4 100644 --- a/Engine/Ioxide/Infrastructure/Server.Certificates.cs +++ b/Engine/Ioxide/Infrastructure/Server.Certificates.cs @@ -89,43 +89,39 @@ private List ResolveRotations(TcpTlsRegistry? started, out return rotations; } - // Installs the resolved sets on every reactor, collecting failures rather than stopping half way. + // Installs the resolved sets, collecting failures rather than stopping half way. Every reactor was started with + // each port's one TlsOptions, so its reactors share the port's contexts, and rotating them through one service + // rotates them all: another call per reactor would only build the same set again. private void Publish(TcpTlsRegistry?[] registries, List rotations) { List? failures = null; - for (var reactor = 0; reactor < registries.Length; reactor++) + var registry = Array.Find(registries, static r => r is not null); + + foreach (var rotation in rotations) { - if (registries[reactor] is not { } registry) + if (registry is null || !registry.TryFor(rotation.Port, out var service)) { continue; } - foreach (var rotation in rotations) + try + { + service.ReplaceCertificates(rotation.Default, rotation.ByHost); + } + catch (Exception e) { - if (!registry.TryFor(rotation.Port, out var service)) - { - continue; - } - - try - { - service.ReplaceCertificates(rotation.Default, rotation.ByHost); - } - catch (Exception e) - { - _logger.LogError(e, "Reactor {Reactor} kept the previous certificate on port {Port}", reactor, rotation.Port); - - (failures ??= []).Add(e); - } + _logger.LogError(e, "Port {Port} kept its previous certificate", rotation.Port); + + (failures ??= []).Add(e); } } if (failures is not null) { throw new InvalidOperationException( - $"{failures.Count} of the server's TLS services refused the new certificates and still serve the previous ones; " - + "the rest were replaced. See the inner exceptions, and the errors logged against each reactor and port.", + $"{failures.Count} of the server's secure ports refused the new certificates and still serve the previous ones; " + + "the rest were replaced. See the inner exceptions, and the errors logged against each port.", new AggregateException(failures)); } } diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index 4aca23d2c..c41448417 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -17,14 +17,15 @@ internal static class Http3Driver { private static readonly ReadOnlyMemory Head = "HEAD"u8.ToArray(); - // Runs one QUIC connection as HTTP/3, dispatching each stream as it opens. + // Runs one QUIC connection as HTTP/3, dispatching each stream as soon as its headers are in. Taking + // the body reader is what makes ioxide stream the request body instead of assembling it first. internal static Task RunAsync(IServer server, IEndPoint endPoint, QuicConnection connection, Nghttp3Options options) => new Nghttp3Connection(connection, options) - .RunStreamedResponseAsync((request, writer) => DispatchAsync(server, endPoint, request, writer)); + .RunAsync((request, body, writer) => DispatchAsync(server, endPoint, request, body, writer)); // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Nghttp3Request request, - Nghttp3ResponseWriter writer) + Nghttp3BodyReader body, Nghttp3ResponseWriter writer) { StreamedRequest? mapped = null; @@ -32,8 +33,6 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, { var headRequest = request.Method.Span.SequenceEqual(Head.Span); - var reader = request.BodyReader; - mapped = StreamedRequest.Rent(); for (var i = 0; i < request.Headers.Count; i++) @@ -45,7 +44,7 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, // No remote address: ioxide's QuicConnection exposes none, and a QUIC peer may migrate // mid-connection anyway. mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - reader is null ? null : reader.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); + body.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/3"); } diff --git a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs index 4da352b9a..612979758 100644 --- a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs +++ b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs @@ -6,6 +6,8 @@ using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Responses; +using ioxide; + using Microsoft.Extensions.Logging; namespace GenHTTP.Engine.Ioxide.Protocol.Drivers; @@ -52,11 +54,17 @@ internal static async ValueTask DispatchAsync(IServer server, StreamedR // the only thing ever reported was the masking error and the real fault was never logged. var headersWritten = false; + // Taken here, on the reactor: the handler may come back on another thread. + var reactor = Reactor.Current; + try { var response = await server.Handler.HandleAsync(request) ?? throw new InvalidOperationException("The root request handler did not return a response"); + // The response, with the thread-statics it reads, is written on the reactor. + await new ReactorSwitch(reactor); + writer.WriteHead(StreamedResponder.BuildHeaders(response, request)); headersWritten = true; diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs index 065cc6fc3..49679058b 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs @@ -16,6 +16,8 @@ using Connection = GenHTTP.Api.Protocol.Connection; using IoConnection = ioxide.TcpConnection; +using IoReactor = ioxide.Reactor; +using ReactorSwitch = ioxide.ReactorSwitch; using GenHTTP.Engine.Ioxide.Protocol.Responses; using StringContent = GenHTTP.Modules.IO.Strings.StringContent; @@ -40,11 +42,17 @@ internal static async Task RunAsync(IServer server, IEndPoint endPoint, IDuplexP var reader = pipe.Input; var writer = pipe.Output; + // In cleartext, or with kernel TLS, the writer is the connection's own write buffer, which a file can be read into. + var fileConnection = writer is ioxide.TcpConnectionPipeWriter ? conn : null; + var request = RentRequest(); var into = request.Source; var reactorThreadId = Environment.CurrentManagedThreadId; + // Taken here, on the reactor: the handler may come back on another thread. + var reactor = IoReactor.Current; + try { var dataRemaining = false; @@ -74,7 +82,7 @@ internal static async Task RunAsync(IServer server, IEndPoint endPoint, IDuplexP request.Apply(server, endPoint, reader, buffer.Start, remoteAddress, null, writer); - var keepAlive = await HandleRequestAsync(server, writer, request); + var keepAlive = await HandleRequestAsync(server, writer, request, reactor, fileConnection); WarnIfThreadHopped(server, reactorThreadId, "after-handle"); if (!keepAlive || !await request.FinishAsync()) @@ -154,7 +162,8 @@ private static bool TryParseRequest(ref ReadOnlySequence buffer, BinaryReq } // Runs one request through the handler chain and writes the response; returns whether the connection lives on. - private static async ValueTask HandleRequestAsync(IServer server, PipeWriter writer, Request request) + private static async ValueTask HandleRequestAsync(IServer server, PipeWriter writer, Request request, IoReactor? reactor, + IoConnection? fileConnection) { var header = request.Header; @@ -175,9 +184,13 @@ private static async ValueTask HandleRequestAsync(IServer server, PipeWrit var response = await server.Handler.HandleAsync(request) ?? throw new InvalidOperationException("The root request handler did not return a response"); + // A handler that awaited something other than ioxide resumes on the thread that completed it, and the + // response, with the thread-statics it reads, has to be written on the reactor. + await new ReactorSwitch(reactor); + var closeRequested = response.Mode is Connection.Close or Connection.Upgrade; - await Http1Responder.WriteAsync(writer, request, response, keepAliveRequested && !closeRequested, headRequest); + await Http1Responder.WriteAsync(writer, request, response, keepAliveRequested && !closeRequested, headRequest, fileConnection); return keepAliveRequested && !closeRequested; } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index 713411dc0..1489838d1 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -17,16 +17,15 @@ internal static class Http2Driver { private static readonly ReadOnlyMemory Head = "HEAD"u8.ToArray(); - private static readonly Http2Options Options = new() { StreamRequestBodies = true }; - - // Runs one HTTP/2 connection, dispatching each stream as it opens. + // Runs one HTTP/2 connection, dispatching each stream as soon as its headers are in. Taking the + // body reader is what makes ioxide stream the request body instead of assembling it first. internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pipe, IPAddress? remoteAddress, bool secure) - => new Http2Connection(pipe, Options) - .RunAsync((request, writer) => DispatchAsync(server, endPoint, request, writer, remoteAddress, secure)); + => new Http2Connection(pipe) + .RunAsync((request, body, writer) => DispatchAsync(server, endPoint, request, body, writer, remoteAddress, secure)); // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Http2Request request, - Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) + Http2BodyReader body, Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) { StreamedRequest? mapped = null; @@ -34,8 +33,6 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, { var headRequest = request.Method.Span.SequenceEqual(Head.Span); - var reader = request.BodyReader; - mapped = StreamedRequest.Rent(); for (var i = 0; i < request.Headers.Count; i++) @@ -45,7 +42,7 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, } mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - reader is null ? null : reader.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); + body.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/2"); } diff --git a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs index 96f917594..bdb97b31e 100644 --- a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs +++ b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs @@ -87,7 +87,7 @@ internal static void Return(StreamedRequest request) // Points this request at one stream, once its headers are in. internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory method, ReadOnlyMemory path, - ReadOnlyMemory authority, Func>>? read, IPAddress? remoteAddress, + ReadOnlyMemory authority, Func>> read, IPAddress? remoteAddress, HttpProtocol protocol, bool secure) { _server = server; @@ -100,11 +100,11 @@ internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory met // A stream carries a body only when it declares one: HTTP/2 and HTTP/3 forbid Transfer-Encoding, // so Content-Length is the signal - and its absence means no body, matching what the other engines // report for a request that carries neither header. - _hasBody = read is not null && CheckForBody(_headerEntries); + _hasBody = CheckForBody(_headerEntries); if (_hasBody) { - _body.Apply(read!); + _body.Apply(read); } _client.Apply(remoteAddress, secure ? ClientProtocol.Https : ClientProtocol.Http, null); diff --git a/Engine/Ioxide/Protocol/Responses/Http1Responder.cs b/Engine/Ioxide/Protocol/Responses/Http1Responder.cs index 348b386ff..4e3f92e77 100644 --- a/Engine/Ioxide/Protocol/Responses/Http1Responder.cs +++ b/Engine/Ioxide/Protocol/Responses/Http1Responder.cs @@ -5,6 +5,8 @@ using GenHTTP.Engine.Shared.Types; using GenHTTP.Engine.Ioxide.Protocol.Sinks; +using IoConnection = ioxide.TcpConnection; + namespace GenHTTP.Engine.Ioxide.Protocol.Responses; /// Writes an HTTP/1.1 response onto the connection. @@ -12,8 +14,10 @@ internal static class Http1Responder { private static readonly byte[] ServerHeader = "Server: ioxide-genhttp\r\n"u8.ToArray(); - // Writes the status line and headers, and the body where the request is owed one. - internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, IResponse response, bool keepAlive, bool headRequest) + // Writes the status line and headers, and the body where the request is owed one. The connection is given when + // the writer is its own write buffer, so a file can be read straight into it. + internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, IResponse response, bool keepAlive, bool headRequest, + IoConnection? connection = null) { ResponseSerializer.WriteHead(writer, request, response, keepAlive, ServerHeader, Http1DateHeader.Get(), isHttp10: false); @@ -43,7 +47,7 @@ internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, } else { - await content.WriteAsync(new Http1Sink(writer)); + await content.WriteAsync(connection is null ? new Http1Sink(writer) : new Http1FileSink(writer, connection)); } } finally diff --git a/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs b/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs new file mode 100644 index 000000000..3a158f451 --- /dev/null +++ b/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs @@ -0,0 +1,50 @@ +using System.Buffers; +using System.IO.Pipelines; + +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Ioxide; +using GenHTTP.Api.Protocol; + +using IoConnection = ioxide.TcpConnection; + +namespace GenHTTP.Engine.Ioxide.Protocol.Sinks; + +/// +/// on a connection whose writer is its own write buffer, in cleartext or with kernel TLS: a +/// file is read off the ring straight into that buffer, after the headers, so its bytes are never copied. +/// +internal sealed class Http1FileSink(PipeWriter writer, IoConnection connection) : IResponseSink, IIoxideFileSink +{ + // ReadFileAsync grows the write buffer to whatever it is asked for, so each read stays well under its default + // 16 KiB, leaving room for the status line and headers ahead of the first. + private const int Chunk = 12 * 1024; + + private Stream? _stream; + + public IBufferWriter Writer => writer; + + public Stream Stream => _stream ??= new Http1WriterStream(writer, writer); + + public async ValueTask WriteFileAsync(int fd, long offset, long length) + { + var end = offset + length; + + while (offset < end) + { + var read = await connection.ReadFileAsync(fd, (int)Math.Min(end - offset, Chunk), offset); + + if (read <= 0) + { + break; + } + + connection.AdvanceWrite(read); + offset += read; + + if ((await writer.FlushAsync()).IsCompleted) + { + throw new ConnectionClosedException(); + } + } + } +} diff --git a/Modules/Files/GenHTTP.Modules.Files.csproj b/Modules/Files/GenHTTP.Modules.Files.csproj index 901840567..2052692a1 100644 --- a/Modules/Files/GenHTTP.Modules.Files.csproj +++ b/Modules/Files/GenHTTP.Modules.Files.csproj @@ -22,7 +22,7 @@ - + diff --git a/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs b/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs index 62341cf63..28cb32621 100644 --- a/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs +++ b/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs @@ -14,9 +14,10 @@ namespace GenHTTP.Modules.Files.Multi; /// bytes into the ioxide write slab at once. Re-resolves the asset under its own /// lease, so nothing is held across an await. /// -/// The body is read positionally off the ring through a per-reactor pool. -/// It cannot use the connection's write slab directly - TcpConnection.ReadFileAsync reads -/// into that slab, and this writes into GenHTTP's response sink instead - so the copy stays. +/// Where the sink is an (HTTP/1.1 in cleartext, or with kernel TLS), the +/// body is read off the ring straight into the connection's write buffer. Elsewhere (TLS in BoringSSL, +/// HTTP/2, HTTP/3) it is read positionally through a per-reactor pool and +/// copied into the sink. /// internal sealed class IoxideAssetContent(StaticAssets assets, string path, long length, ContentType contentType, ReadOnlyMemory? contentEncoding) : IResponseContent { @@ -45,6 +46,12 @@ public async ValueTask WriteAsync(IResponseSink sink) // The body is always read off the ring, from the descriptor the snapshot holds. Keeping // that descriptor honest is AssetRefresh's job, and it rebuilds the whole snapshot rather // than having every request ask whether its own file still matches. + if (sink is IIoxideFileSink files) + { + await files.WriteFileAsync(asset.Fd, 0, length); + return; + } + await WriteFromDisk(sink, asset.Fd, length); } diff --git a/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs b/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs index 44d6e1349..25f555553 100644 --- a/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs @@ -116,6 +116,46 @@ public async Task TestCertificatesRotateOnARunningServer() } } + [IoxideTestMethod] + public async Task TestCertificatesRotateOnEveryReactor() + { + // The reactors of a port share its contexts, so the server rotates them through one of its + // services: a connection has to see the new certificate whichever reactor takes it. + var dir = IoxideCertificates.Isolated(); + + var (certificate, key) = IoxideCertificates.Create("localhost", dir); + + var port = (ushort)TestHost.NextPort(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 4 } }; + + var server = Host.Create(options: options) + .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key)); + + await server.StartAsync(); + + try + { + var before = await IoxideCertificates.SerialAsync(port); + + IoxideCertificates.Create("localhost", dir); + + (server.Instance as IoxideServer)!.ReloadCertificates(); + + // Twenty connections over four reactors: the kernel spreads them by hashing each one. + for (var i = 0; i < 20; i++) + { + Assert.AreNotEqual(before, await IoxideCertificates.SerialAsync(port), + $"connection {i} was still served the certificate the server started with"); + } + } + finally + { + await server.StopAsync(); + } + } + [IoxideTestMethod] public async Task TestRotationNeedsASecureEndpoint() { diff --git a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs index 77fed8f65..123e21c95 100644 --- a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs +++ b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs @@ -7,6 +7,7 @@ using GenHTTP.Engine.Ioxide; +using GenHTTP.Modules.Functional; using GenHTTP.Modules.IO; using GenHTTP.Modules.Layouting; @@ -166,6 +167,77 @@ public async Task TestHttp3SharesItsPortNumberWithTcp() } } + [IoxideTestMethod] + public async Task TestHttp3StreamsTheRequestBody() + { + // Large enough to arrive in many chunks, which the handler reads through the body reader + // ioxide hands the driver - an echo that comes back whole proves none went missing. + if (!QuicConnection.IsSupported) + { + Assert.Inconclusive("QUIC is not supported on this machine (msquic missing)."); + } + + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + var port = (ushort)TestHost.NextPort(); + + var echo = Inline.Create().Post(async (Stream body) => + { + var copy = new MemoryStream(); + var buffer = new byte[8192]; + + // not CopyToAsync: its ConfigureAwait(false) resumes off the reactor, where the body + // reader must not be called + int read; + + while ((read = await body.ReadAsync(buffer)) > 0) + { + copy.Write(buffer, 0, read); + } + + copy.Seek(0, SeekOrigin.Begin); + + return copy; + }); + + var server = Host.Create() + .Handler(echo) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), + httpProtocols: HttpProtocols.Http3); + + await server.StartAsync(); + + try + { + using var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + using var client = new HttpClient(handler) + { + DefaultRequestVersion = HttpVersion.Version30, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + }; + + var payload = new byte[256 * 1024]; + new Random(42).NextBytes(payload); + + using var content = new ByteArrayContent(payload); + + using var response = await client.PostAsync($"https://localhost:{port}/", content); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(HttpVersion.Version30, response.Version); + + CollectionAssert.AreEqual(payload, await response.Content.ReadAsByteArrayAsync()); + } + finally + { + await server.StopAsync(); + } + } + private static IServerHost Build() => Host.Create() .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))); diff --git a/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs b/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs new file mode 100644 index 000000000..f6901c698 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs @@ -0,0 +1,93 @@ +using System.Buffers; +using System.Net; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A handler that awaits something other than ioxide comes back on the thread that completed it. The engine +/// returns to the reactor before it writes the response, since the state it keeps per thread lives there. +/// +[TestClass] +public sealed class ReactorReturnTests +{ + + [IoxideTestMethod] + public async Task TestHttp1ResponseIsWrittenOnTheReactor() + => Assert.AreEqual("reactor", await ServeAsync(HttpVersion.Version11)); + + [IoxideTestMethod] + public async Task TestHttp2ResponseIsWrittenOnTheReactor() + => Assert.AreEqual("reactor", await ServeAsync(HttpVersion.Version20)); + + // Asks for exactly one version over cleartext, so h2 goes out as prior knowledge. + private static async Task ServeAsync(Version version) + { + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create().Handler(new LeavingHandler()).Bind(IPAddress.Loopback, port, HttpProtocols.Http1AndHttp2); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(new SocketsHttpHandler()) + { + DefaultRequestVersion = version, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(10), + }; + + using var response = await client.GetAsync($"http://localhost:{port}/"); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + + return await response.Content.ReadAsStringAsync(); + } + finally + { + await server.StopAsync(); + } + } + + // Leaves the reactor before it answers, as a handler awaiting a database or the BCL's HttpClient does. + private sealed class LeavingHandler : IHandler + { + + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public async ValueTask HandleAsync(IRequest request) + { + await Task.Delay(1); + + return request.Respond().Content(new WhereContent()).Build(); + } + + } + + // Writes which thread writes it: a reactor's, or another. + private sealed class WhereContent : IResponseContent + { + + public ulong? Length => null; + + public ContentType? Type => null; + + public ReadOnlyMemory? Encoding => null; + + public ValueTask CalculateChecksumAsync() => new((ulong?)null); + + public ValueTask WriteAsync(IResponseSink sink) + { + sink.Writer.Write(ioxide.Reactor.Current is not null ? "reactor"u8 : "elsewhere"u8); + return ValueTask.CompletedTask; + } + + } + +} From b3c29cec41a7fb77525fb658e524414138ff973b Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:36:03 +0000 Subject: [PATCH 08/18] Websockets serialize their writes by default, and DisableWriteSynchronization opts a connection out A connection written from several threads, as the reactive flavour exists for, raced the pong and close frames the default OnPing and OnClose handlers write from the message pump, which no lock of the application's covers. On ioxide that corrupts the connection's buffer. An uncontended SemaphoreSlim.WaitAsync completes synchronously, so serializing is the default now; a connection only one thread ever writes can opt out. (#959) --- .../Functional/FunctionalWebsocketBuilder.cs | 17 ++++++------ Modules/Websockets/IReactiveHandler.cs | 8 +++--- Modules/Websockets/ISocketConnection.cs | 7 ++--- .../Websockets/Provider/ConnectionSettings.cs | 2 +- .../Websockets/Provider/WebsocketBuilder.cs | 22 +++++++++------- .../Provider/WebsocketConnection.cs | 26 ++++++++----------- .../Websockets/SynchronizedWriteTests.cs | 22 +++++++++++++--- 7 files changed, 61 insertions(+), 43 deletions(-) diff --git a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs index a9bdda148..ef3b1c110 100644 --- a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs +++ b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs @@ -52,13 +52,13 @@ public FunctionalWebsocketBuilder DoNotAllocateFrameData() } /// - /// Serializes every write on a connection, including the pong and close frames the default - /// and handlers send, so frames can be sent from - /// several threads at once. + /// Stops the connection from serializing its writes, which it does by default. Writes from + /// several threads then need to be serialized by the caller, including the pong and close + /// frames the default and handlers send. /// - public FunctionalWebsocketBuilder SynchronizeWrites() + public FunctionalWebsocketBuilder DisableWriteSynchronization() { - _builder.SynchronizeWrites(); + _builder.DisableWriteSynchronization(); return this; } @@ -118,9 +118,10 @@ public FunctionalWebsocketBuilder OnContinue(FuncThe connection to the client /// The incoming ping frame /// - /// The default answers with a pong, written from the message pump. If you write to the - /// connection from other threads and serialize those writes with a lock of your own, replace - /// it with one that goes through that lock as well, or use . + /// The default answers with a pong, written from the message pump. The connection serializes it + /// with every other write, unless built with - then, if + /// you write from other threads behind a lock of your own, replace it with one that goes through + /// that lock as well. /// public FunctionalWebsocketBuilder OnPing(Func handler) { diff --git a/Modules/Websockets/IReactiveHandler.cs b/Modules/Websockets/IReactiveHandler.cs index 72d230bb9..817eb6cf7 100644 --- a/Modules/Websockets/IReactiveHandler.cs +++ b/Modules/Websockets/IReactiveHandler.cs @@ -37,10 +37,10 @@ public interface IReactiveHandler /// The connection to the client /// The incoming ping frame /// - /// The default answers with a pong, written from the message pump. If you write to the - /// connection from other threads and serialize those writes with a lock of your own, override - /// this to go through that lock as well, or let the connection serialize every write - /// (SynchronizeWrites() on the builder). + /// The default answers with a pong, written from the message pump. The connection serializes it + /// with every other write, unless built with DisableWriteSynchronization() - then, if you + /// write from other threads behind a lock of your own, override this to go through that lock + /// as well. /// ValueTask OnPing(IReactiveConnection connection, IWebsocketFrame message) => connection.PongAsync(message.Data); diff --git a/Modules/Websockets/ISocketConnection.cs b/Modules/Websockets/ISocketConnection.cs index cbf7e69e1..3ab4cdde3 100644 --- a/Modules/Websockets/ISocketConnection.cs +++ b/Modules/Websockets/ISocketConnection.cs @@ -10,9 +10,10 @@ namespace GenHTTP.Modules.Websockets; /// to the underlying stream. /// /// -/// A connection is not thread safe unless it was built with SynchronizeWrites(). -/// Without that, writes from several threads have to be serialized by the caller - including -/// the pong and close frames the default OnPing and OnClose handlers write. +/// A connection serializes its writes, so frames can be sent from several threads at once, unless +/// it was built with DisableWriteSynchronization(). Then writes from several threads have to +/// be serialized by the caller - including the pong and close frames the default OnPing and +/// OnClose handlers write. /// public interface ISocketConnection { diff --git a/Modules/Websockets/Provider/ConnectionSettings.cs b/Modules/Websockets/Provider/ConnectionSettings.cs index 133f4df14..410f5fe0e 100644 --- a/Modules/Websockets/Provider/ConnectionSettings.cs +++ b/Modules/Websockets/Provider/ConnectionSettings.cs @@ -21,6 +21,6 @@ public record ConnectionSettings( bool AllocateFrameData, - bool SynchronizeWrites = false + bool SynchronizeWrites = true ); diff --git a/Modules/Websockets/Provider/WebsocketBuilder.cs b/Modules/Websockets/Provider/WebsocketBuilder.cs index f455386a7..18fc836ec 100644 --- a/Modules/Websockets/Provider/WebsocketBuilder.cs +++ b/Modules/Websockets/Provider/WebsocketBuilder.cs @@ -11,7 +11,7 @@ public abstract class WebsocketBuilder : IHandlerBuilder where T : Websock { protected readonly List _concerns = []; - protected bool _handleContinuationFramesManually, _allocateFrameData = true, _synchronizeWrites; + protected bool _handleContinuationFramesManually, _allocateFrameData = true, _synchronizeWrites = true; protected int _maxRxBufferSize = BufferSize.Read; @@ -72,18 +72,22 @@ public T DoNotAllocateFrameData() } /// - /// By default, a connection is not thread safe, and writes from several threads need to be - /// serialized by the caller. With this option, the connection serializes every write itself - - /// including the pong and close frames sent by the default OnPing and OnClose - /// handlers, which run on the message pump rather than behind the caller's own lock. + /// By default, a connection serializes every write and its flush, so frames can be sent from + /// several threads at once - including the pong and close frames sent by the default + /// OnPing and OnClose handlers, which run on the message pump. /// /// - /// Each call is atomic on its own. A message split into several frames (fin: false) - /// still needs the caller's lock across all of them, so another message cannot get in between. + /// If a connection is only ever written by one thread at a time, you can disable this to skip + /// the lock on every write. Writes from several threads then need to be serialized by the + /// caller, the frames of the default OnPing and OnClose handlers included. + /// + /// The synchronization covers one call at a time. A message split into several frames + /// (fin: false) still needs the caller's lock across all of them, so another message + /// cannot get in between. /// - public T SynchronizeWrites() + public T DisableWriteSynchronization() { - _synchronizeWrites = true; + _synchronizeWrites = false; return (T)this; } diff --git a/Modules/Websockets/Provider/WebsocketConnection.cs b/Modules/Websockets/Provider/WebsocketConnection.cs index c42678d87..ebc7e1b07 100644 --- a/Modules/Websockets/Provider/WebsocketConnection.cs +++ b/Modules/Websockets/Provider/WebsocketConnection.cs @@ -19,7 +19,7 @@ public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnec private readonly Stream _stream; private readonly PipeReader _pipeReader; - // Taken across each write and its flush, if the settings ask for writes to be serialized. + // Taken across each write and its flush, unless the settings turn write synchronization off. private readonly SemaphoreSlim? _writeLock; private SequencePosition _consumed; @@ -66,7 +66,7 @@ public WebsocketConnection(IRequest request, IResponseSink sink, ConnectionSetti public async ValueTask WriteAsync(ReadOnlyMemory payload, FrameType opcode = FrameType.Text, bool fin = true, bool flush = true, CancellationToken token = default) { - await LockAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); try { @@ -75,13 +75,13 @@ public async ValueTask WriteAsync(ReadOnlyMemory payload, FrameType opcode } finally { - Unlock(); + _writeLock?.Release(); } } public async ValueTask PingAsync(bool flush = true, CancellationToken token = default) { - await LockAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); try { @@ -90,13 +90,13 @@ public async ValueTask PingAsync(bool flush = true, CancellationToken token = de } finally { - Unlock(); + _writeLock?.Release(); } } public async ValueTask PongAsync(ReadOnlyMemory payload, bool flush = true, CancellationToken token = default) { - await LockAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); try { @@ -105,7 +105,7 @@ public async ValueTask PongAsync(ReadOnlyMemory payload, bool flush = true } finally { - Unlock(); + _writeLock?.Release(); } } @@ -121,7 +121,7 @@ public async ValueTask PongAsync(bool flush = true, CancellationToken token = de public async ValueTask CloseAsync(string? reason = null, ushort statusCode = 1000, bool flush = true, CancellationToken token = default) { - await LockAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); try { @@ -130,13 +130,13 @@ public async ValueTask CloseAsync(string? reason = null, ushort statusCode = 100 } finally { - Unlock(); + _writeLock?.Release(); } } public async ValueTask FlushAsync(CancellationToken token = default) { - await LockAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); try { @@ -144,14 +144,10 @@ public async ValueTask FlushAsync(CancellationToken token = default) } finally { - Unlock(); + _writeLock?.Release(); } } - private ValueTask LockAsync(CancellationToken token) => _writeLock is null ? ValueTask.CompletedTask : new(_writeLock.WaitAsync(token)); - - private void Unlock() => _writeLock?.Release(); - public async ValueTask ReadFrameAsync(CancellationToken token = default) { Advance(); diff --git a/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs index 29b1baa9b..7e90b8758 100644 --- a/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs +++ b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs @@ -7,8 +7,8 @@ namespace GenHTTP.Testing.Acceptance.Modules.Websockets; /// -/// A connection built with SynchronizeWrites() serializes every write, the framework's own -/// pongs included, so several threads can write while the client keeps pinging. +/// A connection serializes every write by default, the framework's own pongs included, so several +/// threads can write while the client keeps pinging. /// [TestClass] public sealed class SynchronizedWriteTests @@ -24,7 +24,6 @@ public async Task TestSeveralThreadsWriteWhileTheClientPings(ServerEngine engine var faults = new ConcurrentQueue(); var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() - .SynchronizeWrites() .OnConnected(connection => { for (var w = 0; w < Writers; w++) @@ -85,4 +84,21 @@ public async Task TestSeveralThreadsWriteWhileTheClientPings(ServerEngine engine Assert.IsTrue(faults.IsEmpty, string.Join(Environment.NewLine, faults)); } + [TestMethod] + [MultiEngineTest] + public async Task TestConnectionWritesWithoutSynchronization(ServerEngine engine) + { + // one writer at a time, as the option requires: the message pump echoing and answering pings + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .DisableWriteSynchronization() + .HandleContinuationFramesManually() + .OnConnected(c => c.PingAsync()) + .OnMessage((c, m) => c.WriteAsync(m.Data)) + .OnContinue((c, m) => c.WriteAsync(m.Data)); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + await Integration.Client.Execute(host.Port); + } + } From 104252670657735c70c6be8ffabe7bf84740c613 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:36:03 +0000 Subject: [PATCH 09/18] Drop the obsolete IdleTimeoutMs aliases of the engine's ReadTimeoutMs options --- Engine/Ioxide/EngineOptions.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Engine/Ioxide/EngineOptions.cs b/Engine/Ioxide/EngineOptions.cs index af3fadc41..a43eeca8c 100644 --- a/Engine/Ioxide/EngineOptions.cs +++ b/Engine/Ioxide/EngineOptions.cs @@ -86,9 +86,6 @@ public sealed record TcpTransportOptions /// public int ReadTimeoutMs { get; init; } - [Obsolete("Please use 'ReadTimeoutMs' instead")] - public int IdleTimeoutMs { get => ReadTimeoutMs; init => ReadTimeoutMs = value; } - /// /// Close a connection whose send has been in flight for this long. 0 (the default) disables it. /// @@ -129,9 +126,6 @@ public sealed record QuicTransportOptions /// public int ReadTimeoutMs { get; init; } = 60_000; - [Obsolete("Please use 'ReadTimeoutMs' instead")] - public int IdleTimeoutMs { get => ReadTimeoutMs; init => ReadTimeoutMs = value; } - public QuicRouting Routing { get; init; } = QuicRouting.Forward; public bool PinMigratedPeers { get; init; } = true; From 8d1bae3a68cb54aa45732022ca3eb7795dfac85e Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:36:03 +0000 Subject: [PATCH 10/18] Playground: start the ioxide functional sample --- Playground/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Playground/Program.cs b/Playground/Program.cs index 8ff2060cb..fa7ffbd65 100644 --- a/Playground/Program.cs +++ b/Playground/Program.cs @@ -1,10 +1,10 @@ using GenHTTP.Engine.Internal; - using GenHTTP.Modules.Practices; using GenHTTP.Playground.Samples; +using GenHTTP.Playground.Samples.Ioxide; -var sample = LayoutingSample.Create(); +var sample = FunctionalSample.Create(); return await Host.Create() .Handler(sample) From a60049ddad1a39403ea28c2d7b00a71d9c8aa757 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:43:40 +0000 Subject: [PATCH 11/18] Websockets refuse writes once the handler has ended A task started from OnConnected, a broadcaster or a timer, still held the connection after the client had gone and the pump had returned, and its writes still went to the response sink of a connection the engine was done with. On ioxide, whose connections are pooled, they landed in the stream of the client that took its place. DisposeAsync now marks the connection closed under the write lock, so a write in flight finishes first, and every later write throws ConnectionClosedException, which a background writer should stop on. (#959) --- Modules/Websockets/ISocketConnection.cs | 4 + .../Provider/WebsocketConnection.cs | 32 +++- .../Modules/Websockets/LateWriteTests.cs | 141 ++++++++++++++++++ 3 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 Testing/Acceptance/Modules/Websockets/LateWriteTests.cs diff --git a/Modules/Websockets/ISocketConnection.cs b/Modules/Websockets/ISocketConnection.cs index 3ab4cdde3..e09dfc329 100644 --- a/Modules/Websockets/ISocketConnection.cs +++ b/Modules/Websockets/ISocketConnection.cs @@ -14,6 +14,10 @@ namespace GenHTTP.Modules.Websockets; /// it was built with DisableWriteSynchronization(). Then writes from several threads have to /// be serialized by the caller - including the pong and close frames the default OnPing and /// OnClose handlers write. +/// +/// Once the handler has ended, every write throws a +/// instead of reaching the client. A background writer, such as a broadcaster or a timer started +/// from OnConnected, should stop on it. /// public interface ISocketConnection { diff --git a/Modules/Websockets/Provider/WebsocketConnection.cs b/Modules/Websockets/Provider/WebsocketConnection.cs index ebc7e1b07..c975f01cc 100644 --- a/Modules/Websockets/Provider/WebsocketConnection.cs +++ b/Modules/Websockets/Provider/WebsocketConnection.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using System.Text; +using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; using GenHTTP.Modules.Websockets.Protocol; @@ -22,6 +23,9 @@ public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnec // Taken across each write and its flush, unless the settings turn write synchronization off. private readonly SemaphoreSlim? _writeLock; + // Set once the handler has ended: the sink is the engine's again, so every later write throws. + private bool _closed; + private SequencePosition _consumed; private SequencePosition _examined; @@ -70,6 +74,7 @@ public async ValueTask WriteAsync(ReadOnlyMemory payload, FrameType opcode try { + ThrowIfClosed(); Frame.Write(_writer, payload, opcode: (byte)opcode, fin); if (flush) await _stream.FlushAsync(token); } @@ -85,6 +90,7 @@ public async ValueTask PingAsync(bool flush = true, CancellationToken token = de try { + ThrowIfClosed(); Frame.WritePing(_writer); if (flush) await _stream.FlushAsync(token); } @@ -100,6 +106,7 @@ public async ValueTask PongAsync(ReadOnlyMemory payload, bool flush = true try { + ThrowIfClosed(); Frame.WritePong(_writer, payload); if (flush) await _stream.FlushAsync(token); } @@ -125,6 +132,7 @@ public async ValueTask CloseAsync(string? reason = null, ushort statusCode = 100 try { + ThrowIfClosed(); Frame.WriteClose(_writer, reason, statusCode); if (flush) await _stream.FlushAsync(token); } @@ -140,6 +148,7 @@ public async ValueTask FlushAsync(CancellationToken token = default) try { + ThrowIfClosed(); await _stream.FlushAsync(token); } finally @@ -148,6 +157,14 @@ public async ValueTask FlushAsync(CancellationToken token = default) } } + private void ThrowIfClosed() + { + if (Volatile.Read(ref _closed)) + { + throw new ConnectionClosedException(); + } + } + public async ValueTask ReadFrameAsync(CancellationToken token = default) { Advance(); @@ -412,7 +429,20 @@ private static bool IsControlFrame(WebsocketFrame frame) #region Disposable Pattern - public ValueTask DisposeAsync() => _pipeReader.CompleteAsync(); + // The handler has ended. A write in flight finishes first, under the lock, and every one after it is refused rather + // than reaching a sink the engine may already have given to another connection. + public async ValueTask DisposeAsync() + { + if (_writeLock is not null) + { + await _writeLock.WaitAsync(); + } + + Volatile.Write(ref _closed, true); + _writeLock?.Release(); + + await _pipeReader.CompleteAsync(); + } #endregion diff --git a/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs b/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs new file mode 100644 index 000000000..500143829 --- /dev/null +++ b/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using System.Net.WebSockets; +using System.Text; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.Websockets; + +using GenHTTP.Testing.Acceptance.Engine.Ioxide; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Modules.Websockets; + +/// +/// A write made after the handler has ended, as a broadcaster or a timer started from OnConnected +/// makes once the client has gone, is refused instead of reaching a sink the engine is done with. +/// +[TestClass] +public sealed class LateWriteTests +{ + + [TestMethod] + [MultiEngineTest] + public async Task TestAWriteAfterTheHandlerHasEndedIsRefused(ServerEngine engine) + { + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .OnConnected(connection => + { + connected.TrySetResult(connection); + return ValueTask.CompletedTask; + }); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + using var client = new ClientWebSocket(); + + await client.ConnectAsync(new Uri($"ws://localhost:{host.Port}"), cts.Token); + + var connection = await connected.Task.WaitAsync(cts.Token); + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + + // The handler ends just after the close handshake. Until then a write may still fail on the transport instead. + Exception? refused = null; + + while (refused is not ConnectionClosedException && !cts.IsCancellationRequested) + { + try + { + await connection.WriteAsync("late"u8.ToArray()); + await Task.Delay(10); + } + catch (Exception e) + { + refused = e; + } + } + + Assert.IsInstanceOfType(refused, $"the late write was not refused: {refused}"); + } + + [IoxideTestMethod] + public async Task TestALateWriteDoesNotReachTheConnectionThatTookItsPlace() + { + // ioxide pools its connections, so on the only reactor a second client takes the first one's place, and a + // write still going to the first one's sink would land in the second client's stream. + var first = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .OnConnected(connection => + { + first.TrySetResult(connection); + return ValueTask.CompletedTask; + }) + .OnMessage((connection, message) => connection.WriteAsync(message.Data)); + + var port = (ushort)TestHost.NextPort(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 1 } }; + + var server = Host.Create(options: options).Handler(websocket).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + IReactiveConnection late; + + using (var client = new ClientWebSocket()) + { + await client.ConnectAsync(new Uri($"ws://localhost:{port}"), cts.Token); + + late = await first.Task.WaitAsync(cts.Token); + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + } + + // Time for the reactor to take the first connection back into its pool. + await Task.Delay(200, cts.Token); + + using var second = new ClientWebSocket(); + + await second.ConnectAsync(new Uri($"ws://localhost:{port}"), cts.Token); + + Exception? refused = null; + + try + { + await late.WriteAsync("late"u8.ToArray()); + } + catch (Exception e) + { + refused = e; + } + + await second.SendAsync("mine"u8.ToArray(), WebSocketMessageType.Text, true, cts.Token); + + var buffer = new byte[64]; + + var result = await second.ReceiveAsync(buffer, cts.Token); + + Assert.AreEqual("mine", Encoding.UTF8.GetString(buffer, 0, result.Count), "the second client received another connection's frame"); + + Assert.IsInstanceOfType(refused, $"the late write was not refused: {refused}"); + } + finally + { + await server.StopAsync(); + } + } + +} From a5d6314e2e1ee79f6657c98dd2081a1e2116b98f Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Wed, 11 Nov 2026 19:53:46 +0000 Subject: [PATCH 12/18] Playground: the ioxide samples set both the read and the flush timeout Each sample with a TCP listener sets ReadTimeoutMs and SendTimeoutMs to a minute, both off by default in the engine: a client that sends nothing is let go, and so is one that stops reading. Http3Sample binds no TCP listener; its QUIC read timeout was set already, and QUIC has no flush timeout of its own. --- Playground/Samples/Ioxide/AllProtocolsSample.cs | 7 +++++++ .../Samples/Ioxide/CertificateRotationSample.cs | 11 ++++++++++- Playground/Samples/Ioxide/Http1Sample.cs | 11 ++++++++++- Playground/Samples/Ioxide/Http2Sample.cs | 11 ++++++++++- Playground/Samples/Ioxide/KernelTlsSample.cs | 4 ++++ Playground/Samples/Ioxide/ShowcaseSample.cs | 4 ++++ Playground/Samples/Ioxide/SniSample.cs | 11 ++++++++++- Playground/Samples/Ioxide/TlsSample.cs | 4 ++++ 8 files changed, 59 insertions(+), 4 deletions(-) diff --git a/Playground/Samples/Ioxide/AllProtocolsSample.cs b/Playground/Samples/Ioxide/AllProtocolsSample.cs index 90dc06ed3..ccb0aa2f5 100644 --- a/Playground/Samples/Ioxide/AllProtocolsSample.cs +++ b/Playground/Samples/Ioxide/AllProtocolsSample.cs @@ -49,6 +49,13 @@ public static IServerHost Create() return Host.Create(options: new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 }, + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, }) .Handler(app) .Bind(IPAddress.Loopback, 8080, HttpProtocols.Http1) diff --git a/Playground/Samples/Ioxide/CertificateRotationSample.cs b/Playground/Samples/Ioxide/CertificateRotationSample.cs index 5f917bc4a..a68a1cd2e 100644 --- a/Playground/Samples/Ioxide/CertificateRotationSample.cs +++ b/Playground/Samples/Ioxide/CertificateRotationSample.cs @@ -60,7 +60,16 @@ public static IServerHost Create() certificates.Add(names[1], alpha, alphaKey); - var host = Host.Create() + var host = Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8443, certificates, httpProtocols: HttpProtocols.Http1AndHttp2); diff --git a/Playground/Samples/Ioxide/Http1Sample.cs b/Playground/Samples/Ioxide/Http1Sample.cs index 1d95dbfed..4717146a3 100644 --- a/Playground/Samples/Ioxide/Http1Sample.cs +++ b/Playground/Samples/Ioxide/Http1Sample.cs @@ -29,7 +29,16 @@ public static IServerHost Create() var app = Layout.Create() .Add("ok", Content.From(Resource.FromString("ok"))); - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8080, HttpProtocols.Http1); } diff --git a/Playground/Samples/Ioxide/Http2Sample.cs b/Playground/Samples/Ioxide/Http2Sample.cs index b15f8becf..75af7bb59 100644 --- a/Playground/Samples/Ioxide/Http2Sample.cs +++ b/Playground/Samples/Ioxide/Http2Sample.cs @@ -36,7 +36,16 @@ public static IServerHost Create() var app = Layout.Create() .Add("ok", Content.From(Resource.FromString("ok"))); - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8081, HttpProtocols.Http2) .Bind(IPAddress.Loopback, 8082, HttpProtocols.Http1AndHttp2); diff --git a/Playground/Samples/Ioxide/KernelTlsSample.cs b/Playground/Samples/Ioxide/KernelTlsSample.cs index 808389819..f22cfd764 100644 --- a/Playground/Samples/Ioxide/KernelTlsSample.cs +++ b/Playground/Samples/Ioxide/KernelTlsSample.cs @@ -53,6 +53,10 @@ public static IServerHost Create() { TxKernelTls = true, RxKernelTls = true, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, }, }) .Handler(app) diff --git a/Playground/Samples/Ioxide/ShowcaseSample.cs b/Playground/Samples/Ioxide/ShowcaseSample.cs index ec040710a..f09e92919 100644 --- a/Playground/Samples/Ioxide/ShowcaseSample.cs +++ b/Playground/Samples/Ioxide/ShowcaseSample.cs @@ -115,6 +115,10 @@ public static IServerHost Create() Tcp = new TcpTransportOptions { HandshakeTimeoutMs = 10_000, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, CipherSuites = null, CipherList = null, // kTLS needs the Linux tls module and pins TLS 1.3 - see diff --git a/Playground/Samples/Ioxide/SniSample.cs b/Playground/Samples/Ioxide/SniSample.cs index c292b9fda..41326cb5e 100644 --- a/Playground/Samples/Ioxide/SniSample.cs +++ b/Playground/Samples/Ioxide/SniSample.cs @@ -54,7 +54,16 @@ public static IServerHost Create() certificates.Add(name, certificate, key); } - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8443, certificates, httpProtocols: HttpProtocols.Http1AndHttp2); } diff --git a/Playground/Samples/Ioxide/TlsSample.cs b/Playground/Samples/Ioxide/TlsSample.cs index d7d0ef26e..bb08eb512 100644 --- a/Playground/Samples/Ioxide/TlsSample.cs +++ b/Playground/Samples/Ioxide/TlsSample.cs @@ -47,6 +47,10 @@ public static IServerHost Create() { // A peer that connects and then says nothing is swept rather than held. HandshakeTimeoutMs = 10_000, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, }, }) .Handler(app) From e3974794fe0eea42d7387c56ed36ca98b99e0595 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 17:24:55 +0000 Subject: [PATCH 13/18] fix(websockets): Enforce the RFC 6455 framing rules, and keep TryReadFrame from splitting messages - A frame with a reserved bit set, a close frame with a one-byte payload or a status code that may not be sent, and a close reason or text message that is not valid UTF-8 are reported as errors, as are a continuation frame with no message to continue and a new message started inside a fragmented one (new FrameErrorType values). - FrameError.CloseStatus gives the status code to close the connection with for an error. - TryReadFrame leaves a fragmented message, and one interrupted by a control frame, to ReadFrameAsync, which reassembles it. It returned the fragments one by one before, and a message interrupted by a ping lost its final fragment. - MaxFrameSize(int) on the websocket builders sets the largest frame payload a client may send (16 KiB, as before). --- Modules/Websockets/Protocol/Frame.Decode.cs | 88 ++++++++- Modules/Websockets/Protocol/FrameError.cs | 22 +++ Modules/Websockets/Protocol/FrameErrorType.cs | 22 ++- .../Websockets/Provider/ConnectionSettings.cs | 5 +- .../Websockets/Provider/WebsocketBuilder.cs | 29 ++- .../Provider/WebsocketConnection.cs | 74 +++++++- .../Imperative/FrameEdgeCaseTests.cs | 178 ++++++++++++++++++ .../Websockets/Protocol/Frame.Decode.Tests.cs | 73 +++++++ 8 files changed, 477 insertions(+), 14 deletions(-) diff --git a/Modules/Websockets/Protocol/Frame.Decode.cs b/Modules/Websockets/Protocol/Frame.Decode.cs index d28fd18a7..7c7a9cdf6 100644 --- a/Modules/Websockets/Protocol/Frame.Decode.cs +++ b/Modules/Websockets/Protocol/Frame.Decode.cs @@ -2,6 +2,7 @@ using System.Buffers.Binary; using System.Runtime.InteropServices; using System.Text; +using System.Text.Unicode; namespace GenHTTP.Modules.Websockets.Protocol; @@ -12,6 +13,12 @@ public static partial class Frame private const string InvalidControlFrame = "Invalid Control Frame"; private const string InvalidControlFrameLength = "Invalid Control Frame Length"; private const string PayloadTooLarge = "Payload is too large"; + private const string ReservedBitsSet = "Reserved Bits Set"; + private const string InvalidCloseFrame = "Invalid Close Frame"; + private const string InvalidCloseCode = "Invalid Close Code"; + private const string InvalidUtf8 = "Invalid UTF-8"; + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); /* Websockets RFC 6455 Frame Decode definition (LLM generated) @@ -20,17 +27,18 @@ The decode algorithm reads a raw WebSocket frame from a buffer and converts it i which determine whether the frame is a complete message, a fragment, or a control frame such as Ping, Pong, or Close. It then checks the MASK bit and the 7-bit base payload length. If the length field indicates an extended payload (126 or 127), the algorithm reads the appropriate 16-bit or 64-bit length field and adjusts the header size accordingly. Control-frame rules are enforced: control frames must - not be fragmented and must not use extended lengths. After determining the final payload length and verifying that the buffer contains + not be fragmented and must not use extended lengths, and no reserved bit may be set. After determining the final payload length and verifying that the buffer contains enough bytes for the full header, mask key, and payload, the decoder extracts the optional 4-byte mask key and applies the unmasking XOR operation in place if masking is enabled. For normal data frames (Text, Binary, Continue), the raw unmasked payload bytes are returned - as-is, while Close frames are additionally parsed to extract the close code and UTF-8 reason. Finally, the method returns a WebsocketFrame + as-is (an unfragmented Text frame is checked to be valid UTF-8), while Close frames are additionally parsed to extract and validate + the close code and UTF-8 reason. Finally, the method returns a WebsocketFrame containing the payload, frame type, and FIN flag, leaving higher-level logic to handle message reassembly and continuation semantics. */ internal static WebsocketFrame Decode( ISocketConnection connection, ref ReadOnlySequence sequence, - int rxMaxBufferSize, + int maxPayloadLength, out SequencePosition consumed, out SequencePosition examined) { @@ -73,6 +81,13 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re return new WebsocketFrame(connection, new FrameError(InvalidOpCode, FrameErrorType.InvalidOpCode)); } + if ((b0 & 0b0111_0000) != 0) + { + consumed = reader.Position; + examined = reader.Position; + return new WebsocketFrame(connection, new FrameError(ReservedBitsSet, FrameErrorType.InvalidReservedBits)); + } + var isControlFrame = frameType is FrameType.Close or FrameType.Ping or FrameType.Pong; var isMasked = (b1 & 0x80) != 0; var payloadLen7 = (byte)(b1 & 0x7F); @@ -110,10 +125,7 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re payloadLen64 = len64; } - const int MaxFrameHeaderSize = 14; - var maxAllowedPayload = rxMaxBufferSize - MaxFrameHeaderSize; - - if (payloadLen64 < 0 || payloadLen64 > maxAllowedPayload || payloadLen64 > int.MaxValue) + if (payloadLen64 < 0 || payloadLen64 > maxPayloadLength || payloadLen64 > int.MaxValue) { consumed = reader.Position; examined = reader.Position; @@ -153,6 +165,11 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re // Close frame if (frameType == FrameType.Close) { + if (payloadLength == 1) + { + return new WebsocketFrame(connection, new FrameError(InvalidCloseFrame, FrameErrorType.InvalidCloseFrame)); + } + if (payloadLength >= 2) { // Read close code @@ -160,6 +177,11 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re payloadSeq.Slice(0, 2).CopyTo(codeBuf); var closeCode = BinaryPrimitives.ReadUInt16BigEndian(codeBuf); + if (!IsValidCloseCode(closeCode)) + { + return new WebsocketFrame(connection, new FrameError(InvalidCloseCode, FrameErrorType.InvalidCloseFrame)); + } + // Read reason (allocates string, only for Close) string? reason = null; if (payloadLength > 2) @@ -168,6 +190,12 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re // so we must materialize reason bytes OR decode manually. // We'll keep it simple: materialize only the reason part. var reasonBytes = payloadSeq.Slice(2).ToArray(); // alloc only for Close + + if (!Utf8.IsValid(reasonBytes)) + { + return new WebsocketFrame(connection, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + reason = Encoding.UTF8.GetString(reasonBytes); } @@ -186,10 +214,56 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re } // ------------------------------------------------------- + // A fragmented text message can only be checked once it has been reassembled + if (frameType == FrameType.Text && fin && !IsValidUtf8(payloadSeq)) + { + return new WebsocketFrame(connection, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + // Normal frames: return raw payload slice (zero-copy) return new WebsocketFrame(connection, ref payloadSeq, frameType, fin: fin); } + /// + /// Checks whether the given payload is valid UTF-8, also when a code point spans two segments. + /// + internal static bool IsValidUtf8(in ReadOnlySequence payload) + { + if (payload.IsSingleSegment) + { + return Utf8.IsValid(payload.FirstSpan); + } + + var decoder = StrictUtf8.GetDecoder(); + + Span chars = stackalloc char[256]; + + try + { + foreach (var segment in payload) + { + var bytes = segment.Span; + + while (!bytes.IsEmpty) + { + decoder.Convert(bytes, chars, flush: false, out var used, out _, out _); + bytes = bytes[used..]; + } + } + + decoder.Convert(ReadOnlySpan.Empty, chars, flush: true, out _, out _, out _); + + return true; + } + catch (DecoderFallbackException) + { + return false; + } + } + + // RFC 6455 7.4 and the IANA registry: 1004-1006 and 1015 are never sent, 1016-2999 are unassigned + private static bool IsValidCloseCode(ushort code) => code is >= 1000 and <= 1003 or >= 1007 and <= 1014 or >= 3000 and <= 4999; + /// /// Brute force modifying the PipeReader's buffer, unmask the bytes. /// diff --git a/Modules/Websockets/Protocol/FrameError.cs b/Modules/Websockets/Protocol/FrameError.cs index fbbcc9ba1..ff8dc2243 100644 --- a/Modules/Websockets/Protocol/FrameError.cs +++ b/Modules/Websockets/Protocol/FrameError.cs @@ -4,4 +4,26 @@ public record FrameError(string Message, FrameErrorType ErrorType = FrameErrorTy { public const string ReadCanceled = "Read was canceled"; public const string UnexpectedEndOfStream = "Unexpected end of stream while reading WebSocket frame."; + + /// + /// The status code to close the connection with because of this error (RFC 6455, 7.4.1), + /// or null if there is no one left to tell, as when the client went away or the read was canceled. + /// + /// + /// + /// if (frame.IsError(out var error) && error.CloseStatus != null) + /// { + /// await connection.CloseAsync(statusCode: error.CloseStatus.Value); + /// } + /// + /// + public ushort? CloseStatus => ErrorType switch + { + FrameErrorType.InvalidOpCode or FrameErrorType.InvalidControlFrame or FrameErrorType.InvalidControlFrameLength + or FrameErrorType.InvalidReservedBits or FrameErrorType.InvalidContinuation or FrameErrorType.InvalidCloseFrame => 1002, + FrameErrorType.InvalidUtf8 => 1007, + FrameErrorType.PayloadTooLarge => 1009, + FrameErrorType.UndefinedBehavior => 1011, + _ => null + }; } \ No newline at end of file diff --git a/Modules/Websockets/Protocol/FrameErrorType.cs b/Modules/Websockets/Protocol/FrameErrorType.cs index 408e9c753..b54fe59e4 100644 --- a/Modules/Websockets/Protocol/FrameErrorType.cs +++ b/Modules/Websockets/Protocol/FrameErrorType.cs @@ -10,5 +10,25 @@ public enum FrameErrorType InvalidControlFrameLength, Canceled, IncompleteForever, // PipeReader completed without a full frame - UndefinedBehavior + UndefinedBehavior, + + /// + /// A reserved bit of the frame header is set, although no extension giving it a meaning was negotiated (RFC 6455, 5.2). + /// + InvalidReservedBits, + + /// + /// A continuation frame arrived with no message to continue, or a new message started before the previous one was finished (RFC 6455, 5.4). + /// + InvalidContinuation, + + /// + /// A close frame carries a payload of one byte or a status code that may not be sent (RFC 6455, 5.5.1 and 7.4). + /// + InvalidCloseFrame, + + /// + /// A text message or the reason of a close frame is not valid UTF-8 (RFC 6455, 8.1). + /// + InvalidUtf8 } diff --git a/Modules/Websockets/Provider/ConnectionSettings.cs b/Modules/Websockets/Provider/ConnectionSettings.cs index 410f5fe0e..11c9bf368 100644 --- a/Modules/Websockets/Provider/ConnectionSettings.cs +++ b/Modules/Websockets/Provider/ConnectionSettings.cs @@ -11,6 +11,7 @@ namespace GenHTTP.Modules.Websockets.Provider; /// Whether the API user is expected to handle continuation frames manually /// Whether data will automatically be allocated so it can still be accessed after the next message has been read /// Whether every write to the connection is serialized, so frames can be sent from several threads at once +/// The largest payload, in bytes, a single frame sent by the client may carry public record ConnectionSettings( FormatterRegistry Formatters, @@ -21,6 +22,8 @@ public record ConnectionSettings( bool AllocateFrameData, - bool SynchronizeWrites = true + bool SynchronizeWrites = true, + + int MaxFrameSize = 16 * 1024 ); diff --git a/Modules/Websockets/Provider/WebsocketBuilder.cs b/Modules/Websockets/Provider/WebsocketBuilder.cs index 18fc836ec..c5165a3d0 100644 --- a/Modules/Websockets/Provider/WebsocketBuilder.cs +++ b/Modules/Websockets/Provider/WebsocketBuilder.cs @@ -50,6 +50,11 @@ public T Serialization(ISerializationFormat format) /// handled and aggregated by the frame readers. If you would like to handle /// such frames manually, you can set this option. /// + /// + /// The frames of a fragmented text message are then passed on as they arrive, so checking + /// that the message is valid UTF-8 is up to the handler. Unfragmented text messages are + /// still checked. + /// public T HandleContinuationFramesManually() { _handleContinuationFramesManually = true; @@ -91,7 +96,29 @@ public T DisableWriteSynchronization() return (T)this; } - protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData, _synchronizeWrites); + /// + /// Sets the largest payload, in bytes, a single frame sent by the client may carry (16 KiB by default). + /// A larger frame is reported as a error. + /// + /// + /// A frame is kept in memory until it has been received completely. A message split into + /// several frames may be larger than this. + /// + /// The largest payload of a single frame + /// + /// + /// Websocket.Imperative().MaxFrameSize(64 * 1024).Handler(new MyHandler()); + /// + /// + public T MaxFrameSize(int bytes) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytes); + + _maxRxBufferSize = bytes; + return (T)this; + } + + protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData, _synchronizeWrites, _maxRxBufferSize); public abstract IHandler Build(); diff --git a/Modules/Websockets/Provider/WebsocketConnection.cs b/Modules/Websockets/Provider/WebsocketConnection.cs index c975f01cc..f39242264 100644 --- a/Modules/Websockets/Provider/WebsocketConnection.cs +++ b/Modules/Websockets/Provider/WebsocketConnection.cs @@ -14,7 +14,9 @@ namespace GenHTTP.Modules.Websockets.Provider; public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnection, IAsyncDisposable { - private const int MaxFrameSize = 16 * 1024; + private const string NothingToContinue = "Continuation frame without a message to continue"; + private const string MessageNotFinished = "New message started before the previous one was finished"; + private const string InvalidUtf8 = "Invalid UTF-8"; private readonly IBufferWriter _writer; private readonly Stream _stream; @@ -39,6 +41,9 @@ public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnec private bool _keepPipeReaderBufferValid; private bool _pendingAdvance; + // A data frame without FIN has been read and the continuation frame finishing its message has not + private bool _inMessage; + private WebsocketFrame _frame = null!; #region Get-/Setters @@ -181,6 +186,13 @@ public bool TryReadFrame(out IWebsocketFrame frame) { Advance(); + if (_skipFrameInit) + { + // A message interrupted by a control frame is completed by ReadFrameAsync, which reassembles it + frame = default!; + return false; + } + if (!_pipeReader.TryRead(out var result)) { frame = default!; @@ -204,7 +216,18 @@ public bool TryReadFrame(out IWebsocketFrame frame) var currentSequence = result.Buffer; - var decoded = Frame.Decode(this, ref currentSequence, MaxFrameSize, out var consumed, out var examined); + if (!Settings.HandleContinuationFramesManually && StartsFragmentedMessage(currentSequence)) + { + // Left in the pipe for ReadFrameAsync, which reassembles the message. It is not decoded here, + // as decoding unmasks the payload in place. + _pipeReader.AdvanceTo(result.Buffer.Start); + _consumed = result.Buffer.Start; + _examined = result.Buffer.Start; + frame = default!; + return false; + } + + var decoded = Frame.Decode(this, ref currentSequence, Settings.MaxFrameSize, out var consumed, out var examined); _consumed = consumed; _examined = examined; @@ -217,6 +240,8 @@ public bool TryReadFrame(out IWebsocketFrame frame) return false; } + decoded = CheckSequence(decoded); + if (Settings.AllocateFrameData) { decoded.SetCachedData(); @@ -288,6 +313,11 @@ private async ValueTask ReadSegmentedFrameAsync(CancellationToke { _frame.Memory = SequenceUtils.ConcatSequences(_frame.Segments); + if (_frame.Type == FrameType.Text && !Frame.IsValidUtf8(_frame.Memory)) + { + return new WebsocketFrame(this, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + return _frame; } } @@ -342,7 +372,7 @@ private async ValueTask ReadFrameSegmentAsync(bool isFirstFrame ? result.Buffer.Slice(innerExamined) : result.Buffer; - var frame = Frame.Decode(this, ref currentSequence, MaxFrameSize, out var consumed, out var examined); + var frame = Frame.Decode(this, ref currentSequence, Settings.MaxFrameSize, out var consumed, out var examined); _consumed = consumed; _examined = examined; @@ -372,12 +402,40 @@ private async ValueTask ReadFrameSegmentAsync(bool isFirstFrame // Any other frame (including other errors) _pendingAdvance = true; - return frame; + return CheckSequence(frame); } throw new InvalidOperationException("Server has been stopped"); } + // RFC 6455 5.4: a continuation frame continues the message in progress, and a new message waits for it to finish + private WebsocketFrame CheckSequence(WebsocketFrame frame) + { + switch (frame.Type) + { + case FrameType.Continue: + if (!_inMessage) + { + return new WebsocketFrame(this, new FrameError(NothingToContinue, FrameErrorType.InvalidContinuation)); + } + + _inMessage = !frame.Fin; + return frame; + + case FrameType.Text or FrameType.Binary: + if (_inMessage) + { + return new WebsocketFrame(this, new FrameError(MessageNotFinished, FrameErrorType.InvalidContinuation)); + } + + _inMessage = !frame.Fin; + return frame; + + default: + return frame; + } + } + private void Examine() { _pipeReader.AdvanceTo(_bufferStart, _examined); @@ -412,6 +470,14 @@ private void Advance() [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsErrorFrame(WebsocketFrame frame) => frame.Type == FrameType.Error; + // A Text or Binary frame without FIN, read from its first header byte + private static bool StartsFragmentedMessage(in ReadOnlySequence buffer) + { + var reader = new SequenceReader(buffer); + + return reader.TryPeek(out var b0) && (b0 & 0x80) == 0 && (b0 & 0x0F) is 0x1 or 0x2; + } + [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsControlFrame(WebsocketFrame frame) diff --git a/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs b/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs index 03f4e51dc..9b48f04f2 100644 --- a/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs +++ b/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Text; using GenHTTP.Api.Infrastructure; using GenHTTP.Modules.Websockets; @@ -86,6 +87,183 @@ public async Task TestTruncatedFrameAtEofYieldsError() Assert.AreEqual(FrameError.UnexpectedEndOfStream, await client.ReceiveTextFrameAsync()); } + [TestMethod] + [MultiEngineTest] + public async Task TestDrainedPingKeepsSegmentedMessage(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Arriving in one read, the ping and the final fragment are both drained with TryReadFrame + var combined = RawWebSocketClient.BuildClientFrame("Hello "u8, opcode: 0x1, fin: false) + .Concat(RawWebSocketClient.BuildClientFrame("ping-data"u8, opcode: 0x9, fin: true)) + .Concat(RawWebSocketClient.BuildClientFrame("World"u8, opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + var (opcode, _, payload) = await client.ReceiveFrameAsync(timeout.Token); + + Assert.AreEqual((byte)0xA, opcode); + Assert.AreEqual("ping-data", Encoding.UTF8.GetString(payload)); + + Assert.AreEqual("Hello World", await client.ReceiveTextFrameAsync(timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestDrainedFragmentedMessageIsReassembled(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var combined = RawWebSocketClient.BuildClientFrame("one"u8, opcode: 0x1, fin: true) + .Concat(RawWebSocketClient.BuildClientFrame("Hello "u8, opcode: 0x1, fin: false)) + .Concat(RawWebSocketClient.BuildClientFrame("World"u8, opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + Assert.AreEqual("one", await client.ReceiveTextFrameAsync(timeout.Token)); + Assert.AreEqual("Hello World", await client.ReceiveTextFrameAsync(timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestInvalidUtf8FailsConnection(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // 0xC3 starts a two-byte sequence that 0x28 does not continue: each fragment alone could still be valid + var combined = RawWebSocketClient.BuildClientFrame([0xC3], opcode: 0x1, fin: false) + .Concat(RawWebSocketClient.BuildClientFrame([0x28], opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + Assert.AreEqual(1007, await ReceiveCloseStatusAsync(client, timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestContinuationWithoutMessageFailsConnection(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var frame = RawWebSocketClient.BuildClientFrame("orphan"u8, opcode: 0x0, fin: true); + + await client.SendRawInChunksAsync(frame, chunkSize: frame.Length, timeout.Token); + + Assert.AreEqual(1002, await ReceiveCloseStatusAsync(client, timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestMaxFrameSizeAcceptsLargerFrames(ServerEngine engine) + { + var websocket = GenHTTP.Modules.Websockets.Websocket.Imperative() + .MaxFrameSize(64 * 1024) + .Handler(new DrainingEchoHandler()); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var message = new string('a', 64 * 1024); + var frame = RawWebSocketClient.BuildClientFrame(Encoding.UTF8.GetBytes(message), opcode: 0x1, fin: true); + + await client.SendRawInChunksAsync(frame, chunkSize: frame.Length, timeout.Token); + + Assert.AreEqual(message, await client.ReceiveTextFrameAsync(timeout.Token)); + } + + private static async Task ReceiveCloseStatusAsync(RawWebSocketClient client, CancellationToken token) + { + var (opcode, _, payload) = await client.ReceiveFrameAsync(token); + + Assert.AreEqual((byte)0x8, opcode); + + return BinaryPrimitives.ReadUInt16BigEndian(payload); + } + + /// + /// Reads the way a high-throughput handler does: one awaited read, then every frame already + /// buffered with TryReadFrame, then one flush. Protocol errors close with their status code. + /// + private sealed class DrainingEchoHandler : IImperativeHandler + { + public async ValueTask HandleAsync(IImperativeConnection connection) + { + while (true) + { + var frame = await connection.ReadFrameAsync(); + + if (!await HandleAsync(connection, frame)) return; + + while (connection.TryReadFrame(out frame)) + { + if (!await HandleAsync(connection, frame)) return; + } + + await connection.FlushAsync(); + } + } + + private static async ValueTask HandleAsync(IImperativeConnection connection, IWebsocketFrame frame) + { + if (frame.IsError(out var error)) + { + var status = error.CloseStatus; + + if (status != null) + { + await connection.CloseAsync(statusCode: status.Value); + } + + return false; + } + + switch (frame.Type) + { + case FrameType.Close: + await connection.CloseAsync(); + return false; + + case FrameType.Ping: + await connection.PongAsync(frame.Data, flush: false); + return true; + + case FrameType.Text or FrameType.Binary: + await connection.WriteAsync(frame.Data, frame.Type, flush: false); + return true; + + default: + return true; + } + } + } + private sealed class EchoAllHandler : IImperativeHandler { public async ValueTask HandleAsync(IImperativeConnection connection) diff --git a/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs b/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs index ecdbedcae..7cc6a0d24 100644 --- a/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs +++ b/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs @@ -5,6 +5,7 @@ using GenHTTP.Modules.Websockets; using GenHTTP.Modules.Websockets.Protocol; +using GenHTTP.Modules.Websockets.Utils; namespace GenHTTP.Testing.Acceptance.Modules.Websockets.Protocol; @@ -327,6 +328,78 @@ public void Decode_IncompleteExtendedFrame_ReturnsIncompleteError() Assert.AreEqual(sequence.End, examined); } + [TestMethod] + public void Decode_ReservedBitSet_ReturnsInvalidReservedBitsError() + { + // FIN=1, RSV1=1, Text "hi" - no extension was negotiated that would give RSV1 a meaning + var seq = new ReadOnlySequence(new byte[] { 0xC1, 0x02, (byte)'h', (byte)'i' }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidReservedBits); + } + + [TestMethod] + public void Decode_CloseFrame_OneBytePayload_ReturnsInvalidCloseFrameError() + { + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x01, 0x03 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidCloseFrame); + } + + [TestMethod] + public void Decode_CloseFrame_CodeNotToBeSent_ReturnsInvalidCloseFrameError() + { + // 1005 (no status received) is reserved for local use and never sent in a frame + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x02, 0x03, 0xED }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidCloseFrame); + } + + [TestMethod] + public void Decode_CloseFrame_InvalidUtf8Reason_ReturnsInvalidUtf8Error() + { + // 1000 followed by 0xC3 0x28, a lead byte without its continuation byte + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x04, 0x03, 0xE8, 0xC3, 0x28 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidUtf8); + } + + [TestMethod] + public void Decode_TextFrame_InvalidUtf8_ReturnsInvalidUtf8Error() + { + var seq = new ReadOnlySequence(new byte[] { 0x81, 0x02, 0xC3, 0x28 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidUtf8); + } + + [TestMethod] + public void IsValidUtf8_CodePointAcrossSegments() + { + // "é" is 0xC3 0xA9: split across two segments it is still valid, while 0xC3 0x28 is not + Assert.IsTrue(Frame.IsValidUtf8(TwoSegments([(byte)'a', 0xC3], [0xA9, (byte)'b']))); + Assert.IsFalse(Frame.IsValidUtf8(TwoSegments([(byte)'a', 0xC3], [0x28, (byte)'b']))); + + // a sequence cut off at the end of the payload + Assert.IsFalse(Frame.IsValidUtf8(TwoSegments([(byte)'a'], [0xC3]))); + } + + private static ReadOnlySequence TwoSegments(byte[] first, byte[] second) + { + var start = new BufferSegment(first); + var end = start.Append(second); + + return new ReadOnlySequence(start, 0, end, second.Length); + } + private static void AssertError(IWebsocketFrame frame, FrameErrorType expected) { Assert.AreEqual(FrameType.Error, frame.Type); From 28b0cd107bb36321408a3a1f892f3bb45225e9d7 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 17:58:35 +0000 Subject: [PATCH 14/18] fix(engine): Take the length of a chunk before advancing the reader past it ChunkedFraming.CopyChunk read the length of the chunk it had copied after handing the reader's buffer back with AdvanceTo. A ReadOnlySequence computes its length from its segments, and a reader may reset and reuse those once advanced past: a chunk starting in a later segment of the buffer then reported a wrong length, larger than the buffer it was copied into. Over the ioxide engine's TLS connections this cut an echoed chunked body short and faulted the connection. --- Engine/Shared/Types/Body/ChunkedFraming.cs | 6 ++- .../Engine/Body/ChunkedFramingTests.cs | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/Engine/Shared/Types/Body/ChunkedFraming.cs b/Engine/Shared/Types/Body/ChunkedFraming.cs index 834e06e41..d8aa0ceb2 100644 --- a/Engine/Shared/Types/Body/ChunkedFraming.cs +++ b/Engine/Shared/Types/Body/ChunkedFraming.cs @@ -251,12 +251,14 @@ private SequencePosition ConsumeBuffered(ReadOnlySequence available, Array private int CopyChunk(ReadOnlySequence data, SequencePosition end, Span destination) { + // Taken before the reader is advanced: the length is computed from the reader's + // segments, which it may reuse once they have been handed back + var length = (int)data.Length; + data.CopyTo(destination); Reader.AdvanceTo(end); - var length = (int)data.Length; - _position += length; return length; diff --git a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs index 78584a330..85ede5ea5 100644 --- a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs +++ b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs @@ -110,6 +110,45 @@ public async Task TestReadWholeChunks() Assert.AreEqual("NEXT", await RestAsync(pipe)); } + [TestMethod] + public async Task TestReadWholeChunkStartingInLaterSegment() + { + // The second chunk starts in the pipe's second segment and ends in its third. Advancing past + // it hands the second segment back to the pool, which resets it, so the length of the chunk + // has to be known before. + var first = new byte[6000]; + var second = new byte[6000]; + + new Random(7).NextBytes(first); + new Random(8).NextBytes(second); + + var pipe = new Pipe(); + + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"{first.Length:X}\r\n")); + await pipe.Writer.WriteAsync(first); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"\r\n{second.Length:X}\r\n")); + await pipe.Writer.WriteAsync(second); + await pipe.Writer.WriteAsync("\r\n0\r\n\r\nNEXT"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var framing = new ChunkedFraming(); + framing.Apply(pipe.Reader); + + var buffer = new byte[8192]; + + Assert.AreEqual(first.Length, await framing.ReadAsync(buffer, CancellationToken.None)); + CollectionAssert.AreEqual(first, buffer[..first.Length]); + + Assert.AreEqual(second.Length, await framing.ReadAsync(buffer, CancellationToken.None)); + CollectionAssert.AreEqual(second, buffer[..second.Length]); + + Assert.AreEqual(0, await framing.ReadAsync(buffer, CancellationToken.None)); + + Assert.IsTrue(await framing.SkipAsync()); + + Assert.AreEqual("NEXT", await RestAsync(pipe)); + } + [TestMethod] public async Task TestReadToEndOfChunkSpanningSegments() { From 5967f5c6bf64d3d67a0eed7a669fa2f8c846dd02 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 18:39:35 +0000 Subject: [PATCH 15/18] test(engine): Read large request bodies through a reader that reuses its buffers, and over TLS - RecyclingPipeReader serves a body a few segments at a time and overwrites, resets and unlinks every segment it is advanced past, so a framing that still touches memory it handed back fails instead of getting away with it. ChunkedFraming and ContentLengthFraming read 256 KiB through it across segment and read sizes, read 4 MiB to the end, and skip 1 MiB unread. - StreamBodyTests: a megabyte in chunks of uneven sizes, streamed straight back while it is still being read, on every engine. - TlsBodyTests (ioxide): bodies streamed back over TLS, chunked over HTTP/1.1 and with a Content-Length over HTTP/2, 64 KiB to 1 MiB, five times each. Without the fix of 28b0cd10, TestChunkedBodiesOverHttp1, the chunked TestLargeBodyFromRecyclingReader and TestReadWholeChunkStartingInLaterSegment fail. --- .../Engine/Body/ChunkedFramingTests.cs | 110 +++++++++++++ .../Engine/Body/ContentLengthFramingTests.cs | 71 ++++++++ .../Engine/Body/RecyclingPipeReader.cs | 153 ++++++++++++++++++ .../Acceptance/Engine/Body/StreamBodyTests.cs | 28 ++++ .../Acceptance/Engine/Body/StreamingEcho.cs | 65 ++++++++ .../Acceptance/Engine/Ioxide/TlsBodyTests.cs | 97 +++++++++++ 6 files changed, 524 insertions(+) create mode 100644 Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs create mode 100644 Testing/Acceptance/Engine/Body/StreamingEcho.cs create mode 100644 Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs diff --git a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs index 85ede5ea5..873ff028d 100644 --- a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs +++ b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs @@ -355,4 +355,114 @@ public async Task TestReuseAfterReset() return (pipe, framing); } + #region Readers reusing their buffers + + [TestMethod] + public async Task TestLargeBodyFromRecyclingReader() + { + // Chunks of uneven sizes, some larger than a read, through a reader that overwrites every + // segment it is advanced past. However segments and reads fall, the payload comes back exactly. + var payload = Payload(256 * 1024, 21); + var body = Chunked(payload, [1, 100, 4095, 4096, 4097, 16_384, 65_536, 100_000]); + + foreach (var segmentSize in (int[])[1000, 4096, 16_384]) + { + foreach (var bufferSize in (int[])[7, 3000, 8192, 70_000]) + { + var layout = $"segments of {segmentSize} bytes, reads of {bufferSize}"; + + var reader = new RecyclingPipeReader(body, segmentSize, segmentsPerRead: 3); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + var result = new MemoryStream(); + var buffer = new byte[bufferSize]; + + int read; + + while ((read = await framing.ReadAsync(buffer, CancellationToken.None)) > 0) + { + Assert.IsTrue(read <= bufferSize, $"{read} bytes read into {bufferSize} ({layout})"); + result.Write(buffer, 0, read); + } + + Assert.IsTrue(payload.AsSpan().SequenceEqual(result.ToArray()), layout); + Assert.AreEqual(payload.Length, framing.Position, layout); + + Assert.IsTrue(await framing.SkipAsync(), layout); + Assert.AreEqual("NEXT", reader.Rest(), layout); + + Assert.IsTrue(reader.Recycled > 0, layout); + } + } + } + + [TestMethod] + public async Task TestLargeBodyToEndFromRecyclingReader() + { + var payload = Payload(4 * 1024 * 1024, 22); + + var reader = new RecyclingPipeReader(Chunked(payload, [4096, 65_536, 4095, 100_000]), 4096, segmentsPerRead: 4); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + var body = await framing.ReadToEndAsync(CancellationToken.None); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(body.Span)); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + [TestMethod] + public async Task TestSkipLargeBodyFromRecyclingReader() + { + var reader = new RecyclingPipeReader(Chunked(Payload(1024 * 1024, 23), [4095, 65_536, 1]), 4096, segmentsPerRead: 2); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + private static byte[] Payload(int length, int seed) + { + var payload = new byte[length]; + new Random(seed).NextBytes(payload); + + return payload; + } + + /// + /// The payload chunk-encoded in chunks of the given sizes, taken in turn, followed by the + /// start of a next request. + /// + private static byte[] Chunked(byte[] payload, int[] chunkSizes) + { + var body = new MemoryStream(); + + var offset = 0; + var i = 0; + + while (offset < payload.Length) + { + var size = Math.Min(chunkSizes[i++ % chunkSizes.Length], payload.Length - offset); + + body.Write(Encoding.ASCII.GetBytes($"{size:X}\r\n")); + body.Write(payload, offset, size); + body.Write("\r\n"u8); + + offset += size; + } + + body.Write("0\r\n\r\nNEXT"u8); + + return body.ToArray(); + } + + #endregion + } diff --git a/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs b/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs index 3c3085225..a6e497658 100644 --- a/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs +++ b/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs @@ -211,4 +211,75 @@ internal static async Task RestAsync(Pipe pipe) return rest; } + #region Readers reusing their buffers + + [TestMethod] + public async Task TestLargeBodyFromRecyclingReader() + { + // Through a reader that overwrites every segment it is advanced past: however segments + // and reads fall, the payload comes back exactly. + var payload = RandomPayload(256 * 1024, 31); + var body = payload.Concat("NEXT"u8.ToArray()).ToArray(); + + foreach (var segmentSize in (int[])[1000, 4096, 16_384]) + { + foreach (var bufferSize in (int[])[7, 3000, 8192, 70_000]) + { + var layout = $"segments of {segmentSize} bytes, reads of {bufferSize}"; + + var reader = new RecyclingPipeReader(body, segmentSize, segmentsPerRead: 3); + + var framing = new ContentLengthFraming(); + framing.Apply(reader, payload.Length); + + var result = new MemoryStream(); + var buffer = new byte[bufferSize]; + + int read; + + while ((read = await framing.ReadAsync(buffer, CancellationToken.None)) > 0) + { + Assert.IsTrue(read <= bufferSize, $"{read} bytes read into {bufferSize} ({layout})"); + result.Write(buffer, 0, read); + } + + Assert.IsTrue(payload.AsSpan().SequenceEqual(result.ToArray()), layout); + Assert.AreEqual(payload.Length, framing.Position, layout); + + Assert.IsTrue(await framing.SkipAsync(), layout); + Assert.AreEqual("NEXT", reader.Rest(), layout); + + Assert.IsTrue(reader.Recycled > 0, layout); + } + } + } + + [TestMethod] + public async Task TestLargeBodyToEndFromRecyclingReader() + { + var payload = RandomPayload(4 * 1024 * 1024, 32); + + var reader = new RecyclingPipeReader(payload.Concat("NEXT"u8.ToArray()).ToArray(), 4096, segmentsPerRead: 4); + + var framing = new ContentLengthFraming(); + framing.Apply(reader, payload.Length); + + var body = await framing.ReadToEndAsync(CancellationToken.None); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(body.Span)); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + private static byte[] RandomPayload(int length, int seed) + { + var payload = new byte[length]; + new Random(seed).NextBytes(payload); + + return payload; + } + + #endregion + } diff --git a/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs b/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs new file mode 100644 index 000000000..66cb44ce1 --- /dev/null +++ b/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs @@ -0,0 +1,153 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; + +namespace GenHTTP.Testing.Acceptance.Engine.Body; + +/// +/// Serves a fixed body in segments of the given size, a few segments per read, and reuses +/// every segment it is advanced past the way a pooling reader may: its bytes are overwritten, +/// its running index is reset and it is unlinked. A reader of the body that still touches +/// memory it has handed back then sees wrong data or a wrong length instead of getting away +/// with it, as it does with a reader that happens not to reuse its segments right away. +/// +internal sealed class RecyclingPipeReader(byte[] data, int segmentSize, int segmentsPerRead) : PipeReader +{ + private Segment? _head; + + private Segment? _tail; + + private int _headIndex; + + private int _delivered; + + private bool _needsData = true; + + /// + /// The number of segments reused after the reader was advanced past them. + /// + public int Recycled { get; private set; } + + public override bool TryRead(out ReadResult result) + { + if (_needsData) + { + Deliver(); + _needsData = false; + } + + var buffer = (_head != null && _tail != null) ? new ReadOnlySequence(_head, _headIndex, _tail, _tail.Memory.Length) : ReadOnlySequence.Empty; + + result = new ReadResult(buffer, isCanceled: false, isCompleted: _delivered == data.Length); + return true; + } + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + TryRead(out var result); + return new(result); + } + + public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + var segment = consumed.GetObject() as Segment; + + if (segment != null) + { + var index = consumed.GetInteger(); + + // A segment consumed to its end goes as well, when the data goes on in the next one + var next = segment.Next as Segment; + + if (index == segment.Memory.Length && next != null) + { + segment = next; + index = 0; + } + + while (_head != null && _head != segment) + { + var released = _head; + + _head = released.Next as Segment; + + released.Recycle(); + Recycled++; + } + + _headIndex = index; + } + + // More data only once everything buffered has been examined, as a pipe waits for its writer then + _needsData = examined.GetObject() == _tail && examined.GetInteger() == (_tail?.Memory.Length ?? 0); + } + + /// + /// What is left of the body, once its reader is done with it. + /// + public string Rest() + { + TryRead(out var result); + + while (!result.IsCompleted) + { + AdvanceTo(result.Buffer.Start, result.Buffer.End); + TryRead(out result); + } + + return Encoding.ASCII.GetString(result.Buffer.ToArray()); + } + + public override void CancelPendingRead() { } + + public override void Complete(Exception? exception = null) { } + + private void Deliver() + { + for (var i = 0; i < segmentsPerRead && _delivered < data.Length; i++) + { + var length = Math.Min(segmentSize, data.Length - _delivered); + + var segment = new Segment(data.AsSpan(_delivered, length).ToArray(), (_tail != null) ? _tail.RunningIndex + _tail.Memory.Length : 0); + + if (_tail == null) + { + _head = segment; + _headIndex = 0; + } + else + { + _tail.Link(segment); + } + + _tail = segment; + _delivered += length; + } + } + + private sealed class Segment : ReadOnlySequenceSegment + { + private readonly byte[] _bytes; + + public Segment(byte[] bytes, long runningIndex) + { + _bytes = bytes; + + Memory = bytes; + RunningIndex = runningIndex; + } + + public void Link(Segment next) => Next = next; + + public void Recycle() + { + _bytes.AsSpan().Fill(0xEE); + + RunningIndex = 0; + Next = null; + } + } + +} diff --git a/Testing/Acceptance/Engine/Body/StreamBodyTests.cs b/Testing/Acceptance/Engine/Body/StreamBodyTests.cs index 7339460ed..1e0a94463 100644 --- a/Testing/Acceptance/Engine/Body/StreamBodyTests.cs +++ b/Testing/Acceptance/Engine/Body/StreamBodyTests.cs @@ -108,6 +108,34 @@ public async Task TestLargeChunked(ServerEngine engine) CollectionAssert.AreEqual(payload, await response.Content.ReadAsByteArrayAsync()); } + [TestMethod] + [MultiEngineTest] + public async Task TestLargeChunkedStreamedBack(ServerEngine engine) + { + // A megabyte in chunks of uneven sizes, streamed straight back while it is still being read + var payload = new byte[1024 * 1024]; + new Random(44).NextBytes(payload); + + await using var runner = await TestHost.RunAsync(new StreamingEchoHandler().Wrap(), engine: engine); + + using var client = TestHost.GetClient(); + + var request = new HttpRequestMessage(HttpMethod.Post, runner.GetUrl()) + { + Content = new UnevenContent(payload, [1, 100, 4095, 4097, 16_384, 65_536, 100_000]) + }; + + request.Headers.TransferEncodingChunked = true; + + using var response = await client.SendAsync(request); + + await response.AssertStatusAsync(HttpStatusCode.OK); + + var echoed = await response.Content.ReadAsByteArrayAsync(); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(echoed)); + } + [TestMethod] [MultiEngineTest] public async Task TestKeepAliveContentLength(ServerEngine engine) diff --git a/Testing/Acceptance/Engine/Body/StreamingEcho.cs b/Testing/Acceptance/Engine/Body/StreamingEcho.cs new file mode 100644 index 000000000..42793c434 --- /dev/null +++ b/Testing/Acceptance/Engine/Body/StreamingEcho.cs @@ -0,0 +1,65 @@ +using System.Net; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using GenHTTP.Modules.IO; + +namespace GenHTTP.Testing.Acceptance.Engine.Body; + +/// +/// Streams the request body straight back as the response, as an echo service does: the response +/// is written while the body is still being read, and the length of neither is known up front. +/// +internal sealed class StreamingEchoHandler : IHandler +{ + + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) + { + var body = request.GetBody(); + + if (body == null) + { + return new(request.Respond().Status(ResponseStatus.BadRequest).Build()); + } + + return new(request.Respond() + .Content(body.AsStream(), ContentType.ApplicationOctetStream) + .Build()); + } + +} + +/// +/// Sends the payload in writes of the given sizes, each flushed on its own, so that every write +/// travels as a chunk (or data frame) of its own. Declares its length only if asked to. +/// +internal sealed class UnevenContent(byte[] payload, int[] sizes, bool declareLength = false) : HttpContent +{ + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + var offset = 0; + var i = 0; + + while (offset < payload.Length) + { + var size = Math.Min(sizes[i++ % sizes.Length], payload.Length - offset); + + await stream.WriteAsync(payload.AsMemory(offset, size)); + await stream.FlushAsync(); + + offset += size; + } + } + + protected override bool TryComputeLength(out long length) + { + length = payload.Length; + return declareLength; + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs b/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs new file mode 100644 index 000000000..253f9cb89 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs @@ -0,0 +1,97 @@ +using System.Net; +using System.Net.Security; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Testing.Acceptance.Engine.Body; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// Request bodies over TLS, streamed straight back. The TLS reader decrypts into a pipe that reuses +/// its segments once they have been read, so this is where a body reader that still touches memory +/// it has handed back shows: an echoed chunked body used to come back short. +/// +[TestClass] +public sealed class TlsBodyTests +{ + private static readonly int[] Sizes = [64 * 1024, 100 * 1024, 1024 * 1024]; + + private static readonly int[] Writes = [1, 100, 4095, 4097, 16_384, 65_536, 100_000]; + + [IoxideTestMethod] + public async Task TestChunkedBodiesOverHttp1() + { + await RunAsync(HttpProtocols.Http1, HttpVersion.Version11, chunked: true); + } + + [IoxideTestMethod] + public async Task TestBodiesOverHttp2() + { + // With a Content-Length: the engine treats an HTTP/2 request without one as having no body + await RunAsync(HttpProtocols.Http1AndHttp2, HttpVersion.Version20, chunked: false); + } + + private static async Task RunAsync(HttpProtocols protocols, Version version, bool chunked) + { + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create() + .Handler(new StreamingEchoHandler().Wrap()) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), httpProtocols: protocols); + + await server.StartAsync(); + + try + { + using var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + using var client = new HttpClient(handler); + + // Each size a few times over, as how the data happens to arrive decides what the reader sees + foreach (var size in Sizes) + { + for (var attempt = 1; attempt <= 5; attempt++) + { + var payload = new byte[size]; + new Random(size + attempt).NextBytes(payload); + + var request = new HttpRequestMessage(HttpMethod.Post, $"https://localhost:{port}/") + { + Content = new UnevenContent(payload, Writes, declareLength: !chunked), + Version = version, + VersionPolicy = HttpVersionPolicy.RequestVersionExact + }; + + if (chunked) + { + request.Headers.TransferEncodingChunked = true; + } + + using var response = await client.SendAsync(request); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + var echoed = await response.Content.ReadAsByteArrayAsync(); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(echoed), $"{size} bytes, attempt {attempt}: {echoed.Length} came back"); + } + } + } + finally + { + await server.StopAsync(); + } + } + +} From 3ac6b93742666dff50a9766c4cdc27cb68704049 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 16 Nov 2026 17:47:39 +0000 Subject: [PATCH 16/18] chore(engine): Update ioxide to 0.27.637 All five ioxide packages move from 0.26.563 to 0.27.637. The release closes MDA2AV/ioxide#566 to #573, which were filed from this pull request's review: a file read under WriteOverflow=Segmented no longer goes out ahead of the head written before it, HTTP/2 checks a request's content-length against its DATA and grows its connection window, a stream that ends early stops the client's upload, a TLS flush after the reactor stopped returns, a failed reactor start gives back its ring, and the HTTP/2 and HTTP/3 stacks gain a request body flag, ResetStreamAsync and OnHandlerFault. The engine builds against it unchanged; the commits after this one adopt the new APIs. --- API/Ioxide/GenHTTP.Api.Ioxide.csproj | 2 +- Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj | 8 ++++---- Modules/Files/GenHTTP.Modules.Files.csproj | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/API/Ioxide/GenHTTP.Api.Ioxide.csproj b/API/Ioxide/GenHTTP.Api.Ioxide.csproj index 03a0cab60..4b9de4162 100644 --- a/API/Ioxide/GenHTTP.Api.Ioxide.csproj +++ b/API/Ioxide/GenHTTP.Api.Ioxide.csproj @@ -9,7 +9,7 @@ - + diff --git a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj index cd42d6dc1..b6c9a7f08 100644 --- a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj +++ b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj @@ -16,11 +16,11 @@ - + - - - + + + diff --git a/Modules/Files/GenHTTP.Modules.Files.csproj b/Modules/Files/GenHTTP.Modules.Files.csproj index 2052692a1..c38396da1 100644 --- a/Modules/Files/GenHTTP.Modules.Files.csproj +++ b/Modules/Files/GenHTTP.Modules.Files.csproj @@ -22,7 +22,7 @@ - + From 60c82781ef9ede3af737419302533d804529436e Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 16 Nov 2026 17:48:21 +0000 Subject: [PATCH 17/18] fix(engine): Log HTTP/2 and HTTP/3 stream faults once, through the server's logging instead of ioxide's stderr A response that failed after its head was ended by rethrowing, the only way ioxide used to have to reset a stream. ioxide then printed the fault to stderr too, so every such fault was reported twice, and the second report could be neither routed nor silenced. The shared streamed dispatch now ends the stream with the writers' ResetStreamAsync from ioxide 0.27, which sends the INTERNAL_ERROR (HTTP/2) or H3_INTERNAL_ERROR (HTTP/3) the rethrow did. What ioxide still catches itself, a fault in the dispatch's own error path or the stack failing to finish a response once its handler has returned, goes to OnHandlerFault, which logs it through the server's logging as "Failed to complete {Protocol} request". HTTP/2 connections share one Http2Options, built with the server. (#984) StreamedResponseEndTests: the reset tests assert that the fault does not reach stderr, and TestFaultReachingIoxideIsLoggedOverHttp2/3 have a log sink refuse the dispatch's own report, so the fault escapes to ioxide and has to arrive in the server's logging instead. Without the change, the reset tests and the two new ones fail. --- Engine/Ioxide/Infrastructure/Server.Quic.cs | 2 + Engine/Ioxide/Infrastructure/Server.Tcp.cs | 10 +- Engine/Ioxide/Infrastructure/Server.cs | 3 +- .../Protocol/Drivers/Quic/Http3Driver.cs | 2 + .../Ioxide/Protocol/Drivers/StreamedDriver.cs | 22 +++- .../Protocol/Drivers/Tcp/Http2Driver.cs | 7 +- .../Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs | 5 +- .../Engine/Ioxide/StreamedResponseEndTests.cs | 109 +++++++++++++++++- 8 files changed, 146 insertions(+), 14 deletions(-) diff --git a/Engine/Ioxide/Infrastructure/Server.Quic.cs b/Engine/Ioxide/Infrastructure/Server.Quic.cs index 8241bab9f..56c63cd73 100644 --- a/Engine/Ioxide/Infrastructure/Server.Quic.cs +++ b/Engine/Ioxide/Infrastructure/Server.Quic.cs @@ -1,5 +1,6 @@ using GenHTTP.Api.Infrastructure; using GenHTTP.Engine.Ioxide.Infrastructure.Endpoints; +using GenHTTP.Engine.Ioxide.Protocol.Drivers; using ioxide; using ioxide.nghttp3; @@ -72,6 +73,7 @@ private ServerConfig WithQuic(ServerConfig serverConfig) { QpackDynamicTableCapacity = _engineOptions.Http3.QpackDynamicTableCapacity, QpackBlockedStreams = _engineOptions.Http3.QpackBlockedStreams, + OnHandlerFault = StreamedDriver.ReportFaults(Logging, "HTTP/3"), }; return serverConfig with diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.cs index bb733fe59..aced96ba8 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.cs @@ -1,5 +1,7 @@ using GenHTTP.Api.Infrastructure; +using GenHTTP.Engine.Ioxide.Protocol.Drivers; using ioxide; +using ioxide.http2; namespace GenHTTP.Engine.Ioxide.Infrastructure; @@ -7,7 +9,13 @@ namespace GenHTTP.Engine.Ioxide.Infrastructure; public sealed partial class Server { private readonly ushort[] _tcpPorts; - + + // Shared by every HTTP/2 connection, so the fault hook is built once. + private readonly Http2Options _h2Options; + + private Http2Options CreateHttp2Options() + => new() { OnHandlerFault = StreamedDriver.ReportFaults(Logging, "HTTP/2") }; + // The ports serving HTTP/1.1 or HTTP/2, primary first. Empty on an HTTP/3-only server. private ushort[] ResolveTcpPorts() => _endPoints.Where(e => (e.Protocols & HttpProtocols.Http1AndHttp2) != 0) diff --git a/Engine/Ioxide/Infrastructure/Server.cs b/Engine/Ioxide/Infrastructure/Server.cs index 5dae9250a..aac2d6992 100644 --- a/Engine/Ioxide/Infrastructure/Server.cs +++ b/Engine/Ioxide/Infrastructure/Server.cs @@ -73,6 +73,7 @@ internal Server( _dualStack = _endPoints[0].DualStack; _tcpPorts = ResolveTcpPorts(); + _h2Options = CreateHttp2Options(); _quicEndPoint = ResolveQuicEndPoint(); EndPoints = new EndPointCollection(_endPoints); @@ -152,7 +153,7 @@ public async ValueTask StartAsync() { EndPoint endPoint = EndPointFor(tcpConnection.ListenerPort); - return TcpDriver.HandleAsync(this, endPoint, tcpConnection, endPoint.Protocols); + return TcpDriver.HandleAsync(this, endPoint, tcpConnection, _h2Options, endPoint.Protocols); }, QuicHandle = _quicEngine is not null diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index c41448417..afad0b3c6 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -81,6 +81,8 @@ public void WriteHead(StreamedResponseData head) public void WriteHead(int status) => writer.WriteHeaders(new Nghttp3Response { Status = status }); public ValueTask CompleteAsync() => writer.CompleteAsync(); + + public ValueTask ResetStreamAsync() => writer.ResetStreamAsync(); } } diff --git a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs index 612979758..5b2354f06 100644 --- a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs +++ b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs @@ -37,11 +37,25 @@ internal interface IStreamedWriter /// Ends the stream. ValueTask CompleteAsync(); + /// Resets the stream, so the client sees the response fail. + ValueTask ResetStreamAsync(); + } /// What the HTTP/2 and HTTP/3 drivers have in common: one stream through the handler chain. internal static class StreamedDriver { + private const string LoggerCategory = "GenHTTP.Engine.Ioxide.Protocol.Drivers.StreamedDriver"; + + // ioxide's OnHandlerFault, so what it still catches itself goes to the server's logging instead of + // stderr: a fault in the error path below, or the stack failing to finish a response once its + // handler has returned. + internal static Action ReportFaults(ILoggerFactory logging, string protocol) + { + var logger = logging.CreateLogger(LoggerCategory); + + return (_, e) => logger.LogError(e, "Failed to complete {Protocol} request", protocol); + } // A fault becomes a 500 while the head is still unwritten, and a reset once it is out. A client // that left mid-response is no fault at all. @@ -79,15 +93,15 @@ internal static async ValueTask DispatchAsync(IServer server, StreamedR } catch (Exception e) { - server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.StreamedDriver") + server.Logging.CreateLogger(LoggerCategory) .LogError(e, "Failed to handle {Protocol} request", protocol); if (headersWritten) { // Too late for a 500, and ending the stream would pass the truncated body off as - // whole. Rethrown, ioxide resets the stream (INTERNAL_ERROR on HTTP/2, - // H3_INTERNAL_ERROR on HTTP/3), so the client sees it fail. - throw; + // whole. Rethrown, ioxide would reset it as well, but report the fault a second time. + await writer.ResetStreamAsync(); + return; } if (!writer.IsCompleted) diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index 1489838d1..d266e9aad 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -19,8 +19,9 @@ internal static class Http2Driver // Runs one HTTP/2 connection, dispatching each stream as soon as its headers are in. Taking the // body reader is what makes ioxide stream the request body instead of assembling it first. - internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pipe, IPAddress? remoteAddress, bool secure) - => new Http2Connection(pipe) + internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pipe, Http2Options options, + IPAddress? remoteAddress, bool secure) + => new Http2Connection(pipe, options) .RunAsync((request, body, writer) => DispatchAsync(server, endPoint, request, body, writer, remoteAddress, secure)); // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. @@ -79,5 +80,7 @@ public void WriteHead(StreamedResponseData head) public void WriteHead(int status) => writer.WriteHeaders(new Http2Response { Status = status }); public ValueTask CompleteAsync() => writer.CompleteAsync(); + + public ValueTask ResetStreamAsync() => writer.ResetStreamAsync(); } } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs index 6c49702b8..ba923a20d 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs @@ -12,6 +12,7 @@ using GenHTTP.Engine.Ioxide.Infrastructure; using GenHTTP.Engine.Ioxide.Infrastructure.Endpoints; +using ioxide.http2; using ioxide.tls; using IoxideTls = ioxide.tls; @@ -30,7 +31,7 @@ internal static partial class TcpDriver private static readonly ReadOnlyMemory Preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"u8.ToArray(); // Takes an accepted connection through TLS if the port has it, then hands it to whichever protocol it turns out to speak. - internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoConnection conn, + internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoConnection conn, Http2Options http2Options, HttpProtocols httpProtocols = HttpProtocols.Http1) { IDuplexPipe pipe; @@ -76,7 +77,7 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon { try { - await Http2Driver.RunAsync(server, endPoint, pipe, remoteAddress, endPoint.Secure); + await Http2Driver.RunAsync(server, endPoint, pipe, http2Options, remoteAddress, endPoint.Secure); } finally { diff --git a/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs index 42c8388e1..5bd28abf9 100644 --- a/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Net; using System.Net.Quic; using System.Net.Security; @@ -11,6 +12,8 @@ using GenHTTP.Modules.ServerSentEvents; +using Microsoft.Extensions.Logging; + using Host = GenHTTP.Engine.Ioxide.Host; namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; @@ -48,6 +51,18 @@ public async Task TestFailureAfterTheHeadersResetsTheHttp3Stream() await AssertResponseFailsAsync(HttpVersion.Version30); } + [IoxideTestMethod] + public async Task TestFaultReachingIoxideIsLoggedOverHttp2() + { + await AssertEscapedFaultIsLoggedAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestFaultReachingIoxideIsLoggedOverHttp3() + { + await AssertEscapedFaultIsLoggedAsync(HttpVersion.Version30); + } + [IoxideTestMethod] public async Task TestFailureBeforeTheHeadersIsStillA500OverHttp2() { @@ -111,10 +126,51 @@ private static async Task AssertGeneratorEndsAsync(Version version) // The headers and the first part of the body are out when the content fails. The response has to // fail rather than end, since there is no content-length to tell the client the body is short. - // Where it fails depends on the client: a reset may overtake the headers it has not read yet. + // The fault goes to the server's logging, so ioxide must not print it to stderr as well. private static async Task AssertResponseFailsAsync(Version version) { - await using var server = await ServerAsync(new FailingMidBodyHandler(), version); + var stderr = await CaptureStandardErrorAsync(() => AssertTransferFailsAsync(version)); + + Assert.DoesNotContain(FailingMidBodyContent.Failure, stderr); + } + + // The engine's own report of the fault fails too, as the log sink refuses it, so the fault escapes + // to ioxide. ioxide hands it back to the server's logging rather than printing it to stderr. + private static async Task AssertEscapedFaultIsLoggedAsync(Version version) + { + var logging = new FailingSinkLoggerFactory(); + + var stderr = await CaptureStandardErrorAsync(() => AssertTransferFailsAsync(version, logging)); + + var protocol = version == HttpVersion.Version30 ? "HTTP/3" : "HTTP/2"; + + Assert.Contains($"Failed to complete {protocol} request", logging.Errors); + Assert.DoesNotContain(FailingSinkLoggerFactory.Failure, stderr); + } + + private static async Task CaptureStandardErrorAsync(Func action) + { + var stderr = new StringWriter(); + var original = Console.Error; + + Console.SetError(stderr); + + try + { + await action(); + } + finally + { + Console.SetError(original); + } + + return stderr.ToString(); + } + + // Where the response fails depends on the client: a reset may overtake the headers it has not read yet. + private static async Task AssertTransferFailsAsync(Version version, ILoggerFactory? logging = null) + { + await using var server = await ServerAsync(new FailingMidBodyHandler(), version, logging); using var client = server.CreateClient(); @@ -133,7 +189,7 @@ private static async Task AssertResponseFailsAsync(Version version) Assert.IsTrue(failure is HttpRequestException or IOException, $"The response did not fail as a transfer: {failure}"); } - private static async Task ServerAsync(IHandler handler, Version version) + private static async Task ServerAsync(IHandler handler, Version version, ILoggerFactory? logging = null) { var port = (ushort)TestHost.NextPort(); @@ -157,6 +213,11 @@ private static async Task ServerAsync(IHandler handler, Version v .Bind(IPAddress.Loopback, port, HttpProtocols.Http2); } + if (logging != null) + { + host = host.Logging(logging); + } + await host.StartAsync(); return new RunningServer(host, port, version); @@ -199,6 +260,8 @@ private sealed class FailingMidBodyHandler : IHandler /// Writes and flushes part of an unknown-length body, then fails. private sealed class FailingMidBodyContent : IResponseContent { + public const string Failure = "The content failed after its headers were sent"; + public ulong? Length => null; public ContentType? Type => ContentType.ApplicationOctetStream; @@ -212,7 +275,45 @@ public async ValueTask WriteAsync(IResponseSink sink) await sink.Stream.WriteAsync(new byte[1024]); await sink.Stream.FlushAsync(); - throw new InvalidOperationException("The content failed after its headers were sent"); + throw new InvalidOperationException(Failure); + } + } + + /// A log sink that refuses the first error it is given, as one whose disk is full would. + private sealed class FailingSinkLoggerFactory : ILoggerFactory + { + public const string Failure = "The log sink refused the entry"; + + private int _refused; + + public ConcurrentQueue Errors { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(this); + + public void AddProvider(ILoggerProvider provider) { } + + public void Dispose() { } + + private sealed class Logger(FailingSinkLoggerFactory factory) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (logLevel < LogLevel.Error) + { + return; + } + + if (Interlocked.Exchange(ref factory._refused, 1) == 0) + { + throw new IOException(Failure); + } + + factory.Errors.Enqueue(formatter(state, exception)); + } } } From db85b66fe6f22d63d66b2388f157aad69f0e25aa Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 16 Nov 2026 17:48:27 +0000 Subject: [PATCH 18/18] fix(engine): HTTP/2 and HTTP/3 uploads without a content-length reach the handler A stream counted as having a body only when it carried a content-length, which neither protocol requires. A body of unknown length, as HttpClient streams one it cannot measure and fetch sends one from a ReadableStream, arrives without it, so the handler was given no body at all. StreamedRequest now takes the request body flag ioxide 0.27 reads off the HEADERS, and still counts a content-length, so an empty body announced by one stays a body, as it is over HTTP/1.1. StreamedRequestBodyTests.TestBodyOfUnknownLengthIsRead echoes a body sent without a content-length over HTTP/2, and Http3Tests.TestHttp3StreamsTheRequestBody now sends its body that way. Both fail without the change. --- .../Protocol/Drivers/Quic/Http3Driver.cs | 2 +- .../Protocol/Drivers/Tcp/Http2Driver.cs | 2 +- .../Protocol/Requests/StreamedRequest.cs | 12 +++++------ .../Acceptance/Engine/Ioxide/Http3Tests.cs | 5 +++-- .../Engine/Ioxide/StreamedRequestBodyTests.cs | 12 +++++++++-- .../Engine/Ioxide/UnknownLengthContent.cs | 21 +++++++++++++++++++ 6 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index afad0b3c6..20a7cb55c 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -44,7 +44,7 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, // No remote address: ioxide's QuicConnection exposes none, and a QUIC peer may migrate // mid-connection anyway. mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - body.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); + request.HasBody, body.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/3"); } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index d266e9aad..fe1472567 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -43,7 +43,7 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, } mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - body.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); + request.HasBody, body.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/2"); } diff --git a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs index bdb97b31e..f3b792c07 100644 --- a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs +++ b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs @@ -87,7 +87,7 @@ internal static void Return(StreamedRequest request) // Points this request at one stream, once its headers are in. internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory method, ReadOnlyMemory path, - ReadOnlyMemory authority, Func>> read, IPAddress? remoteAddress, + ReadOnlyMemory authority, bool hasBody, Func>> read, IPAddress? remoteAddress, HttpProtocol protocol, bool secure) { _server = server; @@ -97,10 +97,10 @@ internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory met _header.Apply(method, path, authority, protocol); - // A stream carries a body only when it declares one: HTTP/2 and HTTP/3 forbid Transfer-Encoding, - // so Content-Length is the signal - and its absence means no body, matching what the other engines - // report for a request that carries neither header. - _hasBody = CheckForBody(_headerEntries); + // The protocol sees a body that comes without a Content-Length, as a streamed upload does. A + // Content-Length still counts on its own, so an empty body announced by one is a body, as it is + // over HTTP/1.1, even when the headers already ended the stream. + _hasBody = hasBody || CheckForBody(_headerEntries); if (_hasBody) { @@ -251,7 +251,7 @@ private static void ParseQuery(ReadOnlyMemory path, List<(ReadOnlyMemory Name, ReadOnlyMemory Value)> headers) { foreach (var (name, _) in headers) diff --git a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs index 123e21c95..822cd5075 100644 --- a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs +++ b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs @@ -171,7 +171,8 @@ public async Task TestHttp3SharesItsPortNumberWithTcp() public async Task TestHttp3StreamsTheRequestBody() { // Large enough to arrive in many chunks, which the handler reads through the body reader - // ioxide hands the driver - an echo that comes back whole proves none went missing. + // ioxide hands the driver - an echo that comes back whole proves none went missing. Sent + // without a content-length, as a client streams an upload, so the handler must still see it. if (!QuicConnection.IsSupported) { Assert.Inconclusive("QUIC is not supported on this machine (msquic missing)."); @@ -223,7 +224,7 @@ public async Task TestHttp3StreamsTheRequestBody() var payload = new byte[256 * 1024]; new Random(42).NextBytes(payload); - using var content = new ByteArrayContent(payload); + using var content = new UnknownLengthContent(payload); using var response = await client.PostAsync($"https://localhost:{port}/", content); diff --git a/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs b/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs index a9a3f99a9..5d05ae301 100644 --- a/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs @@ -50,6 +50,14 @@ public async Task TestBodyIsReadThroughTheArrayOverload() await RunEchoAsync(ReadThroughArrayOverload, RandomBytes(128 * 1024)); } + [IoxideTestMethod] + public async Task TestBodyOfUnknownLengthIsRead() + { + // No content-length, as HttpClient streams a body it cannot measure and fetch sends one from a + // ReadableStream: the handler still gets the body. + await RunEchoAsync(ReadWhole, RandomBytes(64 * 1024), announceLength: false); + } + #region Read strategies // Reads the body whole, exercising AsMemoryAsync and its chunk-assembly loop. @@ -95,7 +103,7 @@ private static async ValueTask ReadThroughArrayOverload(IRequestBody bod #region Infrastructure - private static async Task RunEchoAsync(Func> read, byte[] payload) + private static async Task RunEchoAsync(Func> read, byte[] payload, bool announceLength = true) { var port = (ushort)TestHost.NextPort(); @@ -116,7 +124,7 @@ private static async Task RunEchoAsync(Func> rea Timeout = TimeSpan.FromSeconds(10), }; - using var content = new ByteArrayContent(payload); + using HttpContent content = announceLength ? new ByteArrayContent(payload) : new UnknownLengthContent(payload); using var response = await client.PostAsync($"http://localhost:{port}/", content); diff --git a/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs b/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs new file mode 100644 index 000000000..f0a7b1eef --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs @@ -0,0 +1,21 @@ +using System.Net; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A request body sent without a content-length, the way a client streams one it cannot measure up +/// front: over HTTP/2 and HTTP/3, only the end of the stream says where it stops. +/// +internal sealed class UnknownLengthContent(byte[] payload) : HttpContent +{ + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => stream.WriteAsync(payload).AsTask(); + + protected override bool TryComputeLength(out long length) + { + length = -1; + return false; + } + +}