diff --git a/API/Ioxide/GenHTTP.Api.Ioxide.csproj b/API/Ioxide/GenHTTP.Api.Ioxide.csproj index 19393b847..4b9de4162 100644 --- a/API/Ioxide/GenHTTP.Api.Ioxide.csproj +++ b/API/Ioxide/GenHTTP.Api.Ioxide.csproj @@ -9,7 +9,7 @@ - + diff --git a/API/Ioxide/IIoxideFileSink.cs b/API/Ioxide/IIoxideFileSink.cs new file mode 100644 index 000000000..033c8c6f4 --- /dev/null +++ b/API/Ioxide/IIoxideFileSink.cs @@ -0,0 +1,21 @@ +namespace GenHTTP.Api.Ioxide; + +/// +/// A response sink that appends part of a file to the body itself, reading it off the ring straight into the +/// connection's write buffer, after the headers already there. +/// +/// +/// Offered where that buffer is what the response is written to: HTTP/1.1 in cleartext, or with kernel TLS, where +/// the kernel encrypts on send. Content that serves a file checks the sink for this first, and copies the file +/// through a buffer of its own otherwise. +/// +public interface IIoxideFileSink +{ + + /// + /// Appends bytes of the file , from , to the + /// body. + /// + ValueTask WriteFileAsync(int fd, long offset, long length); + +} diff --git a/Engine/Ioxide/EngineOptions.cs b/Engine/Ioxide/EngineOptions.cs index b9bc4ca61..a43eeca8c 100644 --- a/Engine/Ioxide/EngineOptions.cs +++ b/Engine/Ioxide/EngineOptions.cs @@ -39,6 +39,15 @@ public sealed record TcpTransportOptions public string? CipherList { get; init; } + /// + /// Hands each TLS connection's sending to the kernel after its handshake (kTLS), so a response is + /// encrypted on send, and a static file served over HTTP/1.1 is read straight into the connection's + /// write buffer. + /// + /// + /// Needs the kernel's tls module: without it the server's start fails, rather than every + /// connection after its handshake. + /// public bool TxKernelTls { get; init; } public bool RxKernelTls { get; init; } @@ -56,33 +65,48 @@ public sealed record TcpTransportOptions public int RecvQueueEntries { get; init; } = 64; /// - /// Close a connection that has neither received nor sent anything for this long. 0 (the - /// default) disables it. + /// Close a connection that has waited this long on its client: a read parked, nothing + /// buffered and nothing being sent. 0 (the default) disables it. /// /// + /// The clock only runs while the server waits on the client. Bytes in either direction + /// restart it, a send in flight holds it, and the time a handler spends producing a response + /// does not count - so a slow handler keeps its connection, while an idle keep-alive + /// connection or a silent websocket does not. HTTP/2 is the exception: its frame loop keeps a + /// read parked while handlers work, so a response slower than this, to a client that sends + /// nothing in the meantime, is cut. + /// /// Off by default because this listener also carries upgraded connections, and an idle /// websocket is a healthy one. ioxide's own default is 60s, which is right for HTTP keep-alive /// and wrong for a websocket whose ping interval is longer than that - the transport cannot /// tell the two apart, so the framework does not guess on your behalf. /// /// Turn it on if you serve no upgrades, or set it comfortably above your websocket ping - /// interval so pings keep the connection marked alive. + /// interval - a ping from either side restarts the clock. /// - public int IdleTimeoutMs { get; init; } + public int ReadTimeoutMs { get; init; } /// - /// Close a connection whose response has been outstanding for this long. 0 (the default) - /// disables it. + /// Close a connection whose send has been in flight for this long. 0 (the default) disables it. /// /// /// This is what bounds a client that stops reading: its window shuts, the send never completes, /// and without a clock the connection is pinned for as long as the peer cares to hold it. TCP - /// will not end it on its own. + /// will not end it on its own, and neither will : a client that + /// keeps sending restarts that clock while its own window stays shut. A websocket written from + /// a background task is the case that needs this one. /// - /// Off by default because it is a deadline for the WHOLE response, not for progress within it, - /// so a large download to a slow client counts against it. Size it against your slowest - /// legitimate response rather than against a stall - for an API that serves small payloads, - /// something like 30s is a straightforward win. + /// Off by default because it is a deadline for a whole flush, not for progress within it, and + /// content that does not flush as it writes goes out as one flush - so a large download to a + /// slow client counts against it. Size it against your slowest legitimate response rather than + /// against a stall - for an API that serves small payloads, something like 30s is a + /// straightforward win. + /// + /// The listener also carries it as TCP_USER_TIMEOUT, so the kernel aborts a connection whose + /// client left sent data unacknowledged, or kept its window shut with data waiting, for that long. + /// That covers a client that stopped reading or vanished behind a stream of small flushes, which + /// each complete into the send buffer. A client whose network drops for longer than this, with data + /// in flight, is cut too. /// public int SendTimeoutMs { get; init; } } @@ -92,7 +116,15 @@ public sealed record QuicTransportOptions { public int HandshakeTimeoutMs { get; init; } = 10_000; - public int IdleTimeoutMs { get; init; } = 60_000; + /// + /// Close a connection whose client has sent nothing for this long. 0 disables it. + /// + /// + /// While a response is still owed, the engine keeps the client answering with keep-alive + /// PINGs, so a slow handler is not timed out - only a client that went quiet with nothing + /// pending is. + /// + public int ReadTimeoutMs { get; init; } = 60_000; public QuicRouting Routing { get; init; } = QuicRouting.Forward; diff --git a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj index df36c082a..b6c9a7f08 100644 --- a/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj +++ b/Engine/Ioxide/GenHTTP.Engine.Ioxide.csproj @@ -16,11 +16,11 @@ - + - - - + + + diff --git a/Engine/Ioxide/Infrastructure/Server.Certificates.cs b/Engine/Ioxide/Infrastructure/Server.Certificates.cs index 0afc25dc2..f139fddb4 100644 --- a/Engine/Ioxide/Infrastructure/Server.Certificates.cs +++ b/Engine/Ioxide/Infrastructure/Server.Certificates.cs @@ -89,43 +89,39 @@ private List ResolveRotations(TcpTlsRegistry? started, out return rotations; } - // Installs the resolved sets on every reactor, collecting failures rather than stopping half way. + // Installs the resolved sets, collecting failures rather than stopping half way. Every reactor was started with + // each port's one TlsOptions, so its reactors share the port's contexts, and rotating them through one service + // rotates them all: another call per reactor would only build the same set again. private void Publish(TcpTlsRegistry?[] registries, List rotations) { List? failures = null; - for (var reactor = 0; reactor < registries.Length; reactor++) + var registry = Array.Find(registries, static r => r is not null); + + foreach (var rotation in rotations) { - if (registries[reactor] is not { } registry) + if (registry is null || !registry.TryFor(rotation.Port, out var service)) { continue; } - foreach (var rotation in rotations) + try + { + service.ReplaceCertificates(rotation.Default, rotation.ByHost); + } + catch (Exception e) { - if (!registry.TryFor(rotation.Port, out var service)) - { - continue; - } - - try - { - service.ReplaceCertificates(rotation.Default, rotation.ByHost); - } - catch (Exception e) - { - _logger.LogError(e, "Reactor {Reactor} kept the previous certificate on port {Port}", reactor, rotation.Port); - - (failures ??= []).Add(e); - } + _logger.LogError(e, "Port {Port} kept its previous certificate", rotation.Port); + + (failures ??= []).Add(e); } } if (failures is not null) { throw new InvalidOperationException( - $"{failures.Count} of the server's TLS services refused the new certificates and still serve the previous ones; " - + "the rest were replaced. See the inner exceptions, and the errors logged against each reactor and port.", + $"{failures.Count} of the server's secure ports refused the new certificates and still serve the previous ones; " + + "the rest were replaced. See the inner exceptions, and the errors logged against each port.", new AggregateException(failures)); } } diff --git a/Engine/Ioxide/Infrastructure/Server.Quic.cs b/Engine/Ioxide/Infrastructure/Server.Quic.cs index 072a78968..56c63cd73 100644 --- a/Engine/Ioxide/Infrastructure/Server.Quic.cs +++ b/Engine/Ioxide/Infrastructure/Server.Quic.cs @@ -1,5 +1,6 @@ using GenHTTP.Api.Infrastructure; using GenHTTP.Engine.Ioxide.Infrastructure.Endpoints; +using GenHTTP.Engine.Ioxide.Protocol.Drivers; using ioxide; using ioxide.nghttp3; @@ -72,6 +73,7 @@ private ServerConfig WithQuic(ServerConfig serverConfig) { QpackDynamicTableCapacity = _engineOptions.Http3.QpackDynamicTableCapacity, QpackBlockedStreams = _engineOptions.Http3.QpackBlockedStreams, + OnHandlerFault = StreamedDriver.ReportFaults(Logging, "HTTP/3"), }; return serverConfig with @@ -85,7 +87,7 @@ private ServerConfig WithQuic(ServerConfig serverConfig) Port = quicEndPoint.Port, ConnectionFactory = _quicEngine.CreateFactory(), - IdleTimeoutMs = _engineOptions.Quic.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Quic.ReadTimeoutMs, Routing = _engineOptions.Quic.Routing, PinMigratedPeers = _engineOptions.Quic.PinMigratedPeers, }, diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs index 1eacb5021..f20b53652 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.Tls.cs @@ -11,7 +11,7 @@ namespace GenHTTP.Engine.Ioxide.Infrastructure; -/// TLS termination on the TCP transports, in OpenSSL. +/// TLS termination on the TCP transports, in BoringSSL. public sealed partial class Server { // The TLS options for every secure TCP port that can produce a certificate; the rest refuse their handshakes. @@ -119,7 +119,7 @@ private void StartTlsServices(Reactor reactor, int index, IReadOnlyList 0 ? byHost : null; } - // The TLS floor for one endpoint, widening rather than throwing where SslProtocols asks for something OpenSSL cannot express. + // The TLS floor for one endpoint, widening rather than throwing where SslProtocols asks for something BoringSSL cannot express. private TlsProtocolVersion ResolveMinProtocolVersion(SecureEndPoint endPoint) { var protocols = endPoint.SecurityConfiguration.Protocols; @@ -130,7 +130,7 @@ private TlsProtocolVersion ResolveMinProtocolVersion(SecureEndPoint endPoint) if ((protocols & ~(SslProtocols.Tls12 | SslProtocols.Tls13)) != 0) { _logger.LogWarning( - "Port {Port} was bound asking for a TLS version below 1.2. This engine terminates TLS in OpenSSL, whose floor is 1.2, " + "Port {Port} was bound asking for a TLS version below 1.2. This engine terminates TLS in BoringSSL, whose floor is 1.2, " + "so those versions are not served and never were; the endpoint serves {Served}.", endPoint.Port, tls13 && !tls12 ? "TLS 1.3" : "TLS 1.2 and above"); } @@ -164,7 +164,7 @@ private void WarnAboutValidatorGaps() if (validator.RevocationCheck != X509RevocationMode.NoCheck) { _logger.LogWarning( - "The validator on port {Port} asks for {Mode} revocation checking, which this engine does not perform - neither OpenSSL " + "The validator on port {Port} asks for {Mode} revocation checking, which this engine does not perform - neither BoringSSL " + "nor ngtcp2 is given a CRL or an OCSP responder here, so a revoked client certificate is accepted until it expires. " + "Use short-lived certificates, or check the peer in ICertificateValidator.Validate against your own source of truth.", endPoint.Port, validator.RevocationCheck); @@ -173,9 +173,8 @@ private void WarnAboutValidatorGaps() if (endPoint.Protocols.HasFlag(HttpProtocols.Http3)) { _logger.LogWarning( - "Port {Port} serves HTTP/3 with a certificate validator. Validate runs on the TCP transports, where OpenSSL hands over the " - + "peer certificate, but NOT over QUIC: ngtcp2 exposes only the peer's subject and common name, so there is no certificate " - + "to hand it. An HTTP/3 client is admitted on the chain check and RequireCertificate alone. Keep the two transports on " + "Port {Port} serves HTTP/3 with a certificate validator. Validate runs on the TCP transports, but NOT over QUIC: " + + "an HTTP/3 client is admitted on the chain check and RequireCertificate alone. Keep the two transports on " + "separate ports if that difference matters.", endPoint.Port); } diff --git a/Engine/Ioxide/Infrastructure/Server.Tcp.cs b/Engine/Ioxide/Infrastructure/Server.Tcp.cs index 5666c37cf..aced96ba8 100644 --- a/Engine/Ioxide/Infrastructure/Server.Tcp.cs +++ b/Engine/Ioxide/Infrastructure/Server.Tcp.cs @@ -1,5 +1,7 @@ using GenHTTP.Api.Infrastructure; +using GenHTTP.Engine.Ioxide.Protocol.Drivers; using ioxide; +using ioxide.http2; namespace GenHTTP.Engine.Ioxide.Infrastructure; @@ -7,7 +9,13 @@ namespace GenHTTP.Engine.Ioxide.Infrastructure; public sealed partial class Server { private readonly ushort[] _tcpPorts; - + + // Shared by every HTTP/2 connection, so the fault hook is built once. + private readonly Http2Options _h2Options; + + private Http2Options CreateHttp2Options() + => new() { OnHandlerFault = StreamedDriver.ReportFaults(Logging, "HTTP/2") }; + // The ports serving HTTP/1.1 or HTTP/2, primary first. Empty on an HTTP/3-only server. private ushort[] ResolveTcpPorts() => _endPoints.Where(e => (e.Protocols & HttpProtocols.Http1AndHttp2) != 0) @@ -32,7 +40,7 @@ private ServerConfig WithTcp(ServerConfig serverConfig) => serverConfig with // Set explicitly, because ioxide defaults both to 60s and this listener carries // upgraded connections too - inheriting that would start closing idle websockets. - IdleTimeoutMs = _engineOptions.Tcp.IdleTimeoutMs, + ReadTimeoutMs = _engineOptions.Tcp.ReadTimeoutMs, SendTimeoutMs = _engineOptions.Tcp.SendTimeoutMs, }, }; diff --git a/Engine/Ioxide/Infrastructure/Server.cs b/Engine/Ioxide/Infrastructure/Server.cs index b938bec8b..aac2d6992 100644 --- a/Engine/Ioxide/Infrastructure/Server.cs +++ b/Engine/Ioxide/Infrastructure/Server.cs @@ -73,6 +73,7 @@ internal Server( _dualStack = _endPoints[0].DualStack; _tcpPorts = ResolveTcpPorts(); + _h2Options = CreateHttp2Options(); _quicEndPoint = ResolveQuicEndPoint(); EndPoints = new EndPointCollection(_endPoints); @@ -80,7 +81,8 @@ internal Server( #endregion - // Starts a reactor per thread and waits until every one of them is listening. + // Starts a reactor per thread and waits until every one of them is listening. A reactor that + // cannot start fails the start with its reason, once the others are stopped again. public async ValueTask StartAsync() { await PrepareHandlerAsync(); @@ -95,18 +97,20 @@ public async ValueTask StartAsync() WarnAboutValidatorGaps(); - _threads = new Thread[serverConfig.ReactorCount]; - _reactors = new Reactor[serverConfig.ReactorCount]; + var reactors = new Reactor[serverConfig.ReactorCount]; + _tcpTlsRegistries = secureTcp ? new TcpTlsRegistry?[serverConfig.ReactorCount] : null; - CountdownEvent listening = new CountdownEvent(serverConfig.ReactorCount); + var startup = new ReactorStartup(serverConfig.ReactorCount); - for (var i = 0; i < _threads.Length; i++) + // Every reactor is built before any runs: the constructor refuses a configuration ioxide + // cannot run with, and that should happen before a single thread has started. + for (var i = 0; i < reactors.Length; i++) { // OnStart runs on the reactor's thread, long after the loop variable has moved on. int capturedIndex = i; - Reactor reactor = new Reactor(i, serverConfig) + reactors[i] = new Reactor(i, serverConfig) { OnStart = _reactor_ => { @@ -119,23 +123,51 @@ public async ValueTask StartAsync() _onReactorStart?.Invoke(_reactor_); - listening.Signal(); + startup.Started(capturedIndex); + }, + // Set so a fault reaches the host rather than ending the process from the reactor's + // thread - a kernel refusing the ring, a port already taken, a throw from OnStart. + OnFault = (_, e) => + { + if (startup.TryFail(capturedIndex, e)) + { + return; + } + + if (startup.WasStarted(capturedIndex)) + { + _logger.LogCritical(e, "Reactor {Reactor} stopped after a fault and accepts no more connections; those it served were closed", capturedIndex); + } + else if (startup.Failed < 0) + { + // The start was given up on, so nothing else reports this one. + _logger.LogError(e, "Reactor {Reactor} could not start; the server runs without it", capturedIndex); + } + else if (_logger.IsEnabled(LogLevel.Debug)) + { + // The start fails with another reactor's fault already, most likely this same one. + _logger.LogDebug(e, "Reactor {Reactor} could not start either", capturedIndex); + } }, TcpHandle = (_, tcpConnection) => { EndPoint endPoint = EndPointFor(tcpConnection.ListenerPort); - return TcpDriver.HandleAsync(this, endPoint, tcpConnection, endPoint.Protocols); + return TcpDriver.HandleAsync(this, endPoint, tcpConnection, _h2Options, endPoint.Protocols); }, QuicHandle = _quicEngine is not null ? (_, quicConnection) => Http3Driver.RunAsync(this, _quicEndPoint!, quicConnection, _h3Options!) : null }; + } - _reactors[i] = reactor; + _reactors = reactors; + _threads = new Thread[reactors.Length]; - _threads[i] = new Thread(reactor.Run) + for (var i = 0; i < reactors.Length; i++) + { + _threads[i] = new Thread(reactors[i].Run) { Name = $"ioxide-genhttp-{i}", IsBackground = true, @@ -144,14 +176,23 @@ public async ValueTask StartAsync() _threads[i].Start(); } - if (await Task.Run(() => listening.Wait(TimeSpan.FromSeconds(10)))) + try { - listening.Dispose(); + await startup.Listening.WaitAsync(TimeSpan.FromSeconds(10)); } - else + catch (TimeoutException) { + startup.GiveUp(); + _logger.LogWarning("Not all reactors reported listening within 10s; the server may not be fully accepting yet."); } + catch (Exception e) + { + // Rather than a server serving on fewer reactors than it was asked for. + await DisposeAsync(); + + throw new InvalidOperationException($"The ioxide engine failed to start reactor {startup.Failed}: {e.Message}", e); + } if (_logger.IsEnabled(LogLevel.Information)) { @@ -209,8 +250,8 @@ private ServerConfig BuildServerConfig() DualStack = _dualStack, - // Explicitly null: ioxide's own default is a live listener on 8080, which an - // HTTP/3-only server would otherwise inherit. + // No TCP listener unless a binding asks for one; WithTcp adds it. An HTTP/3-only + // server binds none. Tcp = null, }; @@ -308,6 +349,8 @@ public async ValueTask DisposeAsync() if (reactors is null || threads is null) { + // A start refused before any reactor ran may still have built the QUIC engine. + DisposeQuic(); return; } @@ -333,3 +376,51 @@ await Task.Run(() => _logger.LogInformation("Stopped ioxide reactors"); } } + +/// One start of the reactors: done once every one is listening, failed by the first that cannot. +internal sealed class ReactorStartup(int reactorCount) +{ + private readonly TaskCompletionSource _listening = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private readonly bool[] _started = new bool[reactorCount]; + + private int _pending = reactorCount; + + private int _failed = -1; + + // Completes once every reactor is listening, or faults with the first that could not start. + public Task Listening => _listening.Task; + + // The reactor whose fault failed the start, or -1. + public int Failed => Volatile.Read(ref _failed); + + // A reactor got through OnStart, so it is listening. + public void Started(int reactor) + { + Volatile.Write(ref _started[reactor], true); + + if (Interlocked.Decrement(ref _pending) == 0) + { + _listening.TrySetResult(); + } + } + + // Fails the start with a fault that ended a reactor before it was listening. False for a reactor + // that was serving already, or once the start has failed or been given up on - that fault is + // left to the host to report. + public bool TryFail(int reactor, Exception e) + { + if (WasStarted(reactor) || _listening.Task.IsCompleted || Interlocked.CompareExchange(ref _failed, reactor, -1) != -1) + { + return false; + } + + return _listening.TrySetException(e); + } + + // Whether the reactor got through OnStart before it faulted. + public bool WasStarted(int reactor) => Volatile.Read(ref _started[reactor]); + + // Stops waiting: a reactor failing from here on no longer fails a start nobody waits for. + public void GiveUp() => _listening.TrySetCanceled(); +} diff --git a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs index 39b248e79..20a7cb55c 100644 --- a/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Quic/Http3Driver.cs @@ -1,10 +1,12 @@ +using System.Buffers; +using System.IO.Pipelines; + using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; using ioxide; using ioxide.nghttp3; -using Microsoft.Extensions.Logging; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Responses; @@ -15,30 +17,22 @@ internal static class Http3Driver { private static readonly ReadOnlyMemory Head = "HEAD"u8.ToArray(); - // Runs one QUIC connection as HTTP/3, dispatching each stream as it opens. + // Runs one QUIC connection as HTTP/3, dispatching each stream as soon as its headers are in. Taking + // the body reader is what makes ioxide stream the request body instead of assembling it first. internal static Task RunAsync(IServer server, IEndPoint endPoint, QuicConnection connection, Nghttp3Options options) => new Nghttp3Connection(connection, options) - .RunStreamedResponseAsync((request, writer) => DispatchAsync(server, endPoint, request, writer)); + .RunAsync((request, body, writer) => DispatchAsync(server, endPoint, request, body, writer)); - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream. + // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Nghttp3Request request, - Nghttp3ResponseWriter writer) + Nghttp3BodyReader body, Nghttp3ResponseWriter writer) { StreamedRequest? mapped = null; - // Whether the head is already on the wire. IsCompleted cannot answer this - it means - // END_STREAM has been sent - so a fault AFTER the headers went out used to take the error - // path anyway, where WriteHeaders throws "headers have already been written". That second - // exception replaced the first, so the only thing ever reported was the masking error and - // the real fault was never logged at all. - var headersWritten = false; - try { var headRequest = request.Method.Span.SequenceEqual(Head.Span); - var reader = request.BodyReader; - mapped = StreamedRequest.Rent(); for (var i = 0; i < request.Headers.Count; i++) @@ -50,44 +44,45 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, // No remote address: ioxide's QuicConnection exposes none, and a QUIC peer may migrate // mid-connection anyway. mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - reader is null ? null : reader.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); - - var response = await server.Handler.HandleAsync(mapped) - ?? throw new InvalidOperationException("The root request handler did not return a response"); - - var data = StreamedResponder.BuildHeaders(response, mapped); - - var head = new Nghttp3Response { Status = data.Status }; - - foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in data.Headers) - { - head.Headers.Add(name, value); - } + request.HasBody, body.ReadAsync, remoteAddress: null, HttpProtocol.Http3, secure: true); - writer.WriteHeaders(head); - headersWritten = true; - - await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/3"); } - catch (Exception e) + finally { - server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Quic.Http3Driver") - .LogError(e, "Failed to handle HTTP/3 request"); - - if (!headersWritten && !writer.IsCompleted) + if (mapped is not null) { - writer.WriteHeaders(new Nghttp3Response { Status = 500 }); + StreamedRequest.Return(mapped); } } - finally + } + + /// ioxide's HTTP/3 response writer, as the shared dispatch drives it. + private readonly struct Writer(Nghttp3ResponseWriter writer) : IStreamedWriter + { + public IBufferWriter Body => writer; + + public Func> Flush => writer.FlushAsync; + + public bool IsCompleted => writer.IsCompleted; + + public void WriteHead(StreamedResponseData head) { - if (mapped is not null) + var response = new Nghttp3Response { Status = head.Status }; + + foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in head.Headers) { - StreamedRequest.Return(mapped); + response.Headers.Add(name, value); } - await writer.CompleteAsync(); + writer.WriteHeaders(response); } + + public void WriteHead(int status) => writer.WriteHeaders(new Nghttp3Response { Status = status }); + + public ValueTask CompleteAsync() => writer.CompleteAsync(); + + public ValueTask ResetStreamAsync() => writer.ResetStreamAsync(); } - + } diff --git a/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs new file mode 100644 index 000000000..5b2354f06 --- /dev/null +++ b/Engine/Ioxide/Protocol/Drivers/StreamedDriver.cs @@ -0,0 +1,116 @@ +using System.Buffers; +using System.IO.Pipelines; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide.Protocol.Requests; +using GenHTTP.Engine.Ioxide.Protocol.Responses; + +using ioxide; + +using Microsoft.Extensions.Logging; + +namespace GenHTTP.Engine.Ioxide.Protocol.Drivers; + +/// +/// ioxide's HTTP/2 or HTTP/3 response writer, as drives it. The two share +/// no type of their own, so each driver wraps its writer in one of these. +/// +internal interface IStreamedWriter +{ + + /// The protocol's frame buffer, which the body is written into. + IBufferWriter Body { get; } + + /// Pushes the staged frames out, and reports a peer that has gone. + Func> Flush { get; } + + /// Whether the stream has been ended already. + bool IsCompleted { get; } + + /// Writes the response head. + void WriteHead(StreamedResponseData head); + + /// Writes a head that carries a status and nothing else. + void WriteHead(int status); + + /// Ends the stream. + ValueTask CompleteAsync(); + + /// Resets the stream, so the client sees the response fail. + ValueTask ResetStreamAsync(); + +} + +/// What the HTTP/2 and HTTP/3 drivers have in common: one stream through the handler chain. +internal static class StreamedDriver +{ + private const string LoggerCategory = "GenHTTP.Engine.Ioxide.Protocol.Drivers.StreamedDriver"; + + // ioxide's OnHandlerFault, so what it still catches itself goes to the server's logging instead of + // stderr: a fault in the error path below, or the stack failing to finish a response once its + // handler has returned. + internal static Action ReportFaults(ILoggerFactory logging, string protocol) + { + var logger = logging.CreateLogger(LoggerCategory); + + return (_, e) => logger.LogError(e, "Failed to complete {Protocol} request", protocol); + } + + // A fault becomes a 500 while the head is still unwritten, and a reset once it is out. A client + // that left mid-response is no fault at all. + internal static async ValueTask DispatchAsync(IServer server, StreamedRequest request, TWriter writer, + bool headRequest, string protocol) where TWriter : IStreamedWriter + { + // Whether the head is already on the wire. IsCompleted cannot answer this - it means the + // stream has ended - so a fault AFTER the headers went out used to take the error path + // anyway, where writing a second head throws. That second exception replaced the first, so + // the only thing ever reported was the masking error and the real fault was never logged. + var headersWritten = false; + + // Taken here, on the reactor: the handler may come back on another thread. + var reactor = Reactor.Current; + + try + { + var response = await server.Handler.HandleAsync(request) + ?? throw new InvalidOperationException("The root request handler did not return a response"); + + // The response, with the thread-statics it reads, is written on the reactor. + await new ReactorSwitch(reactor); + + writer.WriteHead(StreamedResponder.BuildHeaders(response, request)); + headersWritten = true; + + await StreamedResponder.WriteBodyAsync(response, writer.Body, writer.Flush, headRequest); + + await writer.CompleteAsync(); + } + catch (ConnectionClosedException) + { + // The client reset or stopped the stream, or went away mid-response: nobody is left to + // answer, and that is no fault. + } + catch (Exception e) + { + server.Logging.CreateLogger(LoggerCategory) + .LogError(e, "Failed to handle {Protocol} request", protocol); + + if (headersWritten) + { + // Too late for a 500, and ending the stream would pass the truncated body off as + // whole. Rethrown, ioxide would reset it as well, but report the fault a second time. + await writer.ResetStreamAsync(); + return; + } + + if (!writer.IsCompleted) + { + writer.WriteHead(500); + } + + await writer.CompleteAsync(); + } + } + +} diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs index 065cc6fc3..49679058b 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http1Driver.cs @@ -16,6 +16,8 @@ using Connection = GenHTTP.Api.Protocol.Connection; using IoConnection = ioxide.TcpConnection; +using IoReactor = ioxide.Reactor; +using ReactorSwitch = ioxide.ReactorSwitch; using GenHTTP.Engine.Ioxide.Protocol.Responses; using StringContent = GenHTTP.Modules.IO.Strings.StringContent; @@ -40,11 +42,17 @@ internal static async Task RunAsync(IServer server, IEndPoint endPoint, IDuplexP var reader = pipe.Input; var writer = pipe.Output; + // In cleartext, or with kernel TLS, the writer is the connection's own write buffer, which a file can be read into. + var fileConnection = writer is ioxide.TcpConnectionPipeWriter ? conn : null; + var request = RentRequest(); var into = request.Source; var reactorThreadId = Environment.CurrentManagedThreadId; + // Taken here, on the reactor: the handler may come back on another thread. + var reactor = IoReactor.Current; + try { var dataRemaining = false; @@ -74,7 +82,7 @@ internal static async Task RunAsync(IServer server, IEndPoint endPoint, IDuplexP request.Apply(server, endPoint, reader, buffer.Start, remoteAddress, null, writer); - var keepAlive = await HandleRequestAsync(server, writer, request); + var keepAlive = await HandleRequestAsync(server, writer, request, reactor, fileConnection); WarnIfThreadHopped(server, reactorThreadId, "after-handle"); if (!keepAlive || !await request.FinishAsync()) @@ -154,7 +162,8 @@ private static bool TryParseRequest(ref ReadOnlySequence buffer, BinaryReq } // Runs one request through the handler chain and writes the response; returns whether the connection lives on. - private static async ValueTask HandleRequestAsync(IServer server, PipeWriter writer, Request request) + private static async ValueTask HandleRequestAsync(IServer server, PipeWriter writer, Request request, IoReactor? reactor, + IoConnection? fileConnection) { var header = request.Header; @@ -175,9 +184,13 @@ private static async ValueTask HandleRequestAsync(IServer server, PipeWrit var response = await server.Handler.HandleAsync(request) ?? throw new InvalidOperationException("The root request handler did not return a response"); + // A handler that awaited something other than ioxide resumes on the thread that completed it, and the + // response, with the thread-statics it reads, has to be written on the reactor. + await new ReactorSwitch(reactor); + var closeRequested = response.Mode is Connection.Close or Connection.Upgrade; - await Http1Responder.WriteAsync(writer, request, response, keepAliveRequested && !closeRequested, headRequest); + await Http1Responder.WriteAsync(writer, request, response, keepAliveRequested && !closeRequested, headRequest, fileConnection); return keepAliveRequested && !closeRequested; } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs index a7c09f4dd..fe1472567 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/Http2Driver.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.IO.Pipelines; using System.Net; @@ -6,7 +7,6 @@ using ioxide.http2; -using Microsoft.Extensions.Logging; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Responses; @@ -17,32 +17,23 @@ internal static class Http2Driver { private static readonly ReadOnlyMemory Head = "HEAD"u8.ToArray(); - private static readonly Http2Options Options = new() { StreamRequestBodies = true }; + // Runs one HTTP/2 connection, dispatching each stream as soon as its headers are in. Taking the + // body reader is what makes ioxide stream the request body instead of assembling it first. + internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pipe, Http2Options options, + IPAddress? remoteAddress, bool secure) + => new Http2Connection(pipe, options) + .RunAsync((request, body, writer) => DispatchAsync(server, endPoint, request, body, writer, remoteAddress, secure)); - // Runs one HTTP/2 connection, dispatching each stream as it opens. - internal static Task RunAsync(IServer server, IEndPoint endPoint, IDuplexPipe pipe, IPAddress? remoteAddress, bool secure) - => new Http2Connection(pipe, Options) - .RunAsync((request, writer) => DispatchAsync(server, endPoint, request, writer, remoteAddress, secure)); - - // One stream through the handler chain; a fault becomes a 500 rather than a dead stream. + // Maps one stream's request, and runs it through the dispatch HTTP/2 and HTTP/3 share. private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, Http2Request request, - Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) + Http2BodyReader body, Http2ResponseWriter writer, IPAddress? remoteAddress, bool secure) { StreamedRequest? mapped = null; - // Whether the head is already on the wire. IsCompleted cannot answer this - it means - // END_STREAM has been sent - so a fault AFTER the headers went out used to take the error - // path anyway, where WriteHeaders throws "headers have already been written". That second - // exception replaced the first, so the only thing ever reported was the masking error and - // the real fault was never logged at all. - var headersWritten = false; - try { var headRequest = request.Method.Span.SequenceEqual(Head.Span); - var reader = request.BodyReader; - mapped = StreamedRequest.Rent(); for (var i = 0; i < request.Headers.Count; i++) @@ -52,43 +43,44 @@ private static async ValueTask DispatchAsync(IServer server, IEndPoint endPoint, } mapped.Apply(server, endPoint, request.Method, request.Path, request.Authority, - reader is null ? null : reader.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); - - var response = await server.Handler.HandleAsync(mapped) - ?? throw new InvalidOperationException("The root request handler did not return a response"); - - var data = StreamedResponder.BuildHeaders(response, mapped); - - var head = new Http2Response { Status = data.Status }; - - foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in data.Headers) - { - head.Headers.Add(name, value); - } + request.HasBody, body.ReadAsync, remoteAddress, HttpProtocol.Http2, secure); - writer.WriteHeaders(head); - headersWritten = true; - - await StreamedResponder.WriteBodyAsync(response, writer, writer.FlushAsync, headRequest); + await StreamedDriver.DispatchAsync(server, mapped, new Writer(writer), headRequest, "HTTP/2"); } - catch (Exception e) + finally { - server.Logging.CreateLogger("GenHTTP.Engine.Ioxide.Protocol.Drivers.Tcp.Http2Driver") - .LogError(e, "Failed to handle HTTP/2 request"); - - if (!headersWritten && !writer.IsCompleted) + if (mapped is not null) { - writer.WriteHeaders(new Http2Response { Status = 500 }); + StreamedRequest.Return(mapped); } } - finally + } + + /// ioxide's HTTP/2 response writer, as the shared dispatch drives it. + private readonly struct Writer(Http2ResponseWriter writer) : IStreamedWriter + { + public IBufferWriter Body => writer; + + public Func> Flush => writer.FlushAsync; + + public bool IsCompleted => writer.IsCompleted; + + public void WriteHead(StreamedResponseData head) { - if (mapped is not null) + var response = new Http2Response { Status = head.Status }; + + foreach ((ReadOnlyMemory name, ReadOnlyMemory value) in head.Headers) { - StreamedRequest.Return(mapped); + response.Headers.Add(name, value); } - await writer.CompleteAsync(); + writer.WriteHeaders(response); } + + public void WriteHead(int status) => writer.WriteHeaders(new Http2Response { Status = status }); + + public ValueTask CompleteAsync() => writer.CompleteAsync(); + + public ValueTask ResetStreamAsync() => writer.ResetStreamAsync(); } } diff --git a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs index 1d2ffd5e5..ba923a20d 100644 --- a/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs +++ b/Engine/Ioxide/Protocol/Drivers/Tcp/TcpDriver.cs @@ -12,6 +12,7 @@ using GenHTTP.Engine.Ioxide.Infrastructure; using GenHTTP.Engine.Ioxide.Infrastructure.Endpoints; +using ioxide.http2; using ioxide.tls; using IoxideTls = ioxide.tls; @@ -23,12 +24,6 @@ namespace GenHTTP.Engine.Ioxide.Protocol.Drivers.Tcp; /// Takes an accepted TCP connection through TLS and up to whichever protocol it turns out to speak. internal static partial class TcpDriver { - private const int ShutWrite = 1; - - // Half-closes the socket, which is what puts a FIN on the wire. - [LibraryImport("libc", EntryPoint = "shutdown")] - private static partial int Shutdown(int sockfd, int how); - // The peer's sockaddr, which ioxide does not surface itself. [LibraryImport("libc", EntryPoint = "getpeername")] private static partial int GetPeerName(int sockfd, [Out] byte[] addr, ref int addrlen); @@ -36,7 +31,7 @@ internal static partial class TcpDriver private static readonly ReadOnlyMemory Preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"u8.ToArray(); // Takes an accepted connection through TLS if the port has it, then hands it to whichever protocol it turns out to speak. - internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoConnection conn, + internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoConnection conn, Http2Options http2Options, HttpProtocols httpProtocols = HttpProtocols.Http1) { IDuplexPipe pipe; @@ -49,7 +44,6 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon { if (!IoxideReactor.Current.GetService().TryFor(conn.ListenerPort, out var service)) { - _ = Shutdown(conn.ClientFd, ShutWrite); conn.DecRef(); return; } @@ -83,7 +77,7 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon { try { - await Http2Driver.RunAsync(server, endPoint, pipe, remoteAddress, endPoint.Secure); + await Http2Driver.RunAsync(server, endPoint, pipe, http2Options, remoteAddress, endPoint.Secure); } finally { @@ -118,7 +112,7 @@ internal static async Task HandleAsync(IServer server, IEndPoint endPoint, IoCon return (new TlsConnectionDualPipe(conn, session), session.NegotiatedAlpn); } - // Asks the endpoint's validator about the peer certificate OpenSSL ended up with. + // Asks the endpoint's validator about the peer certificate BoringSSL ended up with. private static bool Accepts(ICertificateValidator validator, IoxideTls.TlsSession session) { var der = session.PeerCertificateDer; @@ -167,7 +161,9 @@ private static async ValueTask StartsWithPrefaceAsync(PipeReader reader) } } - // Completes both halves of the pipe, then FINs the socket and drops the connection's reference. + // Completes both halves of the pipe, then drops the connection's reference. The release is what + // FINs the socket - once any send still in flight is out, and never on a descriptor the reactor + // has already closed. internal static async ValueTask CloseAsync(IDuplexPipe pipe, IoConnection conn) { await pipe.Input.CompleteAsync(); @@ -178,7 +174,6 @@ internal static async ValueTask CloseAsync(IDuplexPipe pipe, IoConnection conn) await disposable.DisposeAsync(); // tears down a TLS transport (stops the decrypt pump, close_notify) } - Shutdown(conn.ClientFd, ShutWrite); conn.DecRef(); } diff --git a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs index 96f917594..f3b792c07 100644 --- a/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs +++ b/Engine/Ioxide/Protocol/Requests/StreamedRequest.cs @@ -87,7 +87,7 @@ internal static void Return(StreamedRequest request) // Points this request at one stream, once its headers are in. internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory method, ReadOnlyMemory path, - ReadOnlyMemory authority, Func>>? read, IPAddress? remoteAddress, + ReadOnlyMemory authority, bool hasBody, Func>> read, IPAddress? remoteAddress, HttpProtocol protocol, bool secure) { _server = server; @@ -97,14 +97,14 @@ internal void Apply(IServer server, IEndPoint endPoint, ReadOnlyMemory met _header.Apply(method, path, authority, protocol); - // A stream carries a body only when it declares one: HTTP/2 and HTTP/3 forbid Transfer-Encoding, - // so Content-Length is the signal - and its absence means no body, matching what the other engines - // report for a request that carries neither header. - _hasBody = read is not null && CheckForBody(_headerEntries); + // The protocol sees a body that comes without a Content-Length, as a streamed upload does. A + // Content-Length still counts on its own, so an empty body announced by one is a body, as it is + // over HTTP/1.1, even when the headers already ended the stream. + _hasBody = hasBody || CheckForBody(_headerEntries); if (_hasBody) { - _body.Apply(read!); + _body.Apply(read); } _client.Apply(remoteAddress, secure ? ClientProtocol.Https : ClientProtocol.Http, null); @@ -251,7 +251,7 @@ private static void ParseQuery(ReadOnlyMemory path, List<(ReadOnlyMemory Name, ReadOnlyMemory Value)> headers) { foreach (var (name, _) in headers) diff --git a/Engine/Ioxide/Protocol/Responses/Http1Responder.cs b/Engine/Ioxide/Protocol/Responses/Http1Responder.cs index 348b386ff..4e3f92e77 100644 --- a/Engine/Ioxide/Protocol/Responses/Http1Responder.cs +++ b/Engine/Ioxide/Protocol/Responses/Http1Responder.cs @@ -5,6 +5,8 @@ using GenHTTP.Engine.Shared.Types; using GenHTTP.Engine.Ioxide.Protocol.Sinks; +using IoConnection = ioxide.TcpConnection; + namespace GenHTTP.Engine.Ioxide.Protocol.Responses; /// Writes an HTTP/1.1 response onto the connection. @@ -12,8 +14,10 @@ internal static class Http1Responder { private static readonly byte[] ServerHeader = "Server: ioxide-genhttp\r\n"u8.ToArray(); - // Writes the status line and headers, and the body where the request is owed one. - internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, IResponse response, bool keepAlive, bool headRequest) + // Writes the status line and headers, and the body where the request is owed one. The connection is given when + // the writer is its own write buffer, so a file can be read straight into it. + internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, IResponse response, bool keepAlive, bool headRequest, + IoConnection? connection = null) { ResponseSerializer.WriteHead(writer, request, response, keepAlive, ServerHeader, Http1DateHeader.Get(), isHttp10: false); @@ -43,7 +47,7 @@ internal static async ValueTask WriteAsync(PipeWriter writer, Request? request, } else { - await content.WriteAsync(new Http1Sink(writer)); + await content.WriteAsync(connection is null ? new Http1Sink(writer) : new Http1FileSink(writer, connection)); } } finally diff --git a/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs b/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs index 90c06b7f3..a8581088a 100644 --- a/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs +++ b/Engine/Ioxide/Protocol/Responses/StreamedResponder.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.IO.Pipelines; using GenHTTP.Api.Protocol; using GenHTTP.Engine.Ioxide.Protocol.Requests; using GenHTTP.Engine.Ioxide.Protocol.Sinks; @@ -89,7 +90,7 @@ internal static StreamedResponseData BuildHeaders(IResponse response, StreamedRe } // Writes the content, or skips it for a HEAD while still disposing it. - internal static async ValueTask WriteBodyAsync(IResponse response, IBufferWriter writer, Func flush, bool headRequest) + internal static async ValueTask WriteBodyAsync(IResponse response, IBufferWriter writer, Func> flush, bool headRequest) { var content = response.Content; diff --git a/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs b/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs new file mode 100644 index 000000000..3a158f451 --- /dev/null +++ b/Engine/Ioxide/Protocol/Sinks/Http1FileSink.cs @@ -0,0 +1,50 @@ +using System.Buffers; +using System.IO.Pipelines; + +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Ioxide; +using GenHTTP.Api.Protocol; + +using IoConnection = ioxide.TcpConnection; + +namespace GenHTTP.Engine.Ioxide.Protocol.Sinks; + +/// +/// on a connection whose writer is its own write buffer, in cleartext or with kernel TLS: a +/// file is read off the ring straight into that buffer, after the headers, so its bytes are never copied. +/// +internal sealed class Http1FileSink(PipeWriter writer, IoConnection connection) : IResponseSink, IIoxideFileSink +{ + // ReadFileAsync grows the write buffer to whatever it is asked for, so each read stays well under its default + // 16 KiB, leaving room for the status line and headers ahead of the first. + private const int Chunk = 12 * 1024; + + private Stream? _stream; + + public IBufferWriter Writer => writer; + + public Stream Stream => _stream ??= new Http1WriterStream(writer, writer); + + public async ValueTask WriteFileAsync(int fd, long offset, long length) + { + var end = offset + length; + + while (offset < end) + { + var read = await connection.ReadFileAsync(fd, (int)Math.Min(end - offset, Chunk), offset); + + if (read <= 0) + { + break; + } + + connection.AdvanceWrite(read); + offset += read; + + if ((await writer.FlushAsync()).IsCompleted) + { + throw new ConnectionClosedException(); + } + } + } +} diff --git a/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs b/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs index f387ca122..37c42bbdb 100644 --- a/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs +++ b/Engine/Ioxide/Protocol/Sinks/StreamedSink.cs @@ -1,5 +1,7 @@ using System.Buffers; +using System.IO.Pipelines; +using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; namespace GenHTTP.Engine.Ioxide.Protocol.Sinks; @@ -9,12 +11,12 @@ internal sealed class StreamedSink : IResponseSink { private readonly IBufferWriter _writer; - private readonly Func _flush; + private readonly Func> _flush; private Stream? _stream; // The response sink for one stream: write into the frame buffer, flush to pace it. - internal StreamedSink(IBufferWriter writer, Func flush) + internal StreamedSink(IBufferWriter writer, Func> flush) { _writer = writer; _flush = flush; @@ -29,12 +31,12 @@ private sealed class FlushingStream : Stream { private readonly IBufferWriter _target; - private readonly Func _flush; + private readonly Func> _flush; private long _written; // A write-only stream over the frame buffer that flushes as it goes. - internal FlushingStream(IBufferWriter target, Func flush) + internal FlushingStream(IBufferWriter target, Func> flush) { _target = target; _flush = flush; @@ -70,7 +72,7 @@ public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTo _target.Write(buffer.Span); _written += buffer.Length; - return _flush(); + return FlushOrThrowAsync(); } // The array overload, over the memory one. @@ -81,7 +83,19 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati public override void Flush() { } // Pushes the staged frames out to the peer. - public override Task FlushAsync(CancellationToken cancellationToken) => _flush().AsTask(); + public override Task FlushAsync(CancellationToken cancellationToken) => FlushOrThrowAsync().AsTask(); + + // ioxide reports a peer that reset the stream or left as a completed flush, never as an + // exception. A Stream has no other way to say so than to throw - so a writer that loops + // until a write fails (a server-sent event source, a long download) stops instead of + // producing for nobody forever, as it does over HTTP/1.1. + private async ValueTask FlushOrThrowAsync() + { + if ((await _flush()).IsCompleted) + { + throw new ConnectionClosedException(); + } + } // Write-only, so reading is a mistake worth reporting. public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); diff --git a/Engine/Shared/Types/Body/ChunkedFraming.cs b/Engine/Shared/Types/Body/ChunkedFraming.cs index 834e06e41..d8aa0ceb2 100644 --- a/Engine/Shared/Types/Body/ChunkedFraming.cs +++ b/Engine/Shared/Types/Body/ChunkedFraming.cs @@ -251,12 +251,14 @@ private SequencePosition ConsumeBuffered(ReadOnlySequence available, Array private int CopyChunk(ReadOnlySequence data, SequencePosition end, Span destination) { + // Taken before the reader is advanced: the length is computed from the reader's + // segments, which it may reuse once they have been handed back + var length = (int)data.Length; + data.CopyTo(destination); Reader.AdvanceTo(end); - var length = (int)data.Length; - _position += length; return length; diff --git a/Modules/Files/GenHTTP.Modules.Files.csproj b/Modules/Files/GenHTTP.Modules.Files.csproj index cb0b5b150..c38396da1 100644 --- a/Modules/Files/GenHTTP.Modules.Files.csproj +++ b/Modules/Files/GenHTTP.Modules.Files.csproj @@ -22,7 +22,7 @@ - + diff --git a/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs b/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs index 62341cf63..28cb32621 100644 --- a/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs +++ b/Modules/Files/Multi/Ioxide/IoxideAssetContent.cs @@ -14,9 +14,10 @@ namespace GenHTTP.Modules.Files.Multi; /// bytes into the ioxide write slab at once. Re-resolves the asset under its own /// lease, so nothing is held across an await. /// -/// The body is read positionally off the ring through a per-reactor pool. -/// It cannot use the connection's write slab directly - TcpConnection.ReadFileAsync reads -/// into that slab, and this writes into GenHTTP's response sink instead - so the copy stays. +/// Where the sink is an (HTTP/1.1 in cleartext, or with kernel TLS), the +/// body is read off the ring straight into the connection's write buffer. Elsewhere (TLS in BoringSSL, +/// HTTP/2, HTTP/3) it is read positionally through a per-reactor pool and +/// copied into the sink. /// internal sealed class IoxideAssetContent(StaticAssets assets, string path, long length, ContentType contentType, ReadOnlyMemory? contentEncoding) : IResponseContent { @@ -45,6 +46,12 @@ public async ValueTask WriteAsync(IResponseSink sink) // The body is always read off the ring, from the descriptor the snapshot holds. Keeping // that descriptor honest is AssetRefresh's job, and it rebuilds the whole snapshot rather // than having every request ask whether its own file still matches. + if (sink is IIoxideFileSink files) + { + await files.WriteFileAsync(asset.Fd, 0, length); + return; + } + await WriteFromDisk(sink, asset.Fd, length); } diff --git a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs index 3a7cce6c7..ef3b1c110 100644 --- a/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs +++ b/Modules/Websockets/Functional/FunctionalWebsocketBuilder.cs @@ -51,6 +51,17 @@ public FunctionalWebsocketBuilder DoNotAllocateFrameData() return this; } + /// + /// Stops the connection from serializing its writes, which it does by default. Writes from + /// several threads then need to be serialized by the caller, including the pong and close + /// frames the default and handlers send. + /// + public FunctionalWebsocketBuilder DisableWriteSynchronization() + { + _builder.DisableWriteSynchronization(); + return this; + } + /// /// Invoked when a new client has connected. /// @@ -106,6 +117,12 @@ public FunctionalWebsocketBuilder OnContinue(Func /// The connection to the client /// The incoming ping frame + /// + /// The default answers with a pong, written from the message pump. The connection serializes it + /// with every other write, unless built with - then, if + /// you write from other threads behind a lock of your own, replace it with one that goes through + /// that lock as well. + /// public FunctionalWebsocketBuilder OnPing(Func handler) { _onPing = handler; @@ -128,6 +145,10 @@ public FunctionalWebsocketBuilder OnPong(Func /// The connection to the client /// The frame sent by the client + /// + /// The default answers with a close frame, written from the message pump - see + /// on serializing writes made from other threads. + /// public FunctionalWebsocketBuilder OnClose(Func handler) { _onClose = handler; diff --git a/Modules/Websockets/IReactiveHandler.cs b/Modules/Websockets/IReactiveHandler.cs index ac59a07a1..817eb6cf7 100644 --- a/Modules/Websockets/IReactiveHandler.cs +++ b/Modules/Websockets/IReactiveHandler.cs @@ -36,6 +36,12 @@ public interface IReactiveHandler /// /// The connection to the client /// The incoming ping frame + /// + /// The default answers with a pong, written from the message pump. The connection serializes it + /// with every other write, unless built with DisableWriteSynchronization() - then, if you + /// write from other threads behind a lock of your own, override this to go through that lock + /// as well. + /// ValueTask OnPing(IReactiveConnection connection, IWebsocketFrame message) => connection.PongAsync(message.Data); /// @@ -50,6 +56,10 @@ public interface IReactiveHandler /// /// The connection to the client /// The frame sent by the client + /// + /// The default answers with a close frame, written from the message pump - see + /// on serializing writes made from other threads. + /// ValueTask OnClose(IReactiveConnection connection, IWebsocketFrame message) => connection.CloseAsync(); /// diff --git a/Modules/Websockets/ISocketConnection.cs b/Modules/Websockets/ISocketConnection.cs index 16355de59..e09dfc329 100644 --- a/Modules/Websockets/ISocketConnection.cs +++ b/Modules/Websockets/ISocketConnection.cs @@ -9,6 +9,16 @@ namespace GenHTTP.Modules.Websockets; /// A connection which can be used to write websocket messages /// to the underlying stream. /// +/// +/// A connection serializes its writes, so frames can be sent from several threads at once, unless +/// it was built with DisableWriteSynchronization(). Then writes from several threads have to +/// be serialized by the caller - including the pong and close frames the default OnPing and +/// OnClose handlers write. +/// +/// Once the handler has ended, every write throws a +/// instead of reaching the client. A background writer, such as a broadcaster or a timer started +/// from OnConnected, should stop on it. +/// public interface ISocketConnection { diff --git a/Modules/Websockets/Protocol/Frame.Decode.cs b/Modules/Websockets/Protocol/Frame.Decode.cs index d28fd18a7..7c7a9cdf6 100644 --- a/Modules/Websockets/Protocol/Frame.Decode.cs +++ b/Modules/Websockets/Protocol/Frame.Decode.cs @@ -2,6 +2,7 @@ using System.Buffers.Binary; using System.Runtime.InteropServices; using System.Text; +using System.Text.Unicode; namespace GenHTTP.Modules.Websockets.Protocol; @@ -12,6 +13,12 @@ public static partial class Frame private const string InvalidControlFrame = "Invalid Control Frame"; private const string InvalidControlFrameLength = "Invalid Control Frame Length"; private const string PayloadTooLarge = "Payload is too large"; + private const string ReservedBitsSet = "Reserved Bits Set"; + private const string InvalidCloseFrame = "Invalid Close Frame"; + private const string InvalidCloseCode = "Invalid Close Code"; + private const string InvalidUtf8 = "Invalid UTF-8"; + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); /* Websockets RFC 6455 Frame Decode definition (LLM generated) @@ -20,17 +27,18 @@ The decode algorithm reads a raw WebSocket frame from a buffer and converts it i which determine whether the frame is a complete message, a fragment, or a control frame such as Ping, Pong, or Close. It then checks the MASK bit and the 7-bit base payload length. If the length field indicates an extended payload (126 or 127), the algorithm reads the appropriate 16-bit or 64-bit length field and adjusts the header size accordingly. Control-frame rules are enforced: control frames must - not be fragmented and must not use extended lengths. After determining the final payload length and verifying that the buffer contains + not be fragmented and must not use extended lengths, and no reserved bit may be set. After determining the final payload length and verifying that the buffer contains enough bytes for the full header, mask key, and payload, the decoder extracts the optional 4-byte mask key and applies the unmasking XOR operation in place if masking is enabled. For normal data frames (Text, Binary, Continue), the raw unmasked payload bytes are returned - as-is, while Close frames are additionally parsed to extract the close code and UTF-8 reason. Finally, the method returns a WebsocketFrame + as-is (an unfragmented Text frame is checked to be valid UTF-8), while Close frames are additionally parsed to extract and validate + the close code and UTF-8 reason. Finally, the method returns a WebsocketFrame containing the payload, frame type, and FIN flag, leaving higher-level logic to handle message reassembly and continuation semantics. */ internal static WebsocketFrame Decode( ISocketConnection connection, ref ReadOnlySequence sequence, - int rxMaxBufferSize, + int maxPayloadLength, out SequencePosition consumed, out SequencePosition examined) { @@ -73,6 +81,13 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re return new WebsocketFrame(connection, new FrameError(InvalidOpCode, FrameErrorType.InvalidOpCode)); } + if ((b0 & 0b0111_0000) != 0) + { + consumed = reader.Position; + examined = reader.Position; + return new WebsocketFrame(connection, new FrameError(ReservedBitsSet, FrameErrorType.InvalidReservedBits)); + } + var isControlFrame = frameType is FrameType.Close or FrameType.Ping or FrameType.Pong; var isMasked = (b1 & 0x80) != 0; var payloadLen7 = (byte)(b1 & 0x7F); @@ -110,10 +125,7 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re payloadLen64 = len64; } - const int MaxFrameHeaderSize = 14; - var maxAllowedPayload = rxMaxBufferSize - MaxFrameHeaderSize; - - if (payloadLen64 < 0 || payloadLen64 > maxAllowedPayload || payloadLen64 > int.MaxValue) + if (payloadLen64 < 0 || payloadLen64 > maxPayloadLength || payloadLen64 > int.MaxValue) { consumed = reader.Position; examined = reader.Position; @@ -153,6 +165,11 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re // Close frame if (frameType == FrameType.Close) { + if (payloadLength == 1) + { + return new WebsocketFrame(connection, new FrameError(InvalidCloseFrame, FrameErrorType.InvalidCloseFrame)); + } + if (payloadLength >= 2) { // Read close code @@ -160,6 +177,11 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re payloadSeq.Slice(0, 2).CopyTo(codeBuf); var closeCode = BinaryPrimitives.ReadUInt16BigEndian(codeBuf); + if (!IsValidCloseCode(closeCode)) + { + return new WebsocketFrame(connection, new FrameError(InvalidCloseCode, FrameErrorType.InvalidCloseFrame)); + } + // Read reason (allocates string, only for Close) string? reason = null; if (payloadLength > 2) @@ -168,6 +190,12 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re // so we must materialize reason bytes OR decode manually. // We'll keep it simple: materialize only the reason part. var reasonBytes = payloadSeq.Slice(2).ToArray(); // alloc only for Close + + if (!Utf8.IsValid(reasonBytes)) + { + return new WebsocketFrame(connection, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + reason = Encoding.UTF8.GetString(reasonBytes); } @@ -186,10 +214,56 @@ WebsocketFrame Incomplete(ref SequencePosition c, ref SequencePosition e, ref Re } // ------------------------------------------------------- + // A fragmented text message can only be checked once it has been reassembled + if (frameType == FrameType.Text && fin && !IsValidUtf8(payloadSeq)) + { + return new WebsocketFrame(connection, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + // Normal frames: return raw payload slice (zero-copy) return new WebsocketFrame(connection, ref payloadSeq, frameType, fin: fin); } + /// + /// Checks whether the given payload is valid UTF-8, also when a code point spans two segments. + /// + internal static bool IsValidUtf8(in ReadOnlySequence payload) + { + if (payload.IsSingleSegment) + { + return Utf8.IsValid(payload.FirstSpan); + } + + var decoder = StrictUtf8.GetDecoder(); + + Span chars = stackalloc char[256]; + + try + { + foreach (var segment in payload) + { + var bytes = segment.Span; + + while (!bytes.IsEmpty) + { + decoder.Convert(bytes, chars, flush: false, out var used, out _, out _); + bytes = bytes[used..]; + } + } + + decoder.Convert(ReadOnlySpan.Empty, chars, flush: true, out _, out _, out _); + + return true; + } + catch (DecoderFallbackException) + { + return false; + } + } + + // RFC 6455 7.4 and the IANA registry: 1004-1006 and 1015 are never sent, 1016-2999 are unassigned + private static bool IsValidCloseCode(ushort code) => code is >= 1000 and <= 1003 or >= 1007 and <= 1014 or >= 3000 and <= 4999; + /// /// Brute force modifying the PipeReader's buffer, unmask the bytes. /// diff --git a/Modules/Websockets/Protocol/FrameError.cs b/Modules/Websockets/Protocol/FrameError.cs index fbbcc9ba1..ff8dc2243 100644 --- a/Modules/Websockets/Protocol/FrameError.cs +++ b/Modules/Websockets/Protocol/FrameError.cs @@ -4,4 +4,26 @@ public record FrameError(string Message, FrameErrorType ErrorType = FrameErrorTy { public const string ReadCanceled = "Read was canceled"; public const string UnexpectedEndOfStream = "Unexpected end of stream while reading WebSocket frame."; + + /// + /// The status code to close the connection with because of this error (RFC 6455, 7.4.1), + /// or null if there is no one left to tell, as when the client went away or the read was canceled. + /// + /// + /// + /// if (frame.IsError(out var error) && error.CloseStatus != null) + /// { + /// await connection.CloseAsync(statusCode: error.CloseStatus.Value); + /// } + /// + /// + public ushort? CloseStatus => ErrorType switch + { + FrameErrorType.InvalidOpCode or FrameErrorType.InvalidControlFrame or FrameErrorType.InvalidControlFrameLength + or FrameErrorType.InvalidReservedBits or FrameErrorType.InvalidContinuation or FrameErrorType.InvalidCloseFrame => 1002, + FrameErrorType.InvalidUtf8 => 1007, + FrameErrorType.PayloadTooLarge => 1009, + FrameErrorType.UndefinedBehavior => 1011, + _ => null + }; } \ No newline at end of file diff --git a/Modules/Websockets/Protocol/FrameErrorType.cs b/Modules/Websockets/Protocol/FrameErrorType.cs index 408e9c753..b54fe59e4 100644 --- a/Modules/Websockets/Protocol/FrameErrorType.cs +++ b/Modules/Websockets/Protocol/FrameErrorType.cs @@ -10,5 +10,25 @@ public enum FrameErrorType InvalidControlFrameLength, Canceled, IncompleteForever, // PipeReader completed without a full frame - UndefinedBehavior + UndefinedBehavior, + + /// + /// A reserved bit of the frame header is set, although no extension giving it a meaning was negotiated (RFC 6455, 5.2). + /// + InvalidReservedBits, + + /// + /// A continuation frame arrived with no message to continue, or a new message started before the previous one was finished (RFC 6455, 5.4). + /// + InvalidContinuation, + + /// + /// A close frame carries a payload of one byte or a status code that may not be sent (RFC 6455, 5.5.1 and 7.4). + /// + InvalidCloseFrame, + + /// + /// A text message or the reason of a close frame is not valid UTF-8 (RFC 6455, 8.1). + /// + InvalidUtf8 } diff --git a/Modules/Websockets/Provider/ConnectionSettings.cs b/Modules/Websockets/Provider/ConnectionSettings.cs index 95f61978d..11c9bf368 100644 --- a/Modules/Websockets/Provider/ConnectionSettings.cs +++ b/Modules/Websockets/Provider/ConnectionSettings.cs @@ -10,6 +10,8 @@ namespace GenHTTP.Modules.Websockets.Provider; /// The serialization handler to send and receive structured data /// Whether the API user is expected to handle continuation frames manually /// Whether data will automatically be allocated so it can still be accessed after the next message has been read +/// Whether every write to the connection is serialized, so frames can be sent from several threads at once +/// The largest payload, in bytes, a single frame sent by the client may carry public record ConnectionSettings( FormatterRegistry Formatters, @@ -18,6 +20,10 @@ public record ConnectionSettings( bool HandleContinuationFramesManually, - bool AllocateFrameData + bool AllocateFrameData, + + bool SynchronizeWrites = true, + + int MaxFrameSize = 16 * 1024 ); diff --git a/Modules/Websockets/Provider/WebsocketBuilder.cs b/Modules/Websockets/Provider/WebsocketBuilder.cs index 56f6115bc..c5165a3d0 100644 --- a/Modules/Websockets/Provider/WebsocketBuilder.cs +++ b/Modules/Websockets/Provider/WebsocketBuilder.cs @@ -11,7 +11,7 @@ public abstract class WebsocketBuilder : IHandlerBuilder where T : Websock { protected readonly List _concerns = []; - protected bool _handleContinuationFramesManually, _allocateFrameData = true; + protected bool _handleContinuationFramesManually, _allocateFrameData = true, _synchronizeWrites = true; protected int _maxRxBufferSize = BufferSize.Read; @@ -50,6 +50,11 @@ public T Serialization(ISerializationFormat format) /// handled and aggregated by the frame readers. If you would like to handle /// such frames manually, you can set this option. /// + /// + /// The frames of a fragmented text message are then passed on as they arrive, so checking + /// that the message is valid UTF-8 is up to the handler. Unfragmented text messages are + /// still checked. + /// public T HandleContinuationFramesManually() { _handleContinuationFramesManually = true; @@ -71,7 +76,49 @@ public T DoNotAllocateFrameData() return (T)this; } - protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData); + /// + /// By default, a connection serializes every write and its flush, so frames can be sent from + /// several threads at once - including the pong and close frames sent by the default + /// OnPing and OnClose handlers, which run on the message pump. + /// + /// + /// If a connection is only ever written by one thread at a time, you can disable this to skip + /// the lock on every write. Writes from several threads then need to be serialized by the + /// caller, the frames of the default OnPing and OnClose handlers included. + /// + /// The synchronization covers one call at a time. A message split into several frames + /// (fin: false) still needs the caller's lock across all of them, so another message + /// cannot get in between. + /// + public T DisableWriteSynchronization() + { + _synchronizeWrites = false; + return (T)this; + } + + /// + /// Sets the largest payload, in bytes, a single frame sent by the client may carry (16 KiB by default). + /// A larger frame is reported as a error. + /// + /// + /// A frame is kept in memory until it has been received completely. A message split into + /// several frames may be larger than this. + /// + /// The largest payload of a single frame + /// + /// + /// Websocket.Imperative().MaxFrameSize(64 * 1024).Handler(new MyHandler()); + /// + /// + public T MaxFrameSize(int bytes) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytes); + + _maxRxBufferSize = bytes; + return (T)this; + } + + protected ConnectionSettings BuildSettings() => new(_formatters, _serializationFormat, _handleContinuationFramesManually, _allocateFrameData, _synchronizeWrites, _maxRxBufferSize); public abstract IHandler Build(); diff --git a/Modules/Websockets/Provider/WebsocketConnection.cs b/Modules/Websockets/Provider/WebsocketConnection.cs index e568744db..f39242264 100644 --- a/Modules/Websockets/Provider/WebsocketConnection.cs +++ b/Modules/Websockets/Provider/WebsocketConnection.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using System.Text; +using GenHTTP.Api.Infrastructure; using GenHTTP.Api.Protocol; using GenHTTP.Modules.Websockets.Protocol; @@ -13,12 +14,20 @@ namespace GenHTTP.Modules.Websockets.Provider; public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnection, IAsyncDisposable { - private const int MaxFrameSize = 16 * 1024; + private const string NothingToContinue = "Continuation frame without a message to continue"; + private const string MessageNotFinished = "New message started before the previous one was finished"; + private const string InvalidUtf8 = "Invalid UTF-8"; private readonly IBufferWriter _writer; private readonly Stream _stream; private readonly PipeReader _pipeReader; + // Taken across each write and its flush, unless the settings turn write synchronization off. + private readonly SemaphoreSlim? _writeLock; + + // Set once the handler has ended: the sink is the engine's again, so every later write throws. + private bool _closed; + private SequencePosition _consumed; private SequencePosition _examined; @@ -32,6 +41,9 @@ public sealed class WebsocketConnection : IReactiveConnection, IImperativeConnec private bool _keepPipeReaderBufferValid; private bool _pendingAdvance; + // A data frame without FIN has been read and the continuation frame finishing its message has not + private bool _inMessage; + private WebsocketFrame _frame = null!; #region Get-/Setters @@ -52,6 +64,8 @@ public WebsocketConnection(IRequest request, IResponseSink sink, ConnectionSetti _writer = sink.Writer; _stream = sink.Stream; + _writeLock = settings.SynchronizeWrites ? new SemaphoreSlim(1, 1) : null; + _pipeReader = request.Upgrade(); } @@ -61,20 +75,50 @@ public WebsocketConnection(IRequest request, IResponseSink sink, ConnectionSetti public async ValueTask WriteAsync(ReadOnlyMemory payload, FrameType opcode = FrameType.Text, bool fin = true, bool flush = true, CancellationToken token = default) { - Frame.Write(_writer, payload, opcode: (byte)opcode, fin); - if (flush) await _stream.FlushAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); + + try + { + ThrowIfClosed(); + Frame.Write(_writer, payload, opcode: (byte)opcode, fin); + if (flush) await _stream.FlushAsync(token); + } + finally + { + _writeLock?.Release(); + } } public async ValueTask PingAsync(bool flush = true, CancellationToken token = default) { - Frame.WritePing(_writer); - if (flush) await _stream.FlushAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); + + try + { + ThrowIfClosed(); + Frame.WritePing(_writer); + if (flush) await _stream.FlushAsync(token); + } + finally + { + _writeLock?.Release(); + } } public async ValueTask PongAsync(ReadOnlyMemory payload, bool flush = true, CancellationToken token = default) { - Frame.WritePong(_writer, payload); - if (flush) await _stream.FlushAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); + + try + { + ThrowIfClosed(); + Frame.WritePong(_writer, payload); + if (flush) await _stream.FlushAsync(token); + } + finally + { + _writeLock?.Release(); + } } public async ValueTask PongAsync(string payload, bool flush = true, CancellationToken token = default) @@ -89,11 +133,42 @@ public async ValueTask PongAsync(bool flush = true, CancellationToken token = de public async ValueTask CloseAsync(string? reason = null, ushort statusCode = 1000, bool flush = true, CancellationToken token = default) { - Frame.WriteClose(_writer, reason, statusCode); - if (flush) await _stream.FlushAsync(token); + if (_writeLock is not null) await _writeLock.WaitAsync(token); + + try + { + ThrowIfClosed(); + Frame.WriteClose(_writer, reason, statusCode); + if (flush) await _stream.FlushAsync(token); + } + finally + { + _writeLock?.Release(); + } } - public ValueTask FlushAsync(CancellationToken token = default) => new(_stream.FlushAsync(token)); + public async ValueTask FlushAsync(CancellationToken token = default) + { + if (_writeLock is not null) await _writeLock.WaitAsync(token); + + try + { + ThrowIfClosed(); + await _stream.FlushAsync(token); + } + finally + { + _writeLock?.Release(); + } + } + + private void ThrowIfClosed() + { + if (Volatile.Read(ref _closed)) + { + throw new ConnectionClosedException(); + } + } public async ValueTask ReadFrameAsync(CancellationToken token = default) { @@ -111,6 +186,13 @@ public bool TryReadFrame(out IWebsocketFrame frame) { Advance(); + if (_skipFrameInit) + { + // A message interrupted by a control frame is completed by ReadFrameAsync, which reassembles it + frame = default!; + return false; + } + if (!_pipeReader.TryRead(out var result)) { frame = default!; @@ -134,7 +216,18 @@ public bool TryReadFrame(out IWebsocketFrame frame) var currentSequence = result.Buffer; - var decoded = Frame.Decode(this, ref currentSequence, MaxFrameSize, out var consumed, out var examined); + if (!Settings.HandleContinuationFramesManually && StartsFragmentedMessage(currentSequence)) + { + // Left in the pipe for ReadFrameAsync, which reassembles the message. It is not decoded here, + // as decoding unmasks the payload in place. + _pipeReader.AdvanceTo(result.Buffer.Start); + _consumed = result.Buffer.Start; + _examined = result.Buffer.Start; + frame = default!; + return false; + } + + var decoded = Frame.Decode(this, ref currentSequence, Settings.MaxFrameSize, out var consumed, out var examined); _consumed = consumed; _examined = examined; @@ -147,6 +240,8 @@ public bool TryReadFrame(out IWebsocketFrame frame) return false; } + decoded = CheckSequence(decoded); + if (Settings.AllocateFrameData) { decoded.SetCachedData(); @@ -218,6 +313,11 @@ private async ValueTask ReadSegmentedFrameAsync(CancellationToke { _frame.Memory = SequenceUtils.ConcatSequences(_frame.Segments); + if (_frame.Type == FrameType.Text && !Frame.IsValidUtf8(_frame.Memory)) + { + return new WebsocketFrame(this, new FrameError(InvalidUtf8, FrameErrorType.InvalidUtf8)); + } + return _frame; } } @@ -272,7 +372,7 @@ private async ValueTask ReadFrameSegmentAsync(bool isFirstFrame ? result.Buffer.Slice(innerExamined) : result.Buffer; - var frame = Frame.Decode(this, ref currentSequence, MaxFrameSize, out var consumed, out var examined); + var frame = Frame.Decode(this, ref currentSequence, Settings.MaxFrameSize, out var consumed, out var examined); _consumed = consumed; _examined = examined; @@ -302,12 +402,40 @@ private async ValueTask ReadFrameSegmentAsync(bool isFirstFrame // Any other frame (including other errors) _pendingAdvance = true; - return frame; + return CheckSequence(frame); } throw new InvalidOperationException("Server has been stopped"); } + // RFC 6455 5.4: a continuation frame continues the message in progress, and a new message waits for it to finish + private WebsocketFrame CheckSequence(WebsocketFrame frame) + { + switch (frame.Type) + { + case FrameType.Continue: + if (!_inMessage) + { + return new WebsocketFrame(this, new FrameError(NothingToContinue, FrameErrorType.InvalidContinuation)); + } + + _inMessage = !frame.Fin; + return frame; + + case FrameType.Text or FrameType.Binary: + if (_inMessage) + { + return new WebsocketFrame(this, new FrameError(MessageNotFinished, FrameErrorType.InvalidContinuation)); + } + + _inMessage = !frame.Fin; + return frame; + + default: + return frame; + } + } + private void Examine() { _pipeReader.AdvanceTo(_bufferStart, _examined); @@ -342,6 +470,14 @@ private void Advance() [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsErrorFrame(WebsocketFrame frame) => frame.Type == FrameType.Error; + // A Text or Binary frame without FIN, read from its first header byte + private static bool StartsFragmentedMessage(in ReadOnlySequence buffer) + { + var reader = new SequenceReader(buffer); + + return reader.TryPeek(out var b0) && (b0 & 0x80) == 0 && (b0 & 0x0F) is 0x1 or 0x2; + } + [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsControlFrame(WebsocketFrame frame) @@ -359,7 +495,20 @@ private static bool IsControlFrame(WebsocketFrame frame) #region Disposable Pattern - public ValueTask DisposeAsync() => _pipeReader.CompleteAsync(); + // The handler has ended. A write in flight finishes first, under the lock, and every one after it is refused rather + // than reaching a sink the engine may already have given to another connection. + public async ValueTask DisposeAsync() + { + if (_writeLock is not null) + { + await _writeLock.WaitAsync(); + } + + Volatile.Write(ref _closed, true); + _writeLock?.Release(); + + await _pipeReader.CompleteAsync(); + } #endregion diff --git a/Playground/Samples/Ioxide/AllProtocolsSample.cs b/Playground/Samples/Ioxide/AllProtocolsSample.cs index 90dc06ed3..ccb0aa2f5 100644 --- a/Playground/Samples/Ioxide/AllProtocolsSample.cs +++ b/Playground/Samples/Ioxide/AllProtocolsSample.cs @@ -49,6 +49,13 @@ public static IServerHost Create() return Host.Create(options: new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 }, + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, }) .Handler(app) .Bind(IPAddress.Loopback, 8080, HttpProtocols.Http1) diff --git a/Playground/Samples/Ioxide/CertificateRotationSample.cs b/Playground/Samples/Ioxide/CertificateRotationSample.cs index 5f917bc4a..a68a1cd2e 100644 --- a/Playground/Samples/Ioxide/CertificateRotationSample.cs +++ b/Playground/Samples/Ioxide/CertificateRotationSample.cs @@ -60,7 +60,16 @@ public static IServerHost Create() certificates.Add(names[1], alpha, alphaKey); - var host = Host.Create() + var host = Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8443, certificates, httpProtocols: HttpProtocols.Http1AndHttp2); diff --git a/Playground/Samples/Ioxide/Http1Sample.cs b/Playground/Samples/Ioxide/Http1Sample.cs index 1d95dbfed..4717146a3 100644 --- a/Playground/Samples/Ioxide/Http1Sample.cs +++ b/Playground/Samples/Ioxide/Http1Sample.cs @@ -29,7 +29,16 @@ public static IServerHost Create() var app = Layout.Create() .Add("ok", Content.From(Resource.FromString("ok"))); - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8080, HttpProtocols.Http1); } diff --git a/Playground/Samples/Ioxide/Http2Sample.cs b/Playground/Samples/Ioxide/Http2Sample.cs index b15f8becf..75af7bb59 100644 --- a/Playground/Samples/Ioxide/Http2Sample.cs +++ b/Playground/Samples/Ioxide/Http2Sample.cs @@ -36,7 +36,16 @@ public static IServerHost Create() var app = Layout.Create() .Add("ok", Content.From(Resource.FromString("ok"))); - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8081, HttpProtocols.Http2) .Bind(IPAddress.Loopback, 8082, HttpProtocols.Http1AndHttp2); diff --git a/Playground/Samples/Ioxide/Http3Sample.cs b/Playground/Samples/Ioxide/Http3Sample.cs index a079b4ab1..22f45da68 100644 --- a/Playground/Samples/Ioxide/Http3Sample.cs +++ b/Playground/Samples/Ioxide/Http3Sample.cs @@ -44,7 +44,7 @@ public static IServerHost Create() Quic = new QuicTransportOptions { HandshakeTimeoutMs = 10_000, - IdleTimeoutMs = 60_000, + ReadTimeoutMs = 60_000, }, }) .Handler(app) diff --git a/Playground/Samples/Ioxide/KernelTlsSample.cs b/Playground/Samples/Ioxide/KernelTlsSample.cs index 8201fbeeb..f22cfd764 100644 --- a/Playground/Samples/Ioxide/KernelTlsSample.cs +++ b/Playground/Samples/Ioxide/KernelTlsSample.cs @@ -35,7 +35,7 @@ public static IServerHost Create() * TLS connection exactly as it does on a cleartext one. * * It is not a free win, and it is off by default for that reason: on loopback it trails - * OpenSSL on large writes and costs more CPU per request. Turn it on for what it is for + * userspace TLS on large writes and costs more CPU per request. Turn it on for what it is for * - sendfile, and NICs that offload TLS - and measure your own traffic. * * curl -k --http1.1 https://localhost:8443/ok @@ -53,6 +53,10 @@ public static IServerHost Create() { TxKernelTls = true, RxKernelTls = true, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, }, }) .Handler(app) diff --git a/Playground/Samples/Ioxide/README.md b/Playground/Samples/Ioxide/README.md index c076b9c8d..ec0c1b71a 100644 --- a/Playground/Samples/Ioxide/README.md +++ b/Playground/Samples/Ioxide/README.md @@ -19,7 +19,7 @@ A deployment points the providers at the files its ACME client already writes in | `Http1Sample` | `http://localhost:8080` | the smallest binding there is - no certificate, no negotiation | | `Http2Sample` | `http://localhost:8081`, `:8082` | h2c: HTTP/2 alone, and sharing one socket with HTTP/1.1 | | `Http3Sample` | `https://localhost:8443` | QUIC only, and why it needs the certificate as files | -| `TlsSample` | `https://localhost:8443` | TLS in OpenSSL, with ALPN choosing h2 or http/1.1 | +| `TlsSample` | `https://localhost:8443` | TLS in BoringSSL, with ALPN choosing h2 or http/1.1 | | `KernelTlsSample` | `https://localhost:8443` | the record layer in the kernel, and what it pins | | `SniSample` | `https://localhost:8443` | a certificate per host name, chosen by the client | | `CertificateRotationSample` | `https://localhost:8443` | replacing certificates on a running server (`SIGHUP`) | diff --git a/Playground/Samples/Ioxide/ShowcaseSample.cs b/Playground/Samples/Ioxide/ShowcaseSample.cs index 3ff3f3bdc..f09e92919 100644 --- a/Playground/Samples/Ioxide/ShowcaseSample.cs +++ b/Playground/Samples/Ioxide/ShowcaseSample.cs @@ -115,6 +115,10 @@ public static IServerHost Create() Tcp = new TcpTransportOptions { HandshakeTimeoutMs = 10_000, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, CipherSuites = null, CipherList = null, // kTLS needs the Linux tls module and pins TLS 1.3 - see @@ -132,7 +136,7 @@ public static IServerHost Create() Quic = new QuicTransportOptions { HandshakeTimeoutMs = 10_000, - IdleTimeoutMs = 60_000, + ReadTimeoutMs = 60_000, Routing = QuicRouting.Forward, PinMigratedPeers = true, // Clamped by net.core.rmem_max, and worth measuring rather than @@ -184,7 +188,7 @@ public static IServerHost Create() /// different issuer. /// /// -/// OpenSSL (or ngtcp2 on HTTP/3) validates the chain against that CA before the handshake +/// BoringSSL (or ngtcp2 on HTTP/3) validates the chain against that CA before the handshake /// completes, so a bad one is refused before any request exists. Validate still runs afterwards on /// the TCP transports, for whatever the application wants to decide for itself. /// diff --git a/Playground/Samples/Ioxide/SniSample.cs b/Playground/Samples/Ioxide/SniSample.cs index c292b9fda..41326cb5e 100644 --- a/Playground/Samples/Ioxide/SniSample.cs +++ b/Playground/Samples/Ioxide/SniSample.cs @@ -54,7 +54,16 @@ public static IServerHost Create() certificates.Add(name, certificate, key); } - return Host.Create() + return Host.Create(options: new EngineOptions + { + Tcp = new TcpTransportOptions + { + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, + }, + }) .Handler(app) .Bind(IPAddress.Loopback, 8443, certificates, httpProtocols: HttpProtocols.Http1AndHttp2); } diff --git a/Playground/Samples/Ioxide/TlsSample.cs b/Playground/Samples/Ioxide/TlsSample.cs index d1f39a162..bb08eb512 100644 --- a/Playground/Samples/Ioxide/TlsSample.cs +++ b/Playground/Samples/Ioxide/TlsSample.cs @@ -18,17 +18,17 @@ public static IServerHost Create() { /* * - * TLS is terminated in OpenSSL, and the handshake rides the same io_uring reads and + * TLS is terminated in BoringSSL, and the handshake rides the same io_uring reads and * writes as everything else. Where a port serves both protocols, ALPN decides: the * server offers h2 first, so a client that speaks it gets HTTP/2 and everyone else * falls back to HTTP/1.1. * * The certificate can come as files or as an X509Certificate2 here - unlike HTTP/3, - * OpenSSL takes the certificate as data. Files are still preferred: OpenSSL reads a + * BoringSSL takes the certificate as data. Files are still preferred: BoringSSL reads a * chain file whole, so intermediates come from the file rather than the machine store, * and the private key never enters managed memory. * - * SslProtocols names the floor. OpenSSL takes a minimum version and has no maximum, so + * SslProtocols names the floor. ioxide takes a minimum version and has no maximum, so * Tls12 | Tls13 means "1.2 and up" and Tls13 alone means 1.3-only. * * curl -k --http1.1 https://localhost:8443/ok @@ -47,6 +47,10 @@ public static IServerHost Create() { // A peer that connects and then says nothing is swept rather than held. HandshakeTimeoutMs = 10_000, + // A client that sends nothing for a minute, between requests or within one, is let go, + // and so is one that stops reading, once a flush has waited a minute. + ReadTimeoutMs = 60_000, + SendTimeoutMs = 60_000, }, }) .Handler(app) diff --git a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs index 78584a330..873ff028d 100644 --- a/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs +++ b/Testing/Acceptance/Engine/Body/ChunkedFramingTests.cs @@ -110,6 +110,45 @@ public async Task TestReadWholeChunks() Assert.AreEqual("NEXT", await RestAsync(pipe)); } + [TestMethod] + public async Task TestReadWholeChunkStartingInLaterSegment() + { + // The second chunk starts in the pipe's second segment and ends in its third. Advancing past + // it hands the second segment back to the pool, which resets it, so the length of the chunk + // has to be known before. + var first = new byte[6000]; + var second = new byte[6000]; + + new Random(7).NextBytes(first); + new Random(8).NextBytes(second); + + var pipe = new Pipe(); + + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"{first.Length:X}\r\n")); + await pipe.Writer.WriteAsync(first); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"\r\n{second.Length:X}\r\n")); + await pipe.Writer.WriteAsync(second); + await pipe.Writer.WriteAsync("\r\n0\r\n\r\nNEXT"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var framing = new ChunkedFraming(); + framing.Apply(pipe.Reader); + + var buffer = new byte[8192]; + + Assert.AreEqual(first.Length, await framing.ReadAsync(buffer, CancellationToken.None)); + CollectionAssert.AreEqual(first, buffer[..first.Length]); + + Assert.AreEqual(second.Length, await framing.ReadAsync(buffer, CancellationToken.None)); + CollectionAssert.AreEqual(second, buffer[..second.Length]); + + Assert.AreEqual(0, await framing.ReadAsync(buffer, CancellationToken.None)); + + Assert.IsTrue(await framing.SkipAsync()); + + Assert.AreEqual("NEXT", await RestAsync(pipe)); + } + [TestMethod] public async Task TestReadToEndOfChunkSpanningSegments() { @@ -316,4 +355,114 @@ public async Task TestReuseAfterReset() return (pipe, framing); } + #region Readers reusing their buffers + + [TestMethod] + public async Task TestLargeBodyFromRecyclingReader() + { + // Chunks of uneven sizes, some larger than a read, through a reader that overwrites every + // segment it is advanced past. However segments and reads fall, the payload comes back exactly. + var payload = Payload(256 * 1024, 21); + var body = Chunked(payload, [1, 100, 4095, 4096, 4097, 16_384, 65_536, 100_000]); + + foreach (var segmentSize in (int[])[1000, 4096, 16_384]) + { + foreach (var bufferSize in (int[])[7, 3000, 8192, 70_000]) + { + var layout = $"segments of {segmentSize} bytes, reads of {bufferSize}"; + + var reader = new RecyclingPipeReader(body, segmentSize, segmentsPerRead: 3); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + var result = new MemoryStream(); + var buffer = new byte[bufferSize]; + + int read; + + while ((read = await framing.ReadAsync(buffer, CancellationToken.None)) > 0) + { + Assert.IsTrue(read <= bufferSize, $"{read} bytes read into {bufferSize} ({layout})"); + result.Write(buffer, 0, read); + } + + Assert.IsTrue(payload.AsSpan().SequenceEqual(result.ToArray()), layout); + Assert.AreEqual(payload.Length, framing.Position, layout); + + Assert.IsTrue(await framing.SkipAsync(), layout); + Assert.AreEqual("NEXT", reader.Rest(), layout); + + Assert.IsTrue(reader.Recycled > 0, layout); + } + } + } + + [TestMethod] + public async Task TestLargeBodyToEndFromRecyclingReader() + { + var payload = Payload(4 * 1024 * 1024, 22); + + var reader = new RecyclingPipeReader(Chunked(payload, [4096, 65_536, 4095, 100_000]), 4096, segmentsPerRead: 4); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + var body = await framing.ReadToEndAsync(CancellationToken.None); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(body.Span)); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + [TestMethod] + public async Task TestSkipLargeBodyFromRecyclingReader() + { + var reader = new RecyclingPipeReader(Chunked(Payload(1024 * 1024, 23), [4095, 65_536, 1]), 4096, segmentsPerRead: 2); + + var framing = new ChunkedFraming(); + framing.Apply(reader); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + private static byte[] Payload(int length, int seed) + { + var payload = new byte[length]; + new Random(seed).NextBytes(payload); + + return payload; + } + + /// + /// The payload chunk-encoded in chunks of the given sizes, taken in turn, followed by the + /// start of a next request. + /// + private static byte[] Chunked(byte[] payload, int[] chunkSizes) + { + var body = new MemoryStream(); + + var offset = 0; + var i = 0; + + while (offset < payload.Length) + { + var size = Math.Min(chunkSizes[i++ % chunkSizes.Length], payload.Length - offset); + + body.Write(Encoding.ASCII.GetBytes($"{size:X}\r\n")); + body.Write(payload, offset, size); + body.Write("\r\n"u8); + + offset += size; + } + + body.Write("0\r\n\r\nNEXT"u8); + + return body.ToArray(); + } + + #endregion + } diff --git a/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs b/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs index 3c3085225..a6e497658 100644 --- a/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs +++ b/Testing/Acceptance/Engine/Body/ContentLengthFramingTests.cs @@ -211,4 +211,75 @@ internal static async Task RestAsync(Pipe pipe) return rest; } + #region Readers reusing their buffers + + [TestMethod] + public async Task TestLargeBodyFromRecyclingReader() + { + // Through a reader that overwrites every segment it is advanced past: however segments + // and reads fall, the payload comes back exactly. + var payload = RandomPayload(256 * 1024, 31); + var body = payload.Concat("NEXT"u8.ToArray()).ToArray(); + + foreach (var segmentSize in (int[])[1000, 4096, 16_384]) + { + foreach (var bufferSize in (int[])[7, 3000, 8192, 70_000]) + { + var layout = $"segments of {segmentSize} bytes, reads of {bufferSize}"; + + var reader = new RecyclingPipeReader(body, segmentSize, segmentsPerRead: 3); + + var framing = new ContentLengthFraming(); + framing.Apply(reader, payload.Length); + + var result = new MemoryStream(); + var buffer = new byte[bufferSize]; + + int read; + + while ((read = await framing.ReadAsync(buffer, CancellationToken.None)) > 0) + { + Assert.IsTrue(read <= bufferSize, $"{read} bytes read into {bufferSize} ({layout})"); + result.Write(buffer, 0, read); + } + + Assert.IsTrue(payload.AsSpan().SequenceEqual(result.ToArray()), layout); + Assert.AreEqual(payload.Length, framing.Position, layout); + + Assert.IsTrue(await framing.SkipAsync(), layout); + Assert.AreEqual("NEXT", reader.Rest(), layout); + + Assert.IsTrue(reader.Recycled > 0, layout); + } + } + } + + [TestMethod] + public async Task TestLargeBodyToEndFromRecyclingReader() + { + var payload = RandomPayload(4 * 1024 * 1024, 32); + + var reader = new RecyclingPipeReader(payload.Concat("NEXT"u8.ToArray()).ToArray(), 4096, segmentsPerRead: 4); + + var framing = new ContentLengthFraming(); + framing.Apply(reader, payload.Length); + + var body = await framing.ReadToEndAsync(CancellationToken.None); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(body.Span)); + + Assert.IsTrue(await framing.SkipAsync()); + Assert.AreEqual("NEXT", reader.Rest()); + } + + private static byte[] RandomPayload(int length, int seed) + { + var payload = new byte[length]; + new Random(seed).NextBytes(payload); + + return payload; + } + + #endregion + } diff --git a/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs b/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs new file mode 100644 index 000000000..66cb44ce1 --- /dev/null +++ b/Testing/Acceptance/Engine/Body/RecyclingPipeReader.cs @@ -0,0 +1,153 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; + +namespace GenHTTP.Testing.Acceptance.Engine.Body; + +/// +/// Serves a fixed body in segments of the given size, a few segments per read, and reuses +/// every segment it is advanced past the way a pooling reader may: its bytes are overwritten, +/// its running index is reset and it is unlinked. A reader of the body that still touches +/// memory it has handed back then sees wrong data or a wrong length instead of getting away +/// with it, as it does with a reader that happens not to reuse its segments right away. +/// +internal sealed class RecyclingPipeReader(byte[] data, int segmentSize, int segmentsPerRead) : PipeReader +{ + private Segment? _head; + + private Segment? _tail; + + private int _headIndex; + + private int _delivered; + + private bool _needsData = true; + + /// + /// The number of segments reused after the reader was advanced past them. + /// + public int Recycled { get; private set; } + + public override bool TryRead(out ReadResult result) + { + if (_needsData) + { + Deliver(); + _needsData = false; + } + + var buffer = (_head != null && _tail != null) ? new ReadOnlySequence(_head, _headIndex, _tail, _tail.Memory.Length) : ReadOnlySequence.Empty; + + result = new ReadResult(buffer, isCanceled: false, isCompleted: _delivered == data.Length); + return true; + } + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + TryRead(out var result); + return new(result); + } + + public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + var segment = consumed.GetObject() as Segment; + + if (segment != null) + { + var index = consumed.GetInteger(); + + // A segment consumed to its end goes as well, when the data goes on in the next one + var next = segment.Next as Segment; + + if (index == segment.Memory.Length && next != null) + { + segment = next; + index = 0; + } + + while (_head != null && _head != segment) + { + var released = _head; + + _head = released.Next as Segment; + + released.Recycle(); + Recycled++; + } + + _headIndex = index; + } + + // More data only once everything buffered has been examined, as a pipe waits for its writer then + _needsData = examined.GetObject() == _tail && examined.GetInteger() == (_tail?.Memory.Length ?? 0); + } + + /// + /// What is left of the body, once its reader is done with it. + /// + public string Rest() + { + TryRead(out var result); + + while (!result.IsCompleted) + { + AdvanceTo(result.Buffer.Start, result.Buffer.End); + TryRead(out result); + } + + return Encoding.ASCII.GetString(result.Buffer.ToArray()); + } + + public override void CancelPendingRead() { } + + public override void Complete(Exception? exception = null) { } + + private void Deliver() + { + for (var i = 0; i < segmentsPerRead && _delivered < data.Length; i++) + { + var length = Math.Min(segmentSize, data.Length - _delivered); + + var segment = new Segment(data.AsSpan(_delivered, length).ToArray(), (_tail != null) ? _tail.RunningIndex + _tail.Memory.Length : 0); + + if (_tail == null) + { + _head = segment; + _headIndex = 0; + } + else + { + _tail.Link(segment); + } + + _tail = segment; + _delivered += length; + } + } + + private sealed class Segment : ReadOnlySequenceSegment + { + private readonly byte[] _bytes; + + public Segment(byte[] bytes, long runningIndex) + { + _bytes = bytes; + + Memory = bytes; + RunningIndex = runningIndex; + } + + public void Link(Segment next) => Next = next; + + public void Recycle() + { + _bytes.AsSpan().Fill(0xEE); + + RunningIndex = 0; + Next = null; + } + } + +} diff --git a/Testing/Acceptance/Engine/Body/StreamBodyTests.cs b/Testing/Acceptance/Engine/Body/StreamBodyTests.cs index 7339460ed..1e0a94463 100644 --- a/Testing/Acceptance/Engine/Body/StreamBodyTests.cs +++ b/Testing/Acceptance/Engine/Body/StreamBodyTests.cs @@ -108,6 +108,34 @@ public async Task TestLargeChunked(ServerEngine engine) CollectionAssert.AreEqual(payload, await response.Content.ReadAsByteArrayAsync()); } + [TestMethod] + [MultiEngineTest] + public async Task TestLargeChunkedStreamedBack(ServerEngine engine) + { + // A megabyte in chunks of uneven sizes, streamed straight back while it is still being read + var payload = new byte[1024 * 1024]; + new Random(44).NextBytes(payload); + + await using var runner = await TestHost.RunAsync(new StreamingEchoHandler().Wrap(), engine: engine); + + using var client = TestHost.GetClient(); + + var request = new HttpRequestMessage(HttpMethod.Post, runner.GetUrl()) + { + Content = new UnevenContent(payload, [1, 100, 4095, 4097, 16_384, 65_536, 100_000]) + }; + + request.Headers.TransferEncodingChunked = true; + + using var response = await client.SendAsync(request); + + await response.AssertStatusAsync(HttpStatusCode.OK); + + var echoed = await response.Content.ReadAsByteArrayAsync(); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(echoed)); + } + [TestMethod] [MultiEngineTest] public async Task TestKeepAliveContentLength(ServerEngine engine) diff --git a/Testing/Acceptance/Engine/Body/StreamingEcho.cs b/Testing/Acceptance/Engine/Body/StreamingEcho.cs new file mode 100644 index 000000000..42793c434 --- /dev/null +++ b/Testing/Acceptance/Engine/Body/StreamingEcho.cs @@ -0,0 +1,65 @@ +using System.Net; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using GenHTTP.Modules.IO; + +namespace GenHTTP.Testing.Acceptance.Engine.Body; + +/// +/// Streams the request body straight back as the response, as an echo service does: the response +/// is written while the body is still being read, and the length of neither is known up front. +/// +internal sealed class StreamingEchoHandler : IHandler +{ + + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) + { + var body = request.GetBody(); + + if (body == null) + { + return new(request.Respond().Status(ResponseStatus.BadRequest).Build()); + } + + return new(request.Respond() + .Content(body.AsStream(), ContentType.ApplicationOctetStream) + .Build()); + } + +} + +/// +/// Sends the payload in writes of the given sizes, each flushed on its own, so that every write +/// travels as a chunk (or data frame) of its own. Declares its length only if asked to. +/// +internal sealed class UnevenContent(byte[] payload, int[] sizes, bool declareLength = false) : HttpContent +{ + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + var offset = 0; + var i = 0; + + while (offset < payload.Length) + { + var size = Math.Min(sizes[i++ % sizes.Length], payload.Length - offset); + + await stream.WriteAsync(payload.AsMemory(offset, size)); + await stream.FlushAsync(); + + offset += size; + } + } + + protected override bool TryComputeLength(out long length) + { + length = payload.Length; + return declareLength; + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs b/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs index 44d6e1349..25f555553 100644 --- a/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/CertificateTests.cs @@ -116,6 +116,46 @@ public async Task TestCertificatesRotateOnARunningServer() } } + [IoxideTestMethod] + public async Task TestCertificatesRotateOnEveryReactor() + { + // The reactors of a port share its contexts, so the server rotates them through one of its + // services: a connection has to see the new certificate whichever reactor takes it. + var dir = IoxideCertificates.Isolated(); + + var (certificate, key) = IoxideCertificates.Create("localhost", dir); + + var port = (ushort)TestHost.NextPort(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 4 } }; + + var server = Host.Create(options: options) + .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key)); + + await server.StartAsync(); + + try + { + var before = await IoxideCertificates.SerialAsync(port); + + IoxideCertificates.Create("localhost", dir); + + (server.Instance as IoxideServer)!.ReloadCertificates(); + + // Twenty connections over four reactors: the kernel spreads them by hashing each one. + for (var i = 0; i < 20; i++) + { + Assert.AreNotEqual(before, await IoxideCertificates.SerialAsync(port), + $"connection {i} was still served the certificate the server started with"); + } + } + finally + { + await server.StopAsync(); + } + } + [IoxideTestMethod] public async Task TestRotationNeedsASecureEndpoint() { diff --git a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs index 77fed8f65..822cd5075 100644 --- a/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs +++ b/Testing/Acceptance/Engine/Ioxide/Http3Tests.cs @@ -7,6 +7,7 @@ using GenHTTP.Engine.Ioxide; +using GenHTTP.Modules.Functional; using GenHTTP.Modules.IO; using GenHTTP.Modules.Layouting; @@ -166,6 +167,78 @@ public async Task TestHttp3SharesItsPortNumberWithTcp() } } + [IoxideTestMethod] + public async Task TestHttp3StreamsTheRequestBody() + { + // Large enough to arrive in many chunks, which the handler reads through the body reader + // ioxide hands the driver - an echo that comes back whole proves none went missing. Sent + // without a content-length, as a client streams an upload, so the handler must still see it. + if (!QuicConnection.IsSupported) + { + Assert.Inconclusive("QUIC is not supported on this machine (msquic missing)."); + } + + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + var port = (ushort)TestHost.NextPort(); + + var echo = Inline.Create().Post(async (Stream body) => + { + var copy = new MemoryStream(); + var buffer = new byte[8192]; + + // not CopyToAsync: its ConfigureAwait(false) resumes off the reactor, where the body + // reader must not be called + int read; + + while ((read = await body.ReadAsync(buffer)) > 0) + { + copy.Write(buffer, 0, read); + } + + copy.Seek(0, SeekOrigin.Begin); + + return copy; + }); + + var server = Host.Create() + .Handler(echo) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), + httpProtocols: HttpProtocols.Http3); + + await server.StartAsync(); + + try + { + using var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + using var client = new HttpClient(handler) + { + DefaultRequestVersion = HttpVersion.Version30, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + }; + + var payload = new byte[256 * 1024]; + new Random(42).NextBytes(payload); + + using var content = new UnknownLengthContent(payload); + + using var response = await client.PostAsync($"https://localhost:{port}/", content); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(HttpVersion.Version30, response.Version); + + CollectionAssert.AreEqual(payload, await response.Content.ReadAsByteArrayAsync()); + } + finally + { + await server.StopAsync(); + } + } + private static IServerHost Build() => Host.Create() .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))); diff --git a/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs b/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs new file mode 100644 index 000000000..f6901c698 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/ReactorReturnTests.cs @@ -0,0 +1,93 @@ +using System.Buffers; +using System.Net; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A handler that awaits something other than ioxide comes back on the thread that completed it. The engine +/// returns to the reactor before it writes the response, since the state it keeps per thread lives there. +/// +[TestClass] +public sealed class ReactorReturnTests +{ + + [IoxideTestMethod] + public async Task TestHttp1ResponseIsWrittenOnTheReactor() + => Assert.AreEqual("reactor", await ServeAsync(HttpVersion.Version11)); + + [IoxideTestMethod] + public async Task TestHttp2ResponseIsWrittenOnTheReactor() + => Assert.AreEqual("reactor", await ServeAsync(HttpVersion.Version20)); + + // Asks for exactly one version over cleartext, so h2 goes out as prior knowledge. + private static async Task ServeAsync(Version version) + { + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create().Handler(new LeavingHandler()).Bind(IPAddress.Loopback, port, HttpProtocols.Http1AndHttp2); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(new SocketsHttpHandler()) + { + DefaultRequestVersion = version, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(10), + }; + + using var response = await client.GetAsync($"http://localhost:{port}/"); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + + return await response.Content.ReadAsStringAsync(); + } + finally + { + await server.StopAsync(); + } + } + + // Leaves the reactor before it answers, as a handler awaiting a database or the BCL's HttpClient does. + private sealed class LeavingHandler : IHandler + { + + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public async ValueTask HandleAsync(IRequest request) + { + await Task.Delay(1); + + return request.Respond().Content(new WhereContent()).Build(); + } + + } + + // Writes which thread writes it: a reactor's, or another. + private sealed class WhereContent : IResponseContent + { + + public ulong? Length => null; + + public ContentType? Type => null; + + public ReadOnlyMemory? Encoding => null; + + public ValueTask CalculateChecksumAsync() => new((ulong?)null); + + public ValueTask WriteAsync(IResponseSink sink) + { + sink.Writer.Write(ioxide.Reactor.Current is not null ? "reactor"u8 : "elsewhere"u8); + return ValueTask.CompletedTask; + } + + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs b/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs new file mode 100644 index 000000000..7104ecd0d --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/ReactorStartupTests.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Sockets; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.IO; +using GenHTTP.Modules.Layouting; + +using ioxide; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A reactor that cannot start fails the server start with its reason, instead of ending the +/// process from the reactor's thread or leaving a server that listens one reactor short. +/// +[TestClass] +public sealed class ReactorStartupTests +{ + + [IoxideTestMethod] + public async Task TestARingTheKernelRefusesFailsTheStart() + { + // Every kernel refuses a ring this large, so this fails the same way an old kernel does. + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2, RingEntries = 65_536 } }; + + var server = Build(options: options); + + var e = await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + // ioxide names the cause, and the start passes it on + StringAssert.Contains(e.Message, "32768"); + Assert.IsNotNull(e.InnerException); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestAThrowFromOnReactorStartFailsTheStart() + { + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var server = Build(_ => throw new InvalidOperationException("The reactor callback failed"), options); + + var e = await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + StringAssert.Contains(e.Message, "The reactor callback failed"); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestAPortTakenAlreadyFailsTheStart() + { + var port = (ushort)TestHost.NextPort(); + + // a listener without SO_REUSEPORT, which the reactors' listeners cannot share the port with + using var taken = new TcpListener(IPAddress.Any, port); + + taken.Start(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var server = Build(options: options, port: port); + + await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + await server.StopAsync(); + } + + [IoxideTestMethod] + public async Task TestTheReactorsServeOnceStarted() + { + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 2 } }; + + var port = (ushort)TestHost.NextPort(); + + var server = Build(options: options, port: port); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(); + + Assert.AreEqual("ok", await client.GetStringAsync($"http://localhost:{port}/ok")); + } + finally + { + await server.StopAsync(); + } + } + + [IoxideTestMethod] + public async Task TestASizeTheReactorCannotRunWithIsRefusedBeforeItRuns() + { + // ioxide refuses this in the reactor's constructor, which the engine reaches before starting + // any thread - so nothing is left half started to stop afterwards. + var options = new EngineOptions { Tcp = new TcpTransportOptions { RecvQueueEntries = 100 } }; + + var server = Build(options: options); + + await Assert.ThrowsExactlyAsync(async () => await server.StartAsync()); + + await server.StopAsync(); + } + + private static IServerHost Build(Action? onReactorStart = null, EngineOptions? options = null, ushort? port = null) + => Host.Create(onReactorStart, options) + .Handler(Layout.Create().Add("ok", Content.From(Resource.FromString("ok")))) + .Bind(IPAddress.Loopback, port ?? (ushort)TestHost.NextPort()); + +} diff --git a/Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs b/Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs new file mode 100644 index 000000000..ee1067a39 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/ReadTimeoutTests.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.Functional; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// The TCP read timeout runs only while the server waits on its client, so an idle keep-alive +/// connection is closed, while a handler that takes longer than the timeout keeps its connection. +/// +[TestClass] +public sealed class ReadTimeoutTests +{ + + private static readonly EngineOptions Options = new() + { + Reactor = new ReactorOptions { ReactorCount = 2 }, + Tcp = new TcpTransportOptions { ReadTimeoutMs = 500 }, + }; + + [IoxideTestMethod] + public async Task TestAnIdleConnectionIsClosed() + { + var app = Inline.Create().Get(() => "ok"); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create(options: Options).Handler(app).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var client = new TcpClient(); + + await client.ConnectAsync(IPAddress.Loopback, port); + + var stream = client.GetStream(); + + await stream.WriteAsync("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"u8.ToArray()); + + var response = await ReadResponseAsync(stream); + + StringAssert.StartsWith(response, "HTTP/1.1 200"); + + // nothing more is sent, so the connection waits on the client from here on + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(0, await stream.ReadAsync(new byte[1], timeout.Token), "The idle connection was not closed"); + } + finally + { + await server.StopAsync(); + } + } + + [IoxideTestMethod] + public async Task TestASlowHandlerIsNotCut() + { + var app = Inline.Create().Get(async () => + { + await Task.Delay(TimeSpan.FromSeconds(2)); + return "slow"; + }); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create(options: Options).Handler(app).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var client = new HttpClient(); + + Assert.AreEqual("slow", await client.GetStringAsync($"http://localhost:{port}/")); + } + finally + { + await server.StopAsync(); + } + } + + // Reads one response with a known length off the connection, leaving it open. + private static async Task ReadResponseAsync(NetworkStream stream) + { + var buffer = new byte[4096]; + var received = new StringBuilder(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + while (true) + { + var read = await stream.ReadAsync(buffer, timeout.Token); + + Assert.AreNotEqual(0, read, "The connection closed before the response arrived"); + + received.Append(Encoding.ASCII.GetString(buffer, 0, read)); + + var text = received.ToString(); + + var end = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + + if (end >= 0 && text.EndsWith("ok", StringComparison.Ordinal)) + { + return text; + } + } + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs b/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs index a9a3f99a9..5d05ae301 100644 --- a/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs +++ b/Testing/Acceptance/Engine/Ioxide/StreamedRequestBodyTests.cs @@ -50,6 +50,14 @@ public async Task TestBodyIsReadThroughTheArrayOverload() await RunEchoAsync(ReadThroughArrayOverload, RandomBytes(128 * 1024)); } + [IoxideTestMethod] + public async Task TestBodyOfUnknownLengthIsRead() + { + // No content-length, as HttpClient streams a body it cannot measure and fetch sends one from a + // ReadableStream: the handler still gets the body. + await RunEchoAsync(ReadWhole, RandomBytes(64 * 1024), announceLength: false); + } + #region Read strategies // Reads the body whole, exercising AsMemoryAsync and its chunk-assembly loop. @@ -95,7 +103,7 @@ private static async ValueTask ReadThroughArrayOverload(IRequestBody bod #region Infrastructure - private static async Task RunEchoAsync(Func> read, byte[] payload) + private static async Task RunEchoAsync(Func> read, byte[] payload, bool announceLength = true) { var port = (ushort)TestHost.NextPort(); @@ -116,7 +124,7 @@ private static async Task RunEchoAsync(Func> rea Timeout = TimeSpan.FromSeconds(10), }; - using var content = new ByteArrayContent(payload); + using HttpContent content = announceLength ? new ByteArrayContent(payload) : new UnknownLengthContent(payload); using var response = await client.PostAsync($"http://localhost:{port}/", content); diff --git a/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs new file mode 100644 index 000000000..5bd28abf9 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/StreamedResponseEndTests.cs @@ -0,0 +1,330 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Quic; +using System.Net.Security; +using System.Text; + +using GenHTTP.Api.Content; +using GenHTTP.Api.Infrastructure; +using GenHTTP.Api.Protocol; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.ServerSentEvents; + +using Microsoft.Extensions.Logging; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// How a streamed response ends over HTTP/2 and HTTP/3, where the end of the stream is all that tells +/// the client a response is complete: a client that leaves is noticed by the content still writing, +/// and a content that fails after the headers resets the stream rather than ending it cleanly. +/// +[TestClass] +public sealed class StreamedResponseEndTests +{ + + [IoxideTestMethod] + public async Task TestEventSourceNoticesTheClientLeftOverHttp2() + { + await AssertGeneratorEndsAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestEventSourceNoticesTheClientLeftOverHttp3() + { + await AssertGeneratorEndsAsync(HttpVersion.Version30); + } + + [IoxideTestMethod] + public async Task TestFailureAfterTheHeadersResetsTheHttp2Stream() + { + await AssertResponseFailsAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestFailureAfterTheHeadersResetsTheHttp3Stream() + { + await AssertResponseFailsAsync(HttpVersion.Version30); + } + + [IoxideTestMethod] + public async Task TestFaultReachingIoxideIsLoggedOverHttp2() + { + await AssertEscapedFaultIsLoggedAsync(HttpVersion.Version20); + } + + [IoxideTestMethod] + public async Task TestFaultReachingIoxideIsLoggedOverHttp3() + { + await AssertEscapedFaultIsLoggedAsync(HttpVersion.Version30); + } + + [IoxideTestMethod] + public async Task TestFailureBeforeTheHeadersIsStillA500OverHttp2() + { + // The pre-header path is unchanged: there is still time for an honest status. + await using var server = await ServerAsync(new FailingHandler(), HttpVersion.Version20); + + using var client = server.CreateClient(); + + using var response = await client.GetAsync(server.Url); + + Assert.AreEqual(HttpStatusCode.InternalServerError, response.StatusCode); + Assert.AreEqual(HttpVersion.Version20, response.Version); + } + + #region Infrastructure + + // A source that sends until a send fails, which only happens once the engine reports the client gone. + private static async Task AssertGeneratorEndsAsync(Version version) + { + var left = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var source = EventSource.Create() + .Generator(async c => + { + while (await c.CommentAsync("ping")) + { + await Task.Delay(50); + } + + Assert.IsFalse(c.Connected); + left.TrySetResult(); + }); + + await using var server = await ServerAsync(source.Build(), version); + + using (var client = server.CreateClient()) + { + using var response = await client.GetAsync(server.Url, HttpCompletionOption.ResponseHeadersRead); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + // wait for the first event, so the generator is running when the client leaves + await using var stream = await response.Content.ReadAsStreamAsync(); + + var buffer = new byte[1024]; + var received = ""; + + while (!received.Contains(": ping")) + { + var read = await stream.ReadAsync(buffer); + Assert.AreNotEqual(0, read); + received += Encoding.ASCII.GetString(buffer, 0, read); + } + } + + var finished = await Task.WhenAny(left.Task, Task.Delay(TimeSpan.FromSeconds(10))); + + Assert.AreSame(left.Task, finished, $"The generator kept sending after the HTTP/{version} client left"); + } + + // The headers and the first part of the body are out when the content fails. The response has to + // fail rather than end, since there is no content-length to tell the client the body is short. + // The fault goes to the server's logging, so ioxide must not print it to stderr as well. + private static async Task AssertResponseFailsAsync(Version version) + { + var stderr = await CaptureStandardErrorAsync(() => AssertTransferFailsAsync(version)); + + Assert.DoesNotContain(FailingMidBodyContent.Failure, stderr); + } + + // The engine's own report of the fault fails too, as the log sink refuses it, so the fault escapes + // to ioxide. ioxide hands it back to the server's logging rather than printing it to stderr. + private static async Task AssertEscapedFaultIsLoggedAsync(Version version) + { + var logging = new FailingSinkLoggerFactory(); + + var stderr = await CaptureStandardErrorAsync(() => AssertTransferFailsAsync(version, logging)); + + var protocol = version == HttpVersion.Version30 ? "HTTP/3" : "HTTP/2"; + + Assert.Contains($"Failed to complete {protocol} request", logging.Errors); + Assert.DoesNotContain(FailingSinkLoggerFactory.Failure, stderr); + } + + private static async Task CaptureStandardErrorAsync(Func action) + { + var stderr = new StringWriter(); + var original = Console.Error; + + Console.SetError(stderr); + + try + { + await action(); + } + finally + { + Console.SetError(original); + } + + return stderr.ToString(); + } + + // Where the response fails depends on the client: a reset may overtake the headers it has not read yet. + private static async Task AssertTransferFailsAsync(Version version, ILoggerFactory? logging = null) + { + await using var server = await ServerAsync(new FailingMidBodyHandler(), version, logging); + + using var client = server.CreateClient(); + + var failure = await Assert.ThrowsAsync(async () => + { + using var response = await client.GetAsync(server.Url, HttpCompletionOption.ResponseHeadersRead); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + await using var stream = await response.Content.ReadAsStreamAsync(); + + await stream.CopyToAsync(Stream.Null); + }); + + Assert.IsTrue(failure is HttpRequestException or IOException, $"The response did not fail as a transfer: {failure}"); + } + + private static async Task ServerAsync(IHandler handler, Version version, ILoggerFactory? logging = null) + { + var port = (ushort)TestHost.NextPort(); + + IServerHost host; + + if (version == HttpVersion.Version30) + { + if (!QuicConnection.IsSupported) + { + Assert.Inconclusive("QUIC is not supported on this machine (msquic missing)."); + } + + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + host = Host.Create().Handler(handler) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), httpProtocols: HttpProtocols.Http3); + } + else + { + host = Host.Create().Handler(handler) + .Bind(IPAddress.Loopback, port, HttpProtocols.Http2); + } + + if (logging != null) + { + host = host.Logging(logging); + } + + await host.StartAsync(); + + return new RunningServer(host, port, version); + } + + private sealed class RunningServer(IServerHost host, ushort port, Version version) : IAsyncDisposable + { + public string Url => version == HttpVersion.Version30 ? $"https://localhost:{port}/" : $"http://localhost:{port}/"; + + public HttpClient CreateClient() + { + var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + return new HttpClient(handler) + { + DefaultRequestVersion = version, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(10), + }; + } + + public async ValueTask DisposeAsync() => await host.StopAsync(); + } + + /// Answers with a body that fails once its first kilobyte is on the wire. + private sealed class FailingMidBodyHandler : IHandler + { + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) + => new(request.Respond() + .Status(ResponseStatus.Ok) + .Content(new FailingMidBodyContent()) + .Build()); + } + + /// Writes and flushes part of an unknown-length body, then fails. + private sealed class FailingMidBodyContent : IResponseContent + { + public const string Failure = "The content failed after its headers were sent"; + + public ulong? Length => null; + + public ContentType? Type => ContentType.ApplicationOctetStream; + + public ReadOnlyMemory? Encoding => null; + + public ValueTask CalculateChecksumAsync() => new((ulong?)null); + + public async ValueTask WriteAsync(IResponseSink sink) + { + await sink.Stream.WriteAsync(new byte[1024]); + await sink.Stream.FlushAsync(); + + throw new InvalidOperationException(Failure); + } + } + + /// A log sink that refuses the first error it is given, as one whose disk is full would. + private sealed class FailingSinkLoggerFactory : ILoggerFactory + { + public const string Failure = "The log sink refused the entry"; + + private int _refused; + + public ConcurrentQueue Errors { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(this); + + public void AddProvider(ILoggerProvider provider) { } + + public void Dispose() { } + + private sealed class Logger(FailingSinkLoggerFactory factory) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (logLevel < LogLevel.Error) + { + return; + } + + if (Interlocked.Exchange(ref factory._refused, 1) == 0) + { + throw new IOException(Failure); + } + + factory.Errors.Enqueue(formatter(state, exception)); + } + } + } + + /// Fails before it has a response at all. + private sealed class FailingHandler : IHandler + { + public ValueTask PrepareAsync(IServer server) => ValueTask.CompletedTask; + + public ValueTask HandleAsync(IRequest request) => throw new InvalidOperationException("The handler failed"); + } + + #endregion + +} diff --git a/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs b/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs new file mode 100644 index 000000000..253f9cb89 --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/TlsBodyTests.cs @@ -0,0 +1,97 @@ +using System.Net; +using System.Net.Security; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Testing.Acceptance.Engine.Body; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// Request bodies over TLS, streamed straight back. The TLS reader decrypts into a pipe that reuses +/// its segments once they have been read, so this is where a body reader that still touches memory +/// it has handed back shows: an echoed chunked body used to come back short. +/// +[TestClass] +public sealed class TlsBodyTests +{ + private static readonly int[] Sizes = [64 * 1024, 100 * 1024, 1024 * 1024]; + + private static readonly int[] Writes = [1, 100, 4095, 4097, 16_384, 65_536, 100_000]; + + [IoxideTestMethod] + public async Task TestChunkedBodiesOverHttp1() + { + await RunAsync(HttpProtocols.Http1, HttpVersion.Version11, chunked: true); + } + + [IoxideTestMethod] + public async Task TestBodiesOverHttp2() + { + // With a Content-Length: the engine treats an HTTP/2 request without one as having no body + await RunAsync(HttpProtocols.Http1AndHttp2, HttpVersion.Version20, chunked: false); + } + + private static async Task RunAsync(HttpProtocols protocols, Version version, bool chunked) + { + var (certificate, key) = IoxideCertificates.Create("localhost", IoxideCertificates.Isolated()); + + var port = (ushort)TestHost.NextPort(); + + var server = Host.Create() + .Handler(new StreamingEchoHandler().Wrap()) + .Bind(IPAddress.Loopback, port, new FileCertificateProvider(certificate, key), httpProtocols: protocols); + + await server.StartAsync(); + + try + { + using var handler = new SocketsHttpHandler + { + SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }, + }; + + using var client = new HttpClient(handler); + + // Each size a few times over, as how the data happens to arrive decides what the reader sees + foreach (var size in Sizes) + { + for (var attempt = 1; attempt <= 5; attempt++) + { + var payload = new byte[size]; + new Random(size + attempt).NextBytes(payload); + + var request = new HttpRequestMessage(HttpMethod.Post, $"https://localhost:{port}/") + { + Content = new UnevenContent(payload, Writes, declareLength: !chunked), + Version = version, + VersionPolicy = HttpVersionPolicy.RequestVersionExact + }; + + if (chunked) + { + request.Headers.TransferEncodingChunked = true; + } + + using var response = await client.SendAsync(request); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(version, response.Version); + + var echoed = await response.Content.ReadAsByteArrayAsync(); + + Assert.IsTrue(payload.AsSpan().SequenceEqual(echoed), $"{size} bytes, attempt {attempt}: {echoed.Length} came back"); + } + } + } + finally + { + await server.StopAsync(); + } + } + +} diff --git a/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs b/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs new file mode 100644 index 000000000..f0a7b1eef --- /dev/null +++ b/Testing/Acceptance/Engine/Ioxide/UnknownLengthContent.cs @@ -0,0 +1,21 @@ +using System.Net; + +namespace GenHTTP.Testing.Acceptance.Engine.Ioxide; + +/// +/// A request body sent without a content-length, the way a client streams one it cannot measure up +/// front: over HTTP/2 and HTTP/3, only the end of the stream says where it stops. +/// +internal sealed class UnknownLengthContent(byte[] payload) : HttpContent +{ + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => stream.WriteAsync(payload).AsTask(); + + protected override bool TryComputeLength(out long length) + { + length = -1; + return false; + } + +} diff --git a/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs b/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs index f02098975..f064f8775 100644 --- a/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs +++ b/Testing/Acceptance/Modules/Authentication/ClientCertificateAuthenticationTests.cs @@ -121,7 +121,7 @@ private static async ValueTask TestAsync(ClientCertificateA { if (engine == ServerEngine.Ioxide && validator.RequireCertificate) { - // ioxide validates the client chain in OpenSSL, which wants the trust anchors before the + // ioxide validates the client chain in BoringSSL, which wants the trust anchors before the // handshake begins - so requiring a certificate there means naming what it is validated // against. ConfigurableValidator is a plain ICertificateValidator and names none, which // the engine refuses when the server starts rather than admitting a client it never diff --git a/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs b/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs index 03f4e51dc..9b48f04f2 100644 --- a/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs +++ b/Testing/Acceptance/Modules/Websockets/Integration/Imperative/FrameEdgeCaseTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Text; using GenHTTP.Api.Infrastructure; using GenHTTP.Modules.Websockets; @@ -86,6 +87,183 @@ public async Task TestTruncatedFrameAtEofYieldsError() Assert.AreEqual(FrameError.UnexpectedEndOfStream, await client.ReceiveTextFrameAsync()); } + [TestMethod] + [MultiEngineTest] + public async Task TestDrainedPingKeepsSegmentedMessage(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Arriving in one read, the ping and the final fragment are both drained with TryReadFrame + var combined = RawWebSocketClient.BuildClientFrame("Hello "u8, opcode: 0x1, fin: false) + .Concat(RawWebSocketClient.BuildClientFrame("ping-data"u8, opcode: 0x9, fin: true)) + .Concat(RawWebSocketClient.BuildClientFrame("World"u8, opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + var (opcode, _, payload) = await client.ReceiveFrameAsync(timeout.Token); + + Assert.AreEqual((byte)0xA, opcode); + Assert.AreEqual("ping-data", Encoding.UTF8.GetString(payload)); + + Assert.AreEqual("Hello World", await client.ReceiveTextFrameAsync(timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestDrainedFragmentedMessageIsReassembled(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var combined = RawWebSocketClient.BuildClientFrame("one"u8, opcode: 0x1, fin: true) + .Concat(RawWebSocketClient.BuildClientFrame("Hello "u8, opcode: 0x1, fin: false)) + .Concat(RawWebSocketClient.BuildClientFrame("World"u8, opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + Assert.AreEqual("one", await client.ReceiveTextFrameAsync(timeout.Token)); + Assert.AreEqual("Hello World", await client.ReceiveTextFrameAsync(timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestInvalidUtf8FailsConnection(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // 0xC3 starts a two-byte sequence that 0x28 does not continue: each fragment alone could still be valid + var combined = RawWebSocketClient.BuildClientFrame([0xC3], opcode: 0x1, fin: false) + .Concat(RawWebSocketClient.BuildClientFrame([0x28], opcode: 0x0, fin: true)) + .ToArray(); + + await client.SendRawInChunksAsync(combined, chunkSize: combined.Length, timeout.Token); + + Assert.AreEqual(1007, await ReceiveCloseStatusAsync(client, timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestContinuationWithoutMessageFailsConnection(ServerEngine engine) + { + await using var host = await TestHost.RunAsync(GenHTTP.Modules.Websockets.Websocket.Imperative().Handler(new DrainingEchoHandler()), engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var frame = RawWebSocketClient.BuildClientFrame("orphan"u8, opcode: 0x0, fin: true); + + await client.SendRawInChunksAsync(frame, chunkSize: frame.Length, timeout.Token); + + Assert.AreEqual(1002, await ReceiveCloseStatusAsync(client, timeout.Token)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestMaxFrameSizeAcceptsLargerFrames(ServerEngine engine) + { + var websocket = GenHTTP.Modules.Websockets.Websocket.Imperative() + .MaxFrameSize(64 * 1024) + .Handler(new DrainingEchoHandler()); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + await using var client = new RawWebSocketClient(); + await client.ConnectAsync("127.0.0.1", host.Port); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var message = new string('a', 64 * 1024); + var frame = RawWebSocketClient.BuildClientFrame(Encoding.UTF8.GetBytes(message), opcode: 0x1, fin: true); + + await client.SendRawInChunksAsync(frame, chunkSize: frame.Length, timeout.Token); + + Assert.AreEqual(message, await client.ReceiveTextFrameAsync(timeout.Token)); + } + + private static async Task ReceiveCloseStatusAsync(RawWebSocketClient client, CancellationToken token) + { + var (opcode, _, payload) = await client.ReceiveFrameAsync(token); + + Assert.AreEqual((byte)0x8, opcode); + + return BinaryPrimitives.ReadUInt16BigEndian(payload); + } + + /// + /// Reads the way a high-throughput handler does: one awaited read, then every frame already + /// buffered with TryReadFrame, then one flush. Protocol errors close with their status code. + /// + private sealed class DrainingEchoHandler : IImperativeHandler + { + public async ValueTask HandleAsync(IImperativeConnection connection) + { + while (true) + { + var frame = await connection.ReadFrameAsync(); + + if (!await HandleAsync(connection, frame)) return; + + while (connection.TryReadFrame(out frame)) + { + if (!await HandleAsync(connection, frame)) return; + } + + await connection.FlushAsync(); + } + } + + private static async ValueTask HandleAsync(IImperativeConnection connection, IWebsocketFrame frame) + { + if (frame.IsError(out var error)) + { + var status = error.CloseStatus; + + if (status != null) + { + await connection.CloseAsync(statusCode: status.Value); + } + + return false; + } + + switch (frame.Type) + { + case FrameType.Close: + await connection.CloseAsync(); + return false; + + case FrameType.Ping: + await connection.PongAsync(frame.Data, flush: false); + return true; + + case FrameType.Text or FrameType.Binary: + await connection.WriteAsync(frame.Data, frame.Type, flush: false); + return true; + + default: + return true; + } + } + } + private sealed class EchoAllHandler : IImperativeHandler { public async ValueTask HandleAsync(IImperativeConnection connection) diff --git a/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs b/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs new file mode 100644 index 000000000..500143829 --- /dev/null +++ b/Testing/Acceptance/Modules/Websockets/LateWriteTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using System.Net.WebSockets; +using System.Text; + +using GenHTTP.Api.Infrastructure; + +using GenHTTP.Engine.Ioxide; + +using GenHTTP.Modules.Websockets; + +using GenHTTP.Testing.Acceptance.Engine.Ioxide; + +using Host = GenHTTP.Engine.Ioxide.Host; + +namespace GenHTTP.Testing.Acceptance.Modules.Websockets; + +/// +/// A write made after the handler has ended, as a broadcaster or a timer started from OnConnected +/// makes once the client has gone, is refused instead of reaching a sink the engine is done with. +/// +[TestClass] +public sealed class LateWriteTests +{ + + [TestMethod] + [MultiEngineTest] + public async Task TestAWriteAfterTheHandlerHasEndedIsRefused(ServerEngine engine) + { + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .OnConnected(connection => + { + connected.TrySetResult(connection); + return ValueTask.CompletedTask; + }); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + using var client = new ClientWebSocket(); + + await client.ConnectAsync(new Uri($"ws://localhost:{host.Port}"), cts.Token); + + var connection = await connected.Task.WaitAsync(cts.Token); + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + + // The handler ends just after the close handshake. Until then a write may still fail on the transport instead. + Exception? refused = null; + + while (refused is not ConnectionClosedException && !cts.IsCancellationRequested) + { + try + { + await connection.WriteAsync("late"u8.ToArray()); + await Task.Delay(10); + } + catch (Exception e) + { + refused = e; + } + } + + Assert.IsInstanceOfType(refused, $"the late write was not refused: {refused}"); + } + + [IoxideTestMethod] + public async Task TestALateWriteDoesNotReachTheConnectionThatTookItsPlace() + { + // ioxide pools its connections, so on the only reactor a second client takes the first one's place, and a + // write still going to the first one's sink would land in the second client's stream. + var first = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .OnConnected(connection => + { + first.TrySetResult(connection); + return ValueTask.CompletedTask; + }) + .OnMessage((connection, message) => connection.WriteAsync(message.Data)); + + var port = (ushort)TestHost.NextPort(); + + var options = new EngineOptions { Reactor = new ReactorOptions { ReactorCount = 1 } }; + + var server = Host.Create(options: options).Handler(websocket).Bind(IPAddress.Loopback, port); + + await server.StartAsync(); + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + IReactiveConnection late; + + using (var client = new ClientWebSocket()) + { + await client.ConnectAsync(new Uri($"ws://localhost:{port}"), cts.Token); + + late = await first.Task.WaitAsync(cts.Token); + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + } + + // Time for the reactor to take the first connection back into its pool. + await Task.Delay(200, cts.Token); + + using var second = new ClientWebSocket(); + + await second.ConnectAsync(new Uri($"ws://localhost:{port}"), cts.Token); + + Exception? refused = null; + + try + { + await late.WriteAsync("late"u8.ToArray()); + } + catch (Exception e) + { + refused = e; + } + + await second.SendAsync("mine"u8.ToArray(), WebSocketMessageType.Text, true, cts.Token); + + var buffer = new byte[64]; + + var result = await second.ReceiveAsync(buffer, cts.Token); + + Assert.AreEqual("mine", Encoding.UTF8.GetString(buffer, 0, result.Count), "the second client received another connection's frame"); + + Assert.IsInstanceOfType(refused, $"the late write was not refused: {refused}"); + } + finally + { + await server.StopAsync(); + } + } + +} diff --git a/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs b/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs index ecdbedcae..7cc6a0d24 100644 --- a/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs +++ b/Testing/Acceptance/Modules/Websockets/Protocol/Frame.Decode.Tests.cs @@ -5,6 +5,7 @@ using GenHTTP.Modules.Websockets; using GenHTTP.Modules.Websockets.Protocol; +using GenHTTP.Modules.Websockets.Utils; namespace GenHTTP.Testing.Acceptance.Modules.Websockets.Protocol; @@ -327,6 +328,78 @@ public void Decode_IncompleteExtendedFrame_ReturnsIncompleteError() Assert.AreEqual(sequence.End, examined); } + [TestMethod] + public void Decode_ReservedBitSet_ReturnsInvalidReservedBitsError() + { + // FIN=1, RSV1=1, Text "hi" - no extension was negotiated that would give RSV1 a meaning + var seq = new ReadOnlySequence(new byte[] { 0xC1, 0x02, (byte)'h', (byte)'i' }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidReservedBits); + } + + [TestMethod] + public void Decode_CloseFrame_OneBytePayload_ReturnsInvalidCloseFrameError() + { + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x01, 0x03 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidCloseFrame); + } + + [TestMethod] + public void Decode_CloseFrame_CodeNotToBeSent_ReturnsInvalidCloseFrameError() + { + // 1005 (no status received) is reserved for local use and never sent in a frame + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x02, 0x03, 0xED }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidCloseFrame); + } + + [TestMethod] + public void Decode_CloseFrame_InvalidUtf8Reason_ReturnsInvalidUtf8Error() + { + // 1000 followed by 0xC3 0x28, a lead byte without its continuation byte + var seq = new ReadOnlySequence(new byte[] { 0x88, 0x04, 0x03, 0xE8, 0xC3, 0x28 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidUtf8); + } + + [TestMethod] + public void Decode_TextFrame_InvalidUtf8_ReturnsInvalidUtf8Error() + { + var seq = new ReadOnlySequence(new byte[] { 0x81, 0x02, 0xC3, 0x28 }); + + var result = Frame.Decode(new MockConnection(), ref seq, DefaultRxMaxBufferSize, out _, out _); + + AssertError(result, FrameErrorType.InvalidUtf8); + } + + [TestMethod] + public void IsValidUtf8_CodePointAcrossSegments() + { + // "é" is 0xC3 0xA9: split across two segments it is still valid, while 0xC3 0x28 is not + Assert.IsTrue(Frame.IsValidUtf8(TwoSegments([(byte)'a', 0xC3], [0xA9, (byte)'b']))); + Assert.IsFalse(Frame.IsValidUtf8(TwoSegments([(byte)'a', 0xC3], [0x28, (byte)'b']))); + + // a sequence cut off at the end of the payload + Assert.IsFalse(Frame.IsValidUtf8(TwoSegments([(byte)'a'], [0xC3]))); + } + + private static ReadOnlySequence TwoSegments(byte[] first, byte[] second) + { + var start = new BufferSegment(first); + var end = start.Append(second); + + return new ReadOnlySequence(start, 0, end, second.Length); + } + private static void AssertError(IWebsocketFrame frame, FrameErrorType expected) { Assert.AreEqual(FrameType.Error, frame.Type); diff --git a/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs new file mode 100644 index 000000000..7e90b8758 --- /dev/null +++ b/Testing/Acceptance/Modules/Websockets/SynchronizedWriteTests.cs @@ -0,0 +1,104 @@ +using System.Collections.Concurrent; +using System.Net.WebSockets; +using System.Text; + +using GenHTTP.Api.Infrastructure; + +namespace GenHTTP.Testing.Acceptance.Modules.Websockets; + +/// +/// A connection serializes every write by default, the framework's own pongs included, so several +/// threads can write while the client keeps pinging. +/// +[TestClass] +public sealed class SynchronizedWriteTests +{ + private const int Writers = 4; + + private const int Messages = 250; + + [TestMethod] + [MultiEngineTest] + public async Task TestSeveralThreadsWriteWhileTheClientPings(ServerEngine engine) + { + var faults = new ConcurrentQueue(); + + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .OnConnected(connection => + { + for (var w = 0; w < Writers; w++) + { + var writer = w; + + _ = Task.Run(async () => + { + try + { + for (var m = 0; m < Messages; m++) + { + await connection.WriteAsync(Encoding.UTF8.GetBytes($"{writer}:{m}")); + } + } + catch (Exception e) + { + faults.Enqueue(e); + } + }); + } + + return ValueTask.CompletedTask; + }); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + using var client = new ClientWebSocket(); + + // pings the server keeps answering from its message pump, while the writers write + client.Options.KeepAliveInterval = TimeSpan.FromMilliseconds(5); + client.Options.KeepAliveTimeout = TimeSpan.FromSeconds(10); + + await client.ConnectAsync(new Uri($"ws://localhost:{host.Port}"), cts.Token); + + var next = new int[Writers]; + var buffer = new byte[256]; + + for (var i = 0; i < Writers * Messages; i++) + { + var result = await client.ReceiveAsync(buffer, cts.Token); + + Assert.AreEqual(WebSocketMessageType.Text, result.MessageType); + Assert.IsTrue(result.EndOfMessage); + + var parts = Encoding.UTF8.GetString(buffer, 0, result.Count).Split(':'); + + var writer = int.Parse(parts[0]); + + // each writer's messages arrive whole and in the order it sent them + Assert.AreEqual(next[writer]++, int.Parse(parts[1])); + } + + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", cts.Token); + + Assert.IsTrue(faults.IsEmpty, string.Join(Environment.NewLine, faults)); + } + + [TestMethod] + [MultiEngineTest] + public async Task TestConnectionWritesWithoutSynchronization(ServerEngine engine) + { + // one writer at a time, as the option requires: the message pump echoing and answering pings + var websocket = GenHTTP.Modules.Websockets.Websocket.Functional() + .DisableWriteSynchronization() + .HandleContinuationFramesManually() + .OnConnected(c => c.PingAsync()) + .OnMessage((c, m) => c.WriteAsync(m.Data)) + .OnContinue((c, m) => c.WriteAsync(m.Data)); + + await using var host = await TestHost.RunAsync(websocket, engine: engine); + + await Integration.Client.Execute(host.Port); + } + +}