diff --git a/modules/http3/README.md b/modules/http3/README.md index 43c5cf12861..72729a5032a 100644 --- a/modules/http3/README.md +++ b/modules/http3/README.md @@ -144,3 +144,17 @@ curl --alt-svc /tmp/altsvc.cache -v https://localhost:4433/ - **Multi-Process UDP Listener**: In standard multi-process MPMs, one child process manages the UDP listening socket while others coordinate in the background. - **Connection Migration**: QUIC connection migration (RFC 9000 ยง9) is currently subject to OpenSSL QUIC stack capabilities (see [`QUIC_CONNECTION_MIGRATION.md`](QUIC_CONNECTION_MIGRATION.md)). - **TLS Certificates**: Modern browsers reject self-signed or invalid certificates for HTTP/3 more strictly than for HTTP/1.1 or HTTP/2. Use trusted certificates or valid local CA certificates during testing. + +--- + +## Choosing the QUIC transport (`--with-http3-quic`) + +Measured with `h3bench` (`openssl-h3-examples`), one run per transport on the same machine, so only compare numbers from the same run: + +- **Idle connections:** `ngtcp2` holds 4000 idle connections with no errors at about 50% CPU. `openssl` uses about 4 times the CPU at 1000 connections, gets errors at 2000 and none connect at 4000. +- **Loaded, 1 to 32 connections:** `openssl` serves about 30 to 60% more requests per second, with lower p50 and p99 latency (for example 32 connections of 32 streams: 875 against 577 rps, p50 1.5 s against 2.3 s). +- **One connection, one request at a time (`1x1`):** `ngtcp2` answers in about 1 ms. `openssl` stalls (0.2 rps, p50 over 1 s); the cause is not found yet. +- **Not the transport:** both servers sit near 2000% CPU in every loaded case, even `1x1`, so something outside the QUIC layer, probably the event loop or the worker threads, uses CPU it should not. Look at that before reading much into the throughput gap. +- **Features:** only `ngtcp2` has 0-RTT, routing by connection ID across child processes and connection migration. + +Use `ngtcp2` (the default) unless you need to avoid the ngtcp2 dependency. diff --git a/modules/http3/config2.m4 b/modules/http3/config2.m4 index 2805ec72740..42a80cb4d34 100644 --- a/modules/http3/config2.m4 +++ b/modules/http3/config2.m4 @@ -34,11 +34,10 @@ h3_socket_os.lo dnl h3_socket_bpf.lo dnl h3_stream.lo dnl h3_threads.lo dnl -h3q.lo dnl -h3q_conn.lo dnl -h3q_stream.lo dnl h3q_tls.lo dnl " +dnl # The QUIC transport objects depend on --with-http3-quic and are added +dnl # to http3_objs by the module's prerequisite checks below. dnl dnl APACHE_CHECK_NGHTTP3 @@ -205,11 +204,36 @@ AC_DEFUN([APACHE_CHECK_NGTCP2],[ ]) ]) +dnl +dnl APACHE_CHECK_H3_OSSL_QUIC +dnl +dnl Check that OpenSSL has its own QUIC server (3.5 or later), for +dnl "--with-http3-quic=openssl". OpenSSL itself is found by +dnl APACHE_CHECK_OPENSSL. +dnl +AC_DEFUN([APACHE_CHECK_H3_OSSL_QUIC],[ + AC_CACHE_CHECK([for the OpenSSL QUIC server API], [ac_cv_h3_ossl_quic], [ + ac_cv_h3_ossl_quic=no + saved_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $MOD_CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]],[[ +#if OPENSSL_VERSION_NUMBER < 0x30500000L +#error "OpenSSL 3.5 or later is required" +#endif +(void)SSL_new_listener; +(void)OSSL_QUIC_server_method;]])], + [ac_cv_h3_ossl_quic=yes]) + CPPFLAGS="$saved_CPPFLAGS" + ]) +]) + dnl # hook module into the Autoconf mechanism (--enable-http3) APACHE_MODULE(http3, [HTTP/3 protocol handling in addition to HTTP protocol -handling. Implemented by mod_http3. This module requires libnghttp3, -libngtcp2 with its OpenSSL crypto helper, and OpenSSL with the QUIC TLS API. -See --with-nghttp3 and --with-ngtcp2 on how to manage non-standard locations. This module +handling. Implemented by mod_http3. This module requires libnghttp3 and +either libngtcp2 with its OpenSSL crypto helper and OpenSSL with the QUIC TLS +API (the default), or OpenSSL 3.5 or later with its own QUIC server +(--with-http3-quic=openssl). See --with-nghttp3 and --with-ngtcp2 on how to +manage non-standard locations. This module is usually linked shared and requires loading. ], $http3_objs, , no, [ APACHE_CHECK_OPENSSL if test "$ac_cv_openssl" = "yes" ; then @@ -219,9 +243,35 @@ is usually linked shared and requires loading. ], $http3_objs, , no, [ fi if test "$enable_http3" != "no" ; then + AC_ARG_WITH(http3-quic, APACHE_HELP_STRING(--with-http3-quic=ngtcp2|openssl, [QUIC transport of mod_http3: ngtcp2 (default; supports 0-RTT and connection ID routing) or the OpenSSL QUIC server]), [ + ap_h3_quic="$withval" + ], [ + ap_h3_quic=ngtcp2 + ]) + if test "x$ap_h3_quic" = "xyes" -o "x$ap_h3_quic" = "x"; then + ap_h3_quic=ngtcp2 + fi + if test "x$ap_h3_quic" != "xngtcp2" -a "x$ap_h3_quic" != "xopenssl"; then + AC_MSG_ERROR([--with-http3-quic must be ngtcp2 or openssl, not $ap_h3_quic]) + fi + APACHE_CHECK_NGHTTP3 - APACHE_CHECK_NGTCP2 - if test "$ac_cv_nghttp3" = "yes" -a "$ac_cv_ngtcp2" = "yes" ; then + ap_h3_quic_ok=no + if test "$ap_h3_quic" = "openssl"; then + APACHE_CHECK_H3_OSSL_QUIC + if test "$ac_cv_h3_ossl_quic" = "yes"; then + ap_h3_quic_ok=yes + http3_objs="$http3_objs h3q_ossl.lo h3q_ossl_conn.lo h3q_ossl_stream.lo h3q_ossl_addr.lo" + APR_ADDTO(MOD_CPPFLAGS, ["-DH3Q_BACKEND_OSSL"]) + fi + else + APACHE_CHECK_NGTCP2 + if test "$ac_cv_ngtcp2" = "yes"; then + ap_h3_quic_ok=yes + http3_objs="$http3_objs h3q.lo h3q_conn.lo h3q_stream.lo" + fi + fi + if test "$ac_cv_nghttp3" = "yes" -a "$ap_h3_quic_ok" = "yes" ; then APR_ADDTO(MOD_CFLAGS, [-I\$(top_srcdir)/$modpath_current/include]) if test "x$enable_http3" = "xshared"; then # The only symbol which needs to be exported is the module diff --git a/modules/http3/h3_config.c b/modules/http3/h3_config.c index ecaed9468d5..2cd85a82730 100644 --- a/modules/http3/h3_config.c +++ b/modules/http3/h3_config.c @@ -537,6 +537,12 @@ static const char* set_h3_session_tickets(cmd_parms* cmd, void* dummy H3_UNUSED, static const char* set_h3_early_data(cmd_parms* cmd, void* dummy H3_UNUSED, int flag) { +#ifdef H3Q_BACKEND_OSSL + if (flag) + { + return "H3EarlyData is not supported when mod_http3 is built with --with-http3-quic=openssl"; + } +#endif h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module); CHECK(conf); conf->h3_early_data = flag ? H3_FLAG_ON : H3_FLAG_OFF; diff --git a/modules/http3/h3_server.c b/modules/http3/h3_server.c index aa5960db029..baf832d20f8 100644 --- a/modules/http3/h3_server.c +++ b/modules/http3/h3_server.c @@ -73,6 +73,10 @@ typedef struct * ponytail: top-level directive only, not one inside . */ static int listen_buckets(void) { +#ifdef H3Q_BACKEND_OSSL + /* Routing by connection ID needs ngtcp2 to put the bucket in the connection IDs. */ + return 1; +#endif for (ap_directive_t* d = ap_conftree; d; d = d->next) { if (ap_have_so_reuseport > 0 && !ap_cstr_casecmp(d->directive, "ListenCoresBucketsRatio")) diff --git a/modules/http3/h3_session.c b/modules/http3/h3_session.c index e89f98bd05e..262f68c6064 100644 --- a/modules/http3/h3_session.c +++ b/modules/http3/h3_session.c @@ -100,8 +100,14 @@ apr_status_t h3_session_create(h3_session** psession, server_rec* s, h3q_conn* q } nghttp3_conn_set_max_concurrent_streams(session->ngh3, conf->h3_max_concurrent_streams); - nghttp3_conn_set_max_client_streams_bidi(session->ngh3, conf->h3_max_concurrent_streams); +#ifdef H3Q_BACKEND_OSSL + /* OpenSSL raises the peer's stream limit by itself and does not report it: + * give nghttp3 a limit that no client reaches. */ + session->max_client_streams_bidi = (uint64_t)1 << 30; +#else session->max_client_streams_bidi = conf->h3_max_concurrent_streams; +#endif + nghttp3_conn_set_max_client_streams_bidi(session->ngh3, session->max_client_streams_bidi); h3q_tls_info tls = {0}; if (h3q_conn_tls_info(qconn, &tls)) diff --git a/modules/http3/h3_stream.c b/modules/http3/h3_stream.c index 149e8b29fbc..77e9d6c058a 100644 --- a/modules/http3/h3_stream.c +++ b/modules/http3/h3_stream.c @@ -123,6 +123,10 @@ void flush_nghttp3(h3_session* session) continue; } nghttp3_conn_add_write_offset(session->ngh3, sid, res.accepted); +#ifdef H3Q_BACKEND_OSSL + /* OpenSSL copies what it accepts and reports no acknowledgements. */ + nghttp3_conn_add_ack_offset(session->ngh3, sid, res.accepted); +#endif if (res.blocked) { /* Send buffer full: skip this stream instead of busy-looping on the same vec. diff --git a/modules/http3/include/quic/h3q.h b/modules/http3/include/quic/h3q.h index ea2218525cb..e6597a46639 100644 --- a/modules/http3/include/quic/h3q.h +++ b/modules/http3/include/quic/h3q.h @@ -30,6 +30,15 @@ typedef struct h3q_stream h3q_stream; #define H3Q_ERRLEN 256 +/** + * The QUIC transport behind this API is chosen at build time with + * --with-http3-quic: ngtcp2 (the default, which defines nothing) or OpenSSL's + * own QUIC stack, which defines H3Q_BACKEND_OSSL. The OpenSSL backend copies + * written stream bytes, so they count as acknowledged once accepted; it has no + * 0-RTT, no stateless-reset/Retry secret, no window autotuning cap and no + * connection ID routing, and ignores those h3q_config fields. + */ + /** Everything the listener needs to exist. */ typedef struct h3q_config { diff --git a/modules/http3/quic/detail/h3q_tls.c b/modules/http3/quic/detail/h3q_tls.c index b4ff7929b76..d1c8c4c97f8 100644 --- a/modules/http3/quic/detail/h3q_tls.c +++ b/modules/http3/quic/detail/h3q_tls.c @@ -23,7 +23,9 @@ #include #include +#ifndef H3Q_BACKEND_OSSL #include +#endif #include #include @@ -83,17 +85,26 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const CHECK(cert_files && ncerts > 0); CHECK(key_files || nkeys == 0); +#ifdef H3Q_BACKEND_OSSL + /* OpenSSL runs the whole QUIC transport. */ + SSL_CTX* ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method()); +#else + /* ngtcp2 runs the transport; OpenSSL only does the TLS 1.3 handshake. */ if (ngtcp2_crypto_ossl_init() != 0) { h3q_tls_error(err, errlen, "ngtcp2_crypto_ossl_init failed; OpenSSL lacks the QUIC TLS API"); return NULL; } SSL_CTX* ssl_ctx = SSL_CTX_new(TLS_server_method()); +#endif if (!ssl_ctx) { h3q_tls_error(err, errlen, "SSL_CTX_new failed"); return NULL; } +#ifdef H3Q_BACKEND_OSSL + SSL_CTX_set_domain_flags(ssl_ctx, SSL_DOMAIN_FLAG_MULTI_THREAD); +#endif SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); @@ -117,7 +128,9 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const * with it off invites none. OpenSSL's anti-replay is built for TLS over TCP * and refuses QUIC resumption with early data; QUIC 0-RTT replay safety comes * from accepting only safe methods before the handshake (RFC 8470), in h3_io. */ +#ifndef H3Q_BACKEND_OSSL SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_ANTI_REPLAY); +#endif if (!session_tickets) {