Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions modules/http3/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,17 @@ curl --alt-svc /tmp/altsvc.cache -v https://localhost:4433/
- **Multi-Process UDP Listener**: In standard multi-process MPMs, one child process manages the UDP listening socket while others coordinate in the background.
- **Connection Migration**: QUIC connection migration (RFC 9000 §9) is currently subject to OpenSSL QUIC stack capabilities (see [`QUIC_CONNECTION_MIGRATION.md`](QUIC_CONNECTION_MIGRATION.md)).
- **TLS Certificates**: Modern browsers reject self-signed or invalid certificates for HTTP/3 more strictly than for HTTP/1.1 or HTTP/2. Use trusted certificates or valid local CA certificates during testing.

---

## Choosing the QUIC transport (`--with-http3-quic`)

Measured with `h3bench` (`openssl-h3-examples`), one run per transport on the same machine, so only compare numbers from the same run:

- **Idle connections:** `ngtcp2` holds 4000 idle connections with no errors at about 50% CPU. `openssl` uses about 4 times the CPU at 1000 connections, gets errors at 2000 and none connect at 4000.
- **Loaded, 1 to 32 connections:** `openssl` serves about 30 to 60% more requests per second, with lower p50 and p99 latency (for example 32 connections of 32 streams: 875 against 577 rps, p50 1.5 s against 2.3 s).
- **One connection, one request at a time (`1x1`):** `ngtcp2` answers in about 1 ms. `openssl` stalls (0.2 rps, p50 over 1 s); the cause is not found yet.
- **Not the transport:** both servers sit near 2000% CPU in every loaded case, even `1x1`, so something outside the QUIC layer, probably the event loop or the worker threads, uses CPU it should not. Look at that before reading much into the throughput gap.
- **Features:** only `ngtcp2` has 0-RTT, routing by connection ID across child processes and connection migration.

Use `ngtcp2` (the default) unless you need to avoid the ngtcp2 dependency.
66 changes: 58 additions & 8 deletions modules/http3/config2.m4
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,10 @@ h3_socket_os.lo dnl
h3_socket_bpf.lo dnl
h3_stream.lo dnl
h3_threads.lo dnl
h3q.lo dnl
h3q_conn.lo dnl
h3q_stream.lo dnl
h3q_tls.lo dnl
"
dnl # The QUIC transport objects depend on --with-http3-quic and are added
dnl # to http3_objs by the module's prerequisite checks below.

dnl
dnl APACHE_CHECK_NGHTTP3
Expand Down Expand Up @@ -205,11 +204,36 @@ AC_DEFUN([APACHE_CHECK_NGTCP2],[
])
])

dnl
dnl APACHE_CHECK_H3_OSSL_QUIC
dnl
dnl Check that OpenSSL has its own QUIC server (3.5 or later), for
dnl "--with-http3-quic=openssl". OpenSSL itself is found by
dnl APACHE_CHECK_OPENSSL.
dnl
AC_DEFUN([APACHE_CHECK_H3_OSSL_QUIC],[
AC_CACHE_CHECK([for the OpenSSL QUIC server API], [ac_cv_h3_ossl_quic], [
ac_cv_h3_ossl_quic=no
saved_CPPFLAGS="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $MOD_CFLAGS"
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <openssl/ssl.h>]],[[
#if OPENSSL_VERSION_NUMBER < 0x30500000L
#error "OpenSSL 3.5 or later is required"
#endif
(void)SSL_new_listener;
(void)OSSL_QUIC_server_method;]])],
[ac_cv_h3_ossl_quic=yes])
CPPFLAGS="$saved_CPPFLAGS"
])
])

dnl # hook module into the Autoconf mechanism (--enable-http3)
APACHE_MODULE(http3, [HTTP/3 protocol handling in addition to HTTP protocol
handling. Implemented by mod_http3. This module requires libnghttp3,
libngtcp2 with its OpenSSL crypto helper, and OpenSSL with the QUIC TLS API.
See --with-nghttp3 and --with-ngtcp2 on how to manage non-standard locations. This module
handling. Implemented by mod_http3. This module requires libnghttp3 and
either libngtcp2 with its OpenSSL crypto helper and OpenSSL with the QUIC TLS
API (the default), or OpenSSL 3.5 or later with its own QUIC server
(--with-http3-quic=openssl). See --with-nghttp3 and --with-ngtcp2 on how to
manage non-standard locations. This module
is usually linked shared and requires loading. ], $http3_objs, , no, [
APACHE_CHECK_OPENSSL
if test "$ac_cv_openssl" = "yes" ; then
Expand All @@ -219,9 +243,35 @@ is usually linked shared and requires loading. ], $http3_objs, , no, [
fi

if test "$enable_http3" != "no" ; then
AC_ARG_WITH(http3-quic, APACHE_HELP_STRING(--with-http3-quic=ngtcp2|openssl, [QUIC transport of mod_http3: ngtcp2 (default; supports 0-RTT and connection ID routing) or the OpenSSL QUIC server]), [
ap_h3_quic="$withval"
], [
ap_h3_quic=ngtcp2
])
if test "x$ap_h3_quic" = "xyes" -o "x$ap_h3_quic" = "x"; then
ap_h3_quic=ngtcp2
fi
if test "x$ap_h3_quic" != "xngtcp2" -a "x$ap_h3_quic" != "xopenssl"; then
AC_MSG_ERROR([--with-http3-quic must be ngtcp2 or openssl, not $ap_h3_quic])
fi

APACHE_CHECK_NGHTTP3
APACHE_CHECK_NGTCP2
if test "$ac_cv_nghttp3" = "yes" -a "$ac_cv_ngtcp2" = "yes" ; then
ap_h3_quic_ok=no
if test "$ap_h3_quic" = "openssl"; then
APACHE_CHECK_H3_OSSL_QUIC
if test "$ac_cv_h3_ossl_quic" = "yes"; then
ap_h3_quic_ok=yes
http3_objs="$http3_objs h3q_ossl.lo h3q_ossl_conn.lo h3q_ossl_stream.lo h3q_ossl_addr.lo"
APR_ADDTO(MOD_CPPFLAGS, ["-DH3Q_BACKEND_OSSL"])
fi
else
APACHE_CHECK_NGTCP2
if test "$ac_cv_ngtcp2" = "yes"; then
ap_h3_quic_ok=yes
http3_objs="$http3_objs h3q.lo h3q_conn.lo h3q_stream.lo"
fi
fi
if test "$ac_cv_nghttp3" = "yes" -a "$ap_h3_quic_ok" = "yes" ; then
APR_ADDTO(MOD_CFLAGS, [-I\$(top_srcdir)/$modpath_current/include])
if test "x$enable_http3" = "xshared"; then
# The only symbol which needs to be exported is the module
Expand Down
6 changes: 6 additions & 0 deletions modules/http3/h3_config.c
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,12 @@ static const char* set_h3_session_tickets(cmd_parms* cmd, void* dummy H3_UNUSED,

static const char* set_h3_early_data(cmd_parms* cmd, void* dummy H3_UNUSED, int flag)
{
#ifdef H3Q_BACKEND_OSSL
if (flag)
{
return "H3EarlyData is not supported when mod_http3 is built with --with-http3-quic=openssl";
}
#endif
h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module);
CHECK(conf);
conf->h3_early_data = flag ? H3_FLAG_ON : H3_FLAG_OFF;
Expand Down
4 changes: 4 additions & 0 deletions modules/http3/h3_server.c
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ typedef struct
* ponytail: top-level directive only, not one inside <IfModule>. */
static int listen_buckets(void)
{
#ifdef H3Q_BACKEND_OSSL
/* Routing by connection ID needs ngtcp2 to put the bucket in the connection IDs. */
return 1;
#endif
for (ap_directive_t* d = ap_conftree; d; d = d->next)
{
if (ap_have_so_reuseport > 0 && !ap_cstr_casecmp(d->directive, "ListenCoresBucketsRatio"))
Expand Down
8 changes: 7 additions & 1 deletion modules/http3/h3_session.c
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,14 @@ apr_status_t h3_session_create(h3_session** psession, server_rec* s, h3q_conn* q
}

nghttp3_conn_set_max_concurrent_streams(session->ngh3, conf->h3_max_concurrent_streams);
nghttp3_conn_set_max_client_streams_bidi(session->ngh3, conf->h3_max_concurrent_streams);
#ifdef H3Q_BACKEND_OSSL
/* OpenSSL raises the peer's stream limit by itself and does not report it:
* give nghttp3 a limit that no client reaches. */
session->max_client_streams_bidi = (uint64_t)1 << 30;
#else
session->max_client_streams_bidi = conf->h3_max_concurrent_streams;
#endif
nghttp3_conn_set_max_client_streams_bidi(session->ngh3, session->max_client_streams_bidi);

h3q_tls_info tls = {0};
if (h3q_conn_tls_info(qconn, &tls))
Expand Down
4 changes: 4 additions & 0 deletions modules/http3/h3_stream.c
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ void flush_nghttp3(h3_session* session)
continue;
}
nghttp3_conn_add_write_offset(session->ngh3, sid, res.accepted);
#ifdef H3Q_BACKEND_OSSL
/* OpenSSL copies what it accepts and reports no acknowledgements. */
nghttp3_conn_add_ack_offset(session->ngh3, sid, res.accepted);
#endif
if (res.blocked)
{
/* Send buffer full: skip this stream instead of busy-looping on the same vec.
Expand Down
9 changes: 9 additions & 0 deletions modules/http3/include/quic/h3q.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,15 @@ typedef struct h3q_stream h3q_stream;

#define H3Q_ERRLEN 256

/**
* The QUIC transport behind this API is chosen at build time with
* --with-http3-quic: ngtcp2 (the default, which defines nothing) or OpenSSL's
* own QUIC stack, which defines H3Q_BACKEND_OSSL. The OpenSSL backend copies
* written stream bytes, so they count as acknowledged once accepted; it has no
* 0-RTT, no stateless-reset/Retry secret, no window autotuning cap and no
* connection ID routing, and ignores those h3q_config fields.
*/

/** Everything the listener needs to exist. */
typedef struct h3q_config
{
Expand Down
13 changes: 13 additions & 0 deletions modules/http3/quic/detail/h3q_tls.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
#include <stdio.h>
#include <stdlib.h>

#ifndef H3Q_BACKEND_OSSL
#include <ngtcp2/ngtcp2_crypto_ossl.h>
#endif
#include <openssl/err.h>
#include <openssl/ssl.h>

Expand Down Expand Up @@ -83,17 +85,26 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const
CHECK(cert_files && ncerts > 0);
CHECK(key_files || nkeys == 0);

#ifdef H3Q_BACKEND_OSSL
/* OpenSSL runs the whole QUIC transport. */
SSL_CTX* ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method());
#else
/* ngtcp2 runs the transport; OpenSSL only does the TLS 1.3 handshake. */
if (ngtcp2_crypto_ossl_init() != 0)
{
h3q_tls_error(err, errlen, "ngtcp2_crypto_ossl_init failed; OpenSSL lacks the QUIC TLS API");
return NULL;
}
SSL_CTX* ssl_ctx = SSL_CTX_new(TLS_server_method());
#endif
if (!ssl_ctx)
{
h3q_tls_error(err, errlen, "SSL_CTX_new failed");
return NULL;
}
#ifdef H3Q_BACKEND_OSSL
SSL_CTX_set_domain_flags(ssl_ctx, SSL_DOMAIN_FLAG_MULTI_THREAD);
#endif

SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
Expand All @@ -117,7 +128,9 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const
* with it off invites none. OpenSSL's anti-replay is built for TLS over TCP
* and refuses QUIC resumption with early data; QUIC 0-RTT replay safety comes
* from accepting only safe methods before the handshake (RFC 8470), in h3_io. */
#ifndef H3Q_BACKEND_OSSL
SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_ANTI_REPLAY);
#endif

if (!session_tickets)
{
Expand Down
Loading