Mercurial > nginx
changeset 9413:e992b5db34fc
QUIC: using QUIC API introduced in OpenSSL 3.5.
Similarly to the QUIC API originated in BoringSSL, this API allows
to register custom TLS callbacks for an external QUIC implementation.
See the SSL_set_quic_tls_cbs manual page for details.
Due to a different approach used in OpenSSL 3.5, handling of CRYPTO
frames was streamlined to always write an incoming CRYPTO buffer to
the crypto context. Using SSL_provide_quic_data(), this results in
transient allocation of chain links and buffers for CRYPTO frames
received in order. Testing didn't reveal performance degradation of
QUIC handshakes, https://github.com/nginx/nginx/pull/646 provides
specific results.
| author | Sergey Kandaurov <pluknet@nginx.com> |
|---|---|
| date | Thu, 13 Feb 2025 17:00:56 +0400 |
| parents | ca1713feedae |
| children | 23971bd0b072 |
| files | auto/lib/openssl/conf src/event/quic/ngx_event_quic.h src/event/quic/ngx_event_quic_connection.h src/event/quic/ngx_event_quic_ssl.c |
| diffstat | 4 files changed, 383 insertions(+), 48 deletions(-) [+] |
line wrap: on
line diff
--- a/auto/lib/openssl/conf Fri May 16 01:10:11 2025 +0400 +++ b/auto/lib/openssl/conf Thu Feb 13 17:00:56 2025 +0400 @@ -147,12 +147,18 @@ if [ $USE_OPENSSL_QUIC = YES ]; then - ngx_feature="OpenSSL QUIC support" + ngx_feature="OpenSSL QUIC API" ngx_feature_name="NGX_QUIC" - ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + ngx_feature_test="SSL_set_quic_tls_cbs(NULL, NULL, NULL)" . auto/feature if [ $ngx_found = no ]; then + ngx_feature="BoringSSL-like QUIC API" + ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + . auto/feature + fi + + if [ $ngx_found = no ]; then ngx_feature="OpenSSL QUIC compatibility" ngx_feature_test="SSL_CTX_add_custom_ext(NULL, 0, 0, NULL, NULL, NULL, NULL, NULL)"
--- a/src/event/quic/ngx_event_quic.h Fri May 16 01:10:11 2025 +0400 +++ b/src/event/quic/ngx_event_quic.h Thu Feb 13 17:00:56 2025 +0400 @@ -12,7 +12,10 @@ #include <ngx_core.h> -#ifdef SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION +#ifdef OSSL_RECORD_PROTECTION_LEVEL_NONE +#define NGX_QUIC_OPENSSL_API 1 + +#elif (defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION) #define NGX_QUIC_QUICTLS_API 1 #elif (defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER)
--- a/src/event/quic/ngx_event_quic_connection.h Fri May 16 01:10:11 2025 +0400 +++ b/src/event/quic/ngx_event_quic_connection.h Thu Feb 13 17:00:56 2025 +0400 @@ -301,6 +301,11 @@ unsigned key_phase:1; unsigned validated:1; unsigned client_tp_done:1; + +#if (NGX_QUIC_OPENSSL_API) + unsigned read_level:2; + unsigned write_level:2; +#endif };
--- a/src/event/quic/ngx_event_quic_ssl.c Fri May 16 01:10:11 2025 +0400 +++ b/src/event/quic/ngx_event_quic_ssl.c Thu Feb 13 17:00:56 2025 +0400 @@ -18,6 +18,23 @@ #define NGX_QUIC_MAX_BUFFERED 65535 +#if (NGX_QUIC_OPENSSL_API) + +static int ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg); +static int ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg); +static int ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, + size_t bytes_read, void *arg); +static int ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t level, + int direction, const unsigned char *secret, size_t secret_len, void *arg); +static int ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg); +static int ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, + void *arg); + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static ngx_inline ngx_uint_t ngx_quic_map_encryption_level( enum ssl_encryption_level_t ssl_level); @@ -39,9 +56,270 @@ static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t ssl_level, uint8_t alert); + +#endif + static ngx_int_t ngx_quic_handshake(ngx_connection_t *c); -static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level); +static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level); + + +#if (NGX_QUIC_OPENSSL_API) + +static int +ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *out; + unsigned int alpn_len; + ngx_quic_frame_t *frame; + const unsigned char *alpn_data; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_send len:%uz", len); + + qc = ngx_quic_get_connection(c); + + *consumed = 0; + + SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len); + + if (alpn_len == 0) { + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL); + qc->error_reason = "missing ALPN extension"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic missing ALPN extension"); + return 1; + } + + if (!qc->client_tp_done) { + /* RFC 9001, 8.2. QUIC Transport Parameters Extension */ + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION); + qc->error_reason = "missing transport parameters"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "missing transport parameters"); + return 1; + } + + ctx = ngx_quic_get_send_ctx(qc, qc->write_level); + + out = ngx_quic_copy_buffer(c, (u_char *) data, len); + if (out == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame->data = out; + frame->level = qc->write_level; + frame->type = NGX_QUIC_FT_CRYPTO; + frame->u.crypto.offset = ctx->crypto_sent; + frame->u.crypto.length = len; + + ctx->crypto_sent += len; + *consumed = len; + + ngx_quic_queue_frame(qc, frame); + + return 1; +} + + +static int +ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_recv_rcd"); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + for (cl = ctx->crypto.chain; cl; cl = cl->next) { + b = cl->buf; + + if (b->sync) { + /* hole */ + + *bytes_read = 0; + + break; + } + + *data = b->pos; + *bytes_read = b->last - b->pos; + + break; + } + + return 1; +} + + +static int +ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, size_t bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_release_rcd len:%uz", bytes_read); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + cl = ngx_quic_read_buffer(c, &ctx->crypto, bytes_read); + if (cl == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + ngx_quic_free_chain(c, cl); + + return 1; +} + + +static int +ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t ssl_level, + int direction, const unsigned char *secret, size_t secret_len, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_uint_t level; + const SSL_CIPHER *cipher; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_yield_secret() level:%uD", ssl_level); +#ifdef NGX_QUIC_DEBUG_CRYPTO + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic %s secret len:%uz %*xs", + direction ? "write" : "read", secret_len, + secret_len, secret); +#endif + + qc = ngx_quic_get_connection(c); + cipher = SSL_get_current_cipher(ssl_conn); + + switch (ssl_level) { + case OSSL_RECORD_PROTECTION_LEVEL_NONE: + level = NGX_QUIC_ENCRYPTION_INITIAL; + break; + case OSSL_RECORD_PROTECTION_LEVEL_EARLY: + level = NGX_QUIC_ENCRYPTION_EARLY_DATA; + break; + case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE: + level = NGX_QUIC_ENCRYPTION_HANDSHAKE; + break; + default: /* OSSL_RECORD_PROTECTION_LEVEL_APPLICATION */ + level = NGX_QUIC_ENCRYPTION_APPLICATION; + break; + } + + if (ngx_quic_keys_set_encryption_secret(c->log, direction, qc->keys, level, + cipher, secret, secret_len) + != NGX_OK) + { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + if (direction) { + qc->write_level = level; + + } else { + qc->read_level = level; + } + + return 1; +} + + +static int +ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg) +{ + ngx_connection_t *c = arg; + + u_char *p, *end; + ngx_quic_tp_t ctp; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_got_transport_params() len:%uz", + params_len); + + qc = ngx_quic_get_connection(c); + + /* defaults for parameters not sent by client */ + ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t)); + + p = (u_char *) params; + end = p + params_len; + + if (ngx_quic_parse_transport_params(p, end, &ctp, c->log) != NGX_OK) { + qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR; + qc->error_reason = "failed to process transport parameters"; + + return 1; + } + + if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) { + return 1; + } + + qc->client_tp_done = 1; + + return 1; +} + + +static int +ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_alert() alert:%d", (int) alert); + + /* already closed on regular shutdown */ + + qc = ngx_quic_get_connection(c); + if (qc == NULL) { + return 1; + } + + qc->error = NGX_QUIC_ERR_CRYPTO(alert); + qc->error_reason = "handshake failed"; + + return 1; +} + + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ static ngx_inline ngx_uint_t @@ -340,13 +618,14 @@ return 1; } +#endif + ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame) { uint64_t last; - ngx_chain_t *cl; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; ngx_quic_crypto_frame_t *f; @@ -385,41 +664,18 @@ return NGX_OK; } - if (f->offset == ctx->crypto.offset) { - if (ngx_quic_crypto_provide(c, frame->data, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_skip_buffer(c, &ctx->crypto, last); - - } else { - if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, - f->offset) - == NGX_CHAIN_ERROR) - { - return NGX_ERROR; - } + if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, + f->offset) + == NGX_CHAIN_ERROR) + { + return NGX_ERROR; } - cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); - - if (cl) { - if (ngx_quic_crypto_provide(c, cl, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_free_chain(c, cl); + if (ngx_quic_crypto_provide(c, pkt->level) != NGX_OK) { + return NGX_ERROR; } - return NGX_OK; + return ngx_quic_handshake(c); } @@ -528,13 +784,24 @@ static ngx_int_t -ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level) +ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level) { +#if (NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API) + ngx_buf_t *b; - ngx_chain_t *cl; + ngx_chain_t *out, *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; enum ssl_encryption_level_t ssl_level; + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, level); + + out = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); + if (out == NGX_CHAIN_ERROR) { + return NGX_ERROR; + } + switch (level) { case NGX_QUIC_ENCRYPTION_INITIAL: ssl_level = ssl_encryption_initial; @@ -562,6 +829,10 @@ } } + ngx_quic_free_chain(c, out); + +#endif + return NGX_OK; } @@ -569,14 +840,40 @@ ngx_int_t ngx_quic_init_connection(ngx_connection_t *c) { - u_char *p; - size_t clen; - ssize_t len; - ngx_str_t dcid; - ngx_ssl_conn_t *ssl_conn; - ngx_quic_socket_t *qsock; - ngx_quic_connection_t *qc; - static SSL_QUIC_METHOD quic_method; + u_char *p; + size_t clen; + ssize_t len; + ngx_str_t dcid; + ngx_ssl_conn_t *ssl_conn; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + +#if (NGX_QUIC_OPENSSL_API) + static const OSSL_DISPATCH qtdis[] = { + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, + (void (*)(void)) ngx_quic_cbs_send }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, + (void (*)(void)) ngx_quic_cbs_recv_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, + (void (*)(void)) ngx_quic_cbs_release_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, + (void (*)(void)) ngx_quic_cbs_yield_secret }, + + { OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, + (void (*)(void)) ngx_quic_cbs_got_transport_params }, + + { OSSL_FUNC_SSL_QUIC_TLS_ALERT, + (void (*)(void)) ngx_quic_cbs_alert }, + + { 0, NULL } + }; +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static SSL_QUIC_METHOD quic_method; +#endif qc = ngx_quic_get_connection(c); @@ -588,6 +885,20 @@ ssl_conn = c->ssl->connection; +#if (NGX_QUIC_OPENSSL_API) + + if (SSL_set_quic_tls_cbs(ssl_conn, qtdis, c) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_cbs() failed"); + return NGX_ERROR; + } + + if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { + SSL_set_quic_tls_early_data_enabled(ssl_conn, 1); + } + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + if (!quic_method.send_alert) { #if (NGX_QUIC_BORINGSSL_API) quic_method.set_read_secret = ngx_quic_set_read_secret; @@ -612,6 +923,8 @@ } #endif +#endif + qsock = ngx_quic_get_socket(c); dcid.data = qsock->sid.id; @@ -641,11 +954,19 @@ "quic transport parameters len:%uz %*xs", len, len, p); #endif +#if (NGX_QUIC_OPENSSL_API) + if (SSL_set_quic_tls_transport_params(ssl_conn, p, len) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_transport_params() failed"); + return NGX_ERROR; + } +#else if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) { ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_transport_params() failed"); return NGX_ERROR; } +#endif #ifdef OPENSSL_IS_BORINGSSL if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) {