diff --git a/CHANGES.md b/CHANGES.md index 2c12daf151..ede13f7d79 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -226,6 +226,10 @@ OpenSSL 3.2 *Tianjia Zhang* + * Zerocopy KTLS sendfile() support on Linux. + + *Maxim Mikityanskiy* + OpenSSL 3.0 ----------- diff --git a/apps/s_server.c b/apps/s_server.c index f519505ade..46af6b87da 100644 --- a/apps/s_server.c +++ b/apps/s_server.c @@ -96,6 +96,7 @@ static int keymatexportlen = 20; static int async = 0; static int use_sendfile = 0; +static int use_zc_sendfile = 0; static const char *session_id_prefix = NULL; @@ -716,6 +717,7 @@ typedef enum OPTION_choice { OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA, OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG, OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS, + OPT_USE_ZC_SENDFILE, OPT_TFO, OPT_CERT_COMP, OPT_R_ENUM, OPT_S_ENUM, @@ -966,6 +968,7 @@ const OPTIONS s_server_options[] = { #ifndef OPENSSL_NO_KTLS {"ktls", OPT_KTLS, '-', "Enable Kernel TLS for sending and receiving"}, {"sendfile", OPT_SENDFILE, '-', "Use sendfile to response file with -WWW"}, + {"zerocopy_sendfile", OPT_USE_ZC_SENDFILE, '-', "Use zerocopy mode of KTLS sendfile"}, #endif OPT_R_OPTIONS, @@ -1080,6 +1083,7 @@ int s_server_main(int argc, char *argv[]) s_brief = 0; async = 0; use_sendfile = 0; + use_zc_sendfile = 0; port = OPENSSL_strdup(PORT); cctx = SSL_CONF_CTX_new(); @@ -1654,6 +1658,11 @@ int s_server_main(int argc, char *argv[]) case OPT_SENDFILE: #ifndef OPENSSL_NO_KTLS use_sendfile = 1; +#endif + break; + case OPT_USE_ZC_SENDFILE: +#ifndef OPENSSL_NO_KTLS + use_zc_sendfile = 1; #endif break; case OPT_IGNORE_UNEXPECTED_EOF: @@ -1728,6 +1737,11 @@ int s_server_main(int argc, char *argv[]) #endif #ifndef OPENSSL_NO_KTLS + if (use_zc_sendfile && !use_sendfile) { + BIO_printf(bio_out, "Warning: -zerocopy_sendfile depends on -sendfile, enabling -sendfile now.\n"); + use_sendfile = 1; + } + if (use_sendfile && enable_ktls == 0) { BIO_printf(bio_out, "Warning: -sendfile depends on -ktls, enabling -ktls now.\n"); enable_ktls = 1; @@ -1933,6 +1947,8 @@ int s_server_main(int argc, char *argv[]) #ifndef OPENSSL_NO_KTLS if (enable_ktls) SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS); + if (use_zc_sendfile) + SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE); #endif if (max_send_fragment > 0 diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c index 75bfe64566..f494b14000 100644 --- a/crypto/bio/bss_conn.c +++ b/crypto/bio/bss_conn.c @@ -598,6 +598,11 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; + case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: + ret = ktls_enable_tx_zerocopy_sendfile(b->num); + if (ret) + BIO_set_ktls_zerocopy_sendfile_flag(b); + break; # endif default: ret = 0; diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c index 69dfd37bfe..f64eb8c843 100644 --- a/crypto/bio/bss_sock.c +++ b/crypto/bio/bss_sock.c @@ -235,6 +235,11 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; + case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: + ret = ktls_enable_tx_zerocopy_sendfile(b->num); + if (ret) + BIO_set_ktls_zerocopy_sendfile_flag(b); + break; # endif case BIO_CTRL_EOF: ret = (b->flags & BIO_FLAGS_IN_EOF) != 0; diff --git a/doc/man1/openssl-s_server.pod.in b/doc/man1/openssl-s_server.pod.in index 94f3b4b46c..a1e354908c 100644 --- a/doc/man1/openssl-s_server.pod.in +++ b/doc/man1/openssl-s_server.pod.in @@ -132,6 +132,7 @@ B B [B<-alpn> I] [B<-ktls>] [B<-sendfile>] +[B<-zerocopy_sendfile>] [B<-keylogfile> I] [B<-recv_max_early_data> I] [B<-max_early_data> I] @@ -792,6 +793,15 @@ instead of BIO_write() to send the HTTP response requested by a client. This option is only valid when B<-ktls> along with B<-WWW> or B<-HTTP> are specified. +=item B<-zerocopy_sendfile> + +If this option is set, SSL_sendfile() will use the zerocopy TX mode, which gives +a performance boost when used with KTLS hardware offload. Note that invalid +TLS records might be transmitted if the file is changed while being sent. +This option depends on B<-sendfile>; when used alone, B<-sendfile> is implied, +and a warning is shown. Note that KTLS sendfile on FreeBSD always runs in the +zerocopy mode. + =item B<-keylogfile> I Appends TLS secrets to the specified keylog file such that external programs diff --git a/doc/man3/SSL_CONF_cmd.pod b/doc/man3/SSL_CONF_cmd.pod index c20df37e3b..3717c202bd 100644 --- a/doc/man3/SSL_CONF_cmd.pod +++ b/doc/man3/SSL_CONF_cmd.pod @@ -561,6 +561,14 @@ B: support receiving compressed certificates, enabled default. Inverse of B: that is, B<-RxCertificateCompression> is the same as setting B. +B: use the zerocopy TX mode of sendfile(), which gives +a performance boost when used with KTLS hardware offload. Note that invalid TLS +records might be transmitted if the file is changed while being sent. This +option has no effect if B is not enabled. Equivalent to +B. This option only applies to Linux. +KTLS sendfile on FreeBSD doesn't offer an option to disable zerocopy and +always runs in this mode. + =item B The B argument is a comma separated list of flags to set. diff --git a/doc/man3/SSL_CTX_set_options.pod b/doc/man3/SSL_CTX_set_options.pod index 1da057adb8..b72973f8d0 100644 --- a/doc/man3/SSL_CTX_set_options.pod +++ b/doc/man3/SSL_CTX_set_options.pod @@ -175,6 +175,16 @@ by the kernel directly and not via any available OpenSSL Providers. This might be undesirable if, for example, the application requires all cryptographic operations to be performed by the FIPS provider. +=item SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE + +With this option, sendfile() will use the zerocopy mode, which gives a +performance boost when used with KTLS hardware offload. Note that invalid TLS +records might be transmitted if the file is changed while being sent. This +option has no effect if B is not enabled. + +This option only applies to Linux. KTLS sendfile on FreeBSD doesn't offer an +option to disable zerocopy and always runs in this mode. + =item SSL_OP_ENABLE_MIDDLEBOX_COMPAT If set then dummy Change Cipher Spec (CCS) messages are sent in TLSv1.3. This diff --git a/include/internal/bio.h b/include/internal/bio.h index 40218e1fb0..9481f4c985 100644 --- a/include/internal/bio.h +++ b/include/internal/bio.h @@ -43,16 +43,20 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); # define BIO_CTRL_SET_KTLS 72 # define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG 74 # define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG 75 +# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 /* * This is used with socket BIOs: * BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending. * BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next. * BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving. + * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with + * this BIO for sending using sendfile. */ # define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000 # define BIO_FLAGS_KTLS_RX 0x2000 # define BIO_FLAGS_KTLS_TX 0x4000 +# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000 /* KTLS related controls and flags */ # define BIO_set_ktls_flag(b, is_tx) \ @@ -65,6 +69,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) # define BIO_clear_ktls_ctrl_msg_flag(b) \ BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) +# define BIO_set_ktls_zerocopy_sendfile_flag(b) \ + BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE) # define BIO_set_ktls(b, keyblob, is_tx) \ BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob) @@ -72,6 +78,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL) # define BIO_clear_ktls_ctrl_msg(b) \ BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL) +# define BIO_set_ktls_tx_zerocopy_sendfile(b) \ + BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL) /* Functions to allow the core to offer the CORE_BIO type to providers */ OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio); diff --git a/include/internal/ktls.h b/include/internal/ktls.h index efa4451d7c..af27a32569 100644 --- a/include/internal/ktls.h +++ b/include/internal/ktls.h @@ -214,6 +214,13 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off, # warning "Skipping Compilation of KTLS receive data path" # endif # endif +# if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0) +# define OPENSSL_NO_KTLS_ZC_TX +# ifndef PEDANTIC +# warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile" +# warning "Skipping Compilation of KTLS zerocopy sendfile" +# endif +# endif # define OPENSSL_KTLS_AES_GCM_128 # if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) # define OPENSSL_KTLS_AES_GCM_256 @@ -293,6 +300,18 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info, crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1; } +static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd) +{ +#ifndef OPENSSL_NO_KTLS_ZC_TX + int enable = 1; + + return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO, + &enable, sizeof(enable)) ? 0 : 1; +#else + return 0; +#endif +} + /* * Send a TLS record using the crypto_info provided in ktls_start and use * record_type instead of the default SSL3_RT_APPLICATION_DATA. diff --git a/include/openssl/bio.h.in b/include/openssl/bio.h.in index 6105c602fd..89c88c67e7 100644 --- a/include/openssl/bio.h.in +++ b/include/openssl/bio.h.in @@ -182,6 +182,11 @@ extern "C" { # define BIO_CTRL_DGRAM_GET_NO_TRUNC 88 # define BIO_CTRL_DGRAM_SET_NO_TRUNC 89 +/* + * internal BIO: + * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 + */ + # define BIO_DGRAM_CAP_NONE 0U # define BIO_DGRAM_CAP_HANDLES_SRC_ADDR (1U << 0) # define BIO_DGRAM_CAP_HANDLES_DST_ADDR (1U << 1) @@ -225,7 +230,7 @@ extern "C" { # define BIO_FLAGS_NONCLEAR_RST 0x400 # define BIO_FLAGS_IN_EOF 0x800 -/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */ +/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */ typedef union bio_addr_st BIO_ADDR; typedef struct bio_addrinfo_st BIO_ADDRINFO; diff --git a/include/openssl/ssl.h.in b/include/openssl/ssl.h.in index 2224b3269b..871ad265c5 100644 --- a/include/openssl/ssl.h.in +++ b/include/openssl/ssl.h.in @@ -420,6 +420,8 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); */ # define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION SSL_OP_BIT(32) # define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION SSL_OP_BIT(33) + /* Enable KTLS TX zerocopy on Linux */ +# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE SSL_OP_BIT(34) /* * Option "collections." diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c index 5c94837dc0..2f9b11a512 100644 --- a/ssl/record/methods/ktls_meth.c +++ b/ssl/record/methods/ktls_meth.c @@ -334,6 +334,14 @@ static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction)) return OSSL_RECORD_RETURN_NON_FATAL_ERR; + if (rl->direction == OSSL_RECORD_DIRECTION_WRITE && + (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0) + /* Ignore errors. The application opts in to using the zerocopy + * optimization. If the running kernel doesn't support it, just + * continue without the optimization. + */ + BIO_set_ktls_tx_zerocopy_sendfile(rl->bio); + return OSSL_RECORD_RETURN_SUCCESS; } diff --git a/ssl/ssl_conf.c b/ssl/ssl_conf.c index bebfc501a9..0bea29fb66 100644 --- a/ssl/ssl_conf.c +++ b/ssl/ssl_conf.c @@ -400,6 +400,7 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value) SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT), SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION), SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION), + SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE), }; if (value == NULL) return -3; diff --git a/test/sslapitest.c b/test/sslapitest.c index 8f14381b56..a26f6286f3 100644 --- a/test/sslapitest.c +++ b/test/sslapitest.c @@ -1293,7 +1293,8 @@ end: #define SENDFILE_CHUNK (4 * 4096) #define min(a,b) ((a) > (b) ? (b) : (a)) -static int execute_test_ktls_sendfile(int tls_version, const char *cipher) +static int execute_test_ktls_sendfile(int tls_version, const char *cipher, + int zerocopy) { SSL_CTX *cctx = NULL, *sctx = NULL; SSL *clientssl = NULL, *serverssl = NULL; @@ -1350,6 +1351,12 @@ static int execute_test_ktls_sendfile(int tls_version, const char *cipher) if (!TEST_true(SSL_set_options(serverssl, SSL_OP_ENABLE_KTLS))) goto end; + if (zerocopy) { + if (!TEST_true(SSL_set_options(serverssl, + SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE))) + goto end; + } + if (!TEST_true(create_ssl_connection(serverssl, clientssl, SSL_ERROR_NONE))) goto end; @@ -1480,14 +1487,16 @@ static int test_ktls(int test) cipher->cipher); } -static int test_ktls_sendfile(int tst) +static int test_ktls_sendfile(int test) { struct ktls_test_cipher *cipher; + int tst = test >> 1; OPENSSL_assert(tst < (int)NUM_KTLS_TEST_CIPHERS); cipher = &ktls_test_ciphers[tst]; - return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher); + return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher, + test & 1); } #endif @@ -10544,7 +10553,7 @@ int setup_tests(void) #if !defined(OPENSSL_NO_KTLS) && !defined(OPENSSL_NO_SOCK) # if !defined(OPENSSL_NO_TLS1_2) || !defined(OSSL_NO_USABLE_TLS1_3) ADD_ALL_TESTS(test_ktls, NUM_KTLS_TEST_CIPHERS * 4); - ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS); + ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS * 2); # endif #endif ADD_TEST(test_large_message_tls);