ssl_cipher_get_overhead(): Replace size_t with int and add the checks

Replace the type of "mac", "out", and "blk" with int to avoid implicit
conversion when it is assigned by EVP_MD_get_size(),
EVP_CIPHER_get_iv_length(), and EVP_CIPHER_get_block_size().
Moreover, add the checks to avoid integer overflow.

Fixes: 045bd04706 ("Add DTLS_get_data_mtu() function")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23935)
This commit is contained in:
Jiasheng Jiang 2024-03-22 15:15:09 +00:00 committed by Tomas Mraz
parent d318411019
commit 4a5088259e

View File

@ -2184,7 +2184,7 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
size_t *int_overhead, size_t *blocksize,
size_t *ext_overhead)
{
size_t mac = 0, in = 0, blk = 0, out = 0;
int mac = 0, in = 0, blk = 0, out = 0;
/* Some hard-coded numbers for the CCM/Poly1305 MAC overhead
* because there are no handy #defines for those. */
@ -2208,6 +2208,8 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
return 0;
mac = EVP_MD_get_size(e_md);
if (mac <= 0)
return 0;
if (c->algorithm_enc != SSL_eNULL) {
int cipher_nid = SSL_CIPHER_get_cipher_nid(c);
const EVP_CIPHER *e_ciph = EVP_get_cipherbynid(cipher_nid);
@ -2220,16 +2222,18 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
in = 1; /* padding length byte */
out = EVP_CIPHER_get_iv_length(e_ciph);
if (out < 0)
return 0;
blk = EVP_CIPHER_get_block_size(e_ciph);
if (blk == 0)
if (blk <= 0)
return 0;
}
}
*mac_overhead = mac;
*int_overhead = in;
*blocksize = blk;
*ext_overhead = out;
*mac_overhead = (size_t)mac;
*int_overhead = (size_t)in;
*blocksize = (size_t)blk;
*ext_overhead = (size_t)out;
return 1;
}