Fix signed integer overflow in evp_enc

Fixes #17869.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17870)
This commit is contained in:
Hugo Landau 2022-03-11 06:57:26 +00:00 committed by Tomas Mraz
parent ef9909f3c6
commit 1832bb0f02
2 changed files with 14 additions and 6 deletions

View File

@ -605,7 +605,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in, int inl)
{
int ret;
size_t soutl;
size_t soutl, inl_ = (size_t)inl;
int blocksize;
if (outl != NULL) {
@ -635,9 +635,10 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
return 0;
}
ret = ctx->cipher->cupdate(ctx->algctx, out, &soutl,
inl + (blocksize == 1 ? 0 : blocksize), in,
(size_t)inl);
inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
in, inl_);
if (ret) {
if (soutl > INT_MAX) {
@ -753,7 +754,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
{
int fix_len, cmpl = inl, ret;
unsigned int b;
size_t soutl;
size_t soutl, inl_ = (size_t)inl;
int blocksize;
if (outl != NULL) {
@ -783,8 +784,8 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
return 0;
}
ret = ctx->cipher->cupdate(ctx->algctx, out, &soutl,
inl + (blocksize == 1 ? 0 : blocksize), in,
(size_t)inl);
inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
in, inl_);
if (ret) {
if (soutl > INT_MAX) {

View File

@ -114,6 +114,13 @@ static int test_sanity_range(void)
TEST_error("int must not be wider than size_t");
return 0;
}
/* SIZE_MAX is always greater than 2*INT_MAX */
if (SIZE_MAX - INT_MAX <= INT_MAX) {
TEST_error("SIZE_MAX must exceed 2*INT_MAX");
return 0;
}
return 1;
}