Fix return value checking of BN_check_prime invocations

Negative return value indicates an error so we bail out.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16975)
This commit is contained in:
PW Hu 2021-11-05 17:33:32 +08:00 committed by Tomas Mraz
parent ed5b26ce0b
commit 680827a15f

View File

@ -106,6 +106,7 @@ static int bn_rsa_fips186_4_find_aux_prob_prime(const BIGNUM *Xp1,
{
int ret = 0;
int i = 0;
int tmp = 0;
if (BN_copy(p1, Xp1) == NULL)
return 0;
@ -116,8 +117,11 @@ static int bn_rsa_fips186_4_find_aux_prob_prime(const BIGNUM *Xp1,
i++;
BN_GENCB_call(cb, 0, i);
/* MR test with trial division */
if (BN_check_prime(p1, ctx, cb))
tmp = BN_check_prime(p1, ctx, cb);
if (tmp > 0)
break;
if (tmp < 0)
goto err;
/* Get next odd number */
if (!BN_add_word(p1, 2))
goto err;
@ -329,8 +333,14 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin,
|| !BN_sub_word(y1, 1)
|| !BN_gcd(tmp, y1, e, ctx))
goto err;
if (BN_is_one(tmp) && BN_check_prime(Y, ctx, cb))
goto end;
if (BN_is_one(tmp)) {
int rv = BN_check_prime(Y, ctx, cb);
if (rv > 0)
goto end;
if (rv < 0)
goto err;
}
/* (Step 8-10) */
if (++i >= imax || !BN_add(Y, Y, r1r2x2))
goto err;