BN_gcd(): Avoid shifts of negative values

Fixes #22216

Thanks to Leland Mills for investigation and testing.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22272)
This commit is contained in:
Tomas Mraz 2023-10-04 09:30:43 +02:00
parent f7b80136a3
commit 0f7a3b0caa

View File

@ -642,9 +642,9 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
for (i = 0; i < m; i++) {
/* conditionally flip signs if delta is positive and g is odd */
cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
cond = ((unsigned int)-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
/* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
& (~((g->top - 1) >> (sizeof(g->top) * 8 - 1)));
& (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1)));
delta = (-cond & -delta) | ((cond - 1) & delta);
r->neg ^= cond;
/* swap */
@ -656,7 +656,7 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
goto err;
BN_consttime_swap(g->d[0] & 1 /* g is odd */
/* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
& (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))),
& (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1))),
g, temp, top);
if (!BN_rshift1(g, g))
goto err;