Fix an endless loop in rsa_builtin_keygen.

And add a test case.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2757)
This commit is contained in:
Rich Salz 2017-03-06 09:54:17 -05:00
parent e498d95454
commit 697958313b
2 changed files with 37 additions and 12 deletions

View File

@ -43,6 +43,16 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
int bitsp, bitsq, ok = -1, n = 0;
BN_CTX *ctx = NULL;
/*
* When generating ridiculously small keys, we can get stuck
* continually regenerating the same prime values.
*/
if (bits < 16) {
ok = 0; /* we set our own err */
RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
goto err;
}
ctx = BN_CTX_new();
if (ctx == NULL)
goto err;
@ -94,21 +104,10 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
if (!BN_GENCB_call(cb, 3, 0))
goto err;
for (;;) {
/*
* When generating ridiculously small keys, we can get stuck
* continually regenerating the same prime values. Check for this and
* bail if it happens 3 times.
*/
unsigned int degenerate = 0;
do {
if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
goto err;
} while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
if (degenerate == 3) {
ok = 0; /* we set our own err */
RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
goto err;
}
} while (BN_cmp(rsa->p, rsa->q) == 0);
if (!BN_sub(r2, rsa->q, BN_value_one()))
goto err;
if (!BN_gcd(r1, r2, rsa->e, ctx))

View File

@ -0,0 +1,26 @@
#! /usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
use strict;
use warnings;
use File::Spec;
use OpenSSL::Test qw/:DEFAULT srctop_file/;
use OpenSSL::Test::Utils;
setup("test_genrsa");
plan tests => 5;
is(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '8'])), 0, "genrsa -3 8");
ok(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '16'])), "genrsa -3 16");
ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");
ok(run(app([ 'openssl', 'genrsa', '-f4', '-out', 'genrsatest.pem', '16'])), "genrsa -f4 16");
ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");
unlink 'genrsatest.pem';