mirror/openssl
2
0
Fork 0
mirror of https://github.com/openssl/openssl.git synced 2024-12-15 06:01:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Enable the ability to seed the test RNG without randomising test ordering

Browse Source 
Numerous tests use the test_random() function to get a random number. If a
test fails then the seed that was used for the test RNG is displayed.
Setting the seed to the same value in a future run is supposed to cause the
same random numbers to be generated again.

The way to set the RNG seed again is to use the `OPENSSL_TEST_RAND_ORDER`
environment variable. However setting this environment variable *also*
randomises the test ordering as well as seeding the RNG. This in itself
calls test_random() so, in fact, when the test finally runs it gets
different random numbers to when it originally run (defeating the
repeatability objective).

This means that only way repeatability can be obtained is if the test was
originally run with `OPENSSL_TEST_RAND_ORDER` set to 0. If that wasn't done
then the seed printed when the test failed is not useful.

We introduce a new environment variable `OPENSSL_TEST_RAND_SEED` which can
be used to independently seed the test RNG without randomising the test
ordering. This can be used to get repeatability in cases where test ordering
randomisation was not done in the first place.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22118)
This commit is contained in:
Matt Caswell 2023-09-15 14:29:05 +01:00 committed by Pauli
parent be01f609f9
commit 44fbe0de34
1 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
test/testutil/driver.c
View File

@ -102,15 +102,18 @@ static void set_seed(int s)
int setup_test_framework(int argc, char *argv[]) int setup_test_framework(int argc, char *argv[])
{ {
char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER"); char *test_rand_order = getenv("OPENSSL_TEST_RAND_ORDER");
char *test_rand_seed = getenv("OPENSSL_TEST_RAND_SEED");
char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
if (TAP_levels != NULL) if (TAP_levels != NULL)
level = 4 * atoi(TAP_levels); level = 4 * atoi(TAP_levels);
test_adjust_streams_tap_level(level); test_adjust_streams_tap_level(level);
if (test_seed != NULL) { if (test_rand_order != NULL) {
rand_order = 1; rand_order = 1;
set_seed(atoi(test_seed)); set_seed(atoi(test_rand_order));
} else if (test_rand_seed != NULL) {
set_seed(atoi(test_rand_seed));
} else { } else {
set_seed(0); set_seed(0);
} }
@ -264,8 +267,12 @@ PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
test_flush_stdout(); test_flush_stdout();
test_flush_stderr(); test_flush_stderr();
if (verdict == 0 && seed != 0) if (verdict == 0) {
test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); if (rand_order)
test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
else
test_printf_tapout("# OPENSSL_TEST_RAND_SEED=%d\n", seed);
}
test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
va_start(ap, description); va_start(ap, description);
test_vprintf_tapout(description, ap); test_vprintf_tapout(description, ap);