From 44fbe0de34137c7834dc81c1116d7538a2b4f773 Mon Sep 17 00:00:00 2001 From: Matt Caswell Date: Fri, 15 Sep 2023 14:29:05 +0100 Subject: [PATCH] Enable the ability to seed the test RNG without randomising test ordering Numerous tests use the test_random() function to get a random number. If a test fails then the seed that was used for the test RNG is displayed. Setting the seed to the same value in a future run is supposed to cause the same random numbers to be generated again. The way to set the RNG seed again is to use the `OPENSSL_TEST_RAND_ORDER` environment variable. However setting this environment variable *also* randomises the test ordering as well as seeding the RNG. This in itself calls test_random() so, in fact, when the test finally runs it gets different random numbers to when it originally run (defeating the repeatability objective). This means that only way repeatability can be obtained is if the test was originally run with `OPENSSL_TEST_RAND_ORDER` set to 0. If that wasn't done then the seed printed when the test failed is not useful. We introduce a new environment variable `OPENSSL_TEST_RAND_SEED` which can be used to independently seed the test RNG without randomising the test ordering. This can be used to get repeatability in cases where test ordering randomisation was not done in the first place. Reviewed-by: Dmitry Belyavskiy Reviewed-by: Paul Dale (Merged from https://github.com/openssl/openssl/pull/22118) --- test/testutil/driver.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/testutil/driver.c b/test/testutil/driver.c index 6427d235af..033be2fcc5 100644 --- a/test/testutil/driver.c +++ b/test/testutil/driver.c @@ -102,15 +102,18 @@ static void set_seed(int s) int setup_test_framework(int argc, char *argv[]) { - char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER"); + char *test_rand_order = getenv("OPENSSL_TEST_RAND_ORDER"); + char *test_rand_seed = getenv("OPENSSL_TEST_RAND_SEED"); char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); if (TAP_levels != NULL) level = 4 * atoi(TAP_levels); test_adjust_streams_tap_level(level); - if (test_seed != NULL) { + if (test_rand_order != NULL) { rand_order = 1; - set_seed(atoi(test_seed)); + set_seed(atoi(test_rand_order)); + } else if (test_rand_seed != NULL) { + set_seed(atoi(test_rand_seed)); } else { set_seed(0); } @@ -264,8 +267,12 @@ PRINTF_FORMAT(2, 3) static void test_verdict(int verdict, test_flush_stdout(); test_flush_stderr(); - if (verdict == 0 && seed != 0) - test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); + if (verdict == 0) { + if (rand_order) + test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); + else + test_printf_tapout("# OPENSSL_TEST_RAND_SEED=%d\n", seed); + } test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); va_start(ap, description); test_vprintf_tapout(description, ap);