Test client-side resumption

Add tests for resuming with a different client version.

This happens in reality when clients persist sessions on disk through
upgrades.

Reviewed-by: Rich Salz <rsalz@openssl.org>
This commit is contained in:
Emilia Kasper 2016-07-21 14:04:00 +02:00
parent 2980ae2e78
commit 11279b13f5
9 changed files with 1352 additions and 13 deletions

View File

@ -124,8 +124,13 @@ The following sections may optionally be defined:
matches server.
* resume_server - this section configures the client to resume its session
against a different server. This context is used whenever HandshakeMode is
Resume. If the resume-server section is not present, then the configuration
Resume. If the resume_server section is not present, then the configuration
matches server.
* resume_client - this section configures the client to resume its session with
a different configuration. In practice this may occur when, for example,
upgraded clients reuse sessions persisted on disk. This context is used
whenever HandshakeMode is Resume. If the resume_client section is not present,
then the configuration matches client.
### Default server and client configurations

View File

@ -63,6 +63,16 @@ sub print_templates {
$test->{"resume_server"} = { };
}
$test->{"client"} = { (%ssltests::base_client, %{$test->{"client"}}) };
if (defined $test->{"resume_client"}) {
$test->{"resume_client"} = { (%ssltests::base_client, %{$test->{"resume_client"}}) };
} elsif (defined $test->{"test"}->{"HandshakeMode"} &&
$test->{"test"}->{"HandshakeMode"} eq "Resume") {
# Default is the same as client.
$test->{"resume_client"} = { (%ssltests::base_client, %{$test->{"client"}}) };
} else {
# Do not emit an empty "resume-client" section.
$test->{"resume_client"} = { };
}
}
# ssl_test expects to find a

View File

@ -673,6 +673,7 @@ static HANDSHAKE_RESULT *do_handshake_internal(
HANDSHAKE_RESULT *do_handshake(SSL_CTX *server_ctx, SSL_CTX *server2_ctx,
SSL_CTX *client_ctx, SSL_CTX *resume_server_ctx,
SSL_CTX *resume_client_ctx,
const SSL_TEST_CTX *test_ctx)
{
HANDSHAKE_RESULT *result;
@ -692,8 +693,8 @@ HANDSHAKE_RESULT *do_handshake(SSL_CTX *server_ctx, SSL_CTX *server2_ctx,
HANDSHAKE_RESULT_free(result);
/* We don't support SNI on second handshake yet, so server2_ctx is NULL. */
result = do_handshake_internal(resume_server_ctx, NULL, client_ctx, test_ctx,
session, NULL);
result = do_handshake_internal(resume_server_ctx, NULL, resume_client_ctx,
test_ctx, session, NULL);
end:
SSL_SESSION_free(session);
return result;

View File

@ -47,6 +47,7 @@ void HANDSHAKE_RESULT_free(HANDSHAKE_RESULT *result);
/* Do a handshake and report some information about the result. */
HANDSHAKE_RESULT *do_handshake(SSL_CTX *server_ctx, SSL_CTX *server2_ctx,
SSL_CTX *client_ctx, SSL_CTX *resume_server_ctx,
SSL_CTX *resume_client_ctx,
const SSL_TEST_CTX *test_ctx);
#endif /* HEADER_HANDSHAKE_HELPER_H */

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
# Generated with generate_ssl_tests.pl
num_tests = 8
num_tests = 16
test-0 = 0-resumption
test-1 = 1-resumption
@ -10,6 +10,14 @@ test-4 = 4-resumption
test-5 = 5-resumption
test-6 = 6-resumption
test-7 = 7-resumption
test-8 = 8-resumption
test-9 = 9-resumption
test-10 = 10-resumption
test-11 = 11-resumption
test-12 = 12-resumption
test-13 = 13-resumption
test-14 = 14-resumption
test-15 = 15-resumption
# ===========================================================
[0-resumption]
@ -18,6 +26,7 @@ ssl_conf = 0-resumption-ssl
[0-resumption-ssl]
server = 0-resumption-server
resume-server = 0-resumption-resume-server
resume-client = 0-resumption-resume-client
client = 0-resumption-client
[0-resumption-server]
@ -39,6 +48,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[0-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-0]
HandshakeMode = Resume
Method = DTLS
@ -54,6 +68,7 @@ ssl_conf = 1-resumption-ssl
[1-resumption-ssl]
server = 1-resumption-server
resume-server = 1-resumption-resume-server
resume-client = 1-resumption-resume-client
client = 1-resumption-client
[1-resumption-server]
@ -75,6 +90,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[1-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-1]
HandshakeMode = Resume
Method = DTLS
@ -90,6 +110,7 @@ ssl_conf = 2-resumption-ssl
[2-resumption-ssl]
server = 2-resumption-server
resume-server = 2-resumption-resume-server
resume-client = 2-resumption-resume-client
client = 2-resumption-client
[2-resumption-server]
@ -111,6 +132,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[2-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-2]
HandshakeMode = Resume
Method = DTLS
@ -126,6 +152,7 @@ ssl_conf = 3-resumption-ssl
[3-resumption-ssl]
server = 3-resumption-server
resume-server = 3-resumption-resume-server
resume-client = 3-resumption-resume-client
client = 3-resumption-client
[3-resumption-server]
@ -147,6 +174,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[3-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-3]
HandshakeMode = Resume
Method = DTLS
@ -162,6 +194,7 @@ ssl_conf = 4-resumption-ssl
[4-resumption-ssl]
server = 4-resumption-server
resume-server = 4-resumption-resume-server
resume-client = 4-resumption-resume-client
client = 4-resumption-client
[4-resumption-server]
@ -183,6 +216,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[4-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-4]
HandshakeMode = Resume
Method = DTLS
@ -198,6 +236,7 @@ ssl_conf = 5-resumption-ssl
[5-resumption-ssl]
server = 5-resumption-server
resume-server = 5-resumption-resume-server
resume-client = 5-resumption-resume-client
client = 5-resumption-client
[5-resumption-server]
@ -219,6 +258,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[5-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-5]
HandshakeMode = Resume
Method = DTLS
@ -234,6 +278,7 @@ ssl_conf = 6-resumption-ssl
[6-resumption-ssl]
server = 6-resumption-server
resume-server = 6-resumption-resume-server
resume-client = 6-resumption-resume-client
client = 6-resumption-client
[6-resumption-server]
@ -255,6 +300,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[6-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-6]
HandshakeMode = Resume
Method = DTLS
@ -270,6 +320,7 @@ ssl_conf = 7-resumption-ssl
[7-resumption-ssl]
server = 7-resumption-server
resume-server = 7-resumption-resume-server
resume-client = 7-resumption-resume-client
client = 7-resumption-client
[7-resumption-server]
@ -291,6 +342,11 @@ CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[7-resumption-resume-client]
CipherString = DEFAULT
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-7]
HandshakeMode = Resume
Method = DTLS
@ -298,3 +354,347 @@ Protocol = DTLSv1.2
ResumptionExpected = Yes
# ===========================================================
[8-resumption]
ssl_conf = 8-resumption-ssl
[8-resumption-ssl]
server = 8-resumption-server
resume-server = 8-resumption-resume-server
resume-client = 8-resumption-resume-client
client = 8-resumption-client
[8-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[8-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[8-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
MinProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[8-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-8]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1
ResumptionExpected = Yes
# ===========================================================
[9-resumption]
ssl_conf = 9-resumption-ssl
[9-resumption-ssl]
server = 9-resumption-server
resume-server = 9-resumption-resume-server
resume-client = 9-resumption-resume-client
client = 9-resumption-client
[9-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[9-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[9-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
MinProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[9-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-9]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1
ResumptionExpected = Yes
# ===========================================================
[10-resumption]
ssl_conf = 10-resumption-ssl
[10-resumption-ssl]
server = 10-resumption-server
resume-server = 10-resumption-resume-server
resume-client = 10-resumption-resume-client
client = 10-resumption-client
[10-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[10-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[10-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
MinProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[10-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-10]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1.2
ResumptionExpected = No
# ===========================================================
[11-resumption]
ssl_conf = 11-resumption-ssl
[11-resumption-ssl]
server = 11-resumption-server
resume-server = 11-resumption-resume-server
resume-client = 11-resumption-resume-client
client = 11-resumption-client
[11-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[11-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[11-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
MinProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[11-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-11]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1.2
ResumptionExpected = No
# ===========================================================
[12-resumption]
ssl_conf = 12-resumption-ssl
[12-resumption-ssl]
server = 12-resumption-server
resume-server = 12-resumption-resume-server
resume-client = 12-resumption-resume-client
client = 12-resumption-client
[12-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[12-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[12-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
MinProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[12-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-12]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1
ResumptionExpected = No
# ===========================================================
[13-resumption]
ssl_conf = 13-resumption-ssl
[13-resumption-ssl]
server = 13-resumption-server
resume-server = 13-resumption-resume-server
resume-client = 13-resumption-resume-client
client = 13-resumption-client
[13-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[13-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[13-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
MinProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[13-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-13]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1
ResumptionExpected = No
# ===========================================================
[14-resumption]
ssl_conf = 14-resumption-ssl
[14-resumption-ssl]
server = 14-resumption-server
resume-server = 14-resumption-resume-server
resume-client = 14-resumption-resume-client
client = 14-resumption-client
[14-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[14-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[14-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
MinProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[14-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-14]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1.2
ResumptionExpected = Yes
# ===========================================================
[15-resumption]
ssl_conf = 15-resumption-ssl
[15-resumption-ssl]
server = 15-resumption-server
resume-server = 15-resumption-resume-server
resume-client = 15-resumption-resume-client
client = 15-resumption-client
[15-resumption-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[15-resumption-resume-server]
Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
CipherString = DEFAULT
Options = -SessionTicket
PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
[15-resumption-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
MinProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[15-resumption-resume-client]
CipherString = DEFAULT
MaxProtocol = DTLSv1.2
VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
VerifyMode = Peer
[test-15]
HandshakeMode = Resume
Method = DTLS
Protocol = DTLSv1.2
ResumptionExpected = Yes

View File

@ -142,11 +142,12 @@ sub generate_resumption_tests {
return;
}
my @tests = ();
my @server_tests = ();
my @client_tests = ();
# Obtain the first session against a fixed-version server.
# Obtain the first session against a fixed-version server/client.
foreach my $original_protocol($min_enabled..$#protocols) {
# Upgrade or downgrade the server max version support and test
# Upgrade or downgrade the server/client max version support and test
# that it upgrades, downgrades or resumes the session as well.
foreach my $resume_protocol($min_enabled..$#protocols) {
my $resumption_expected;
@ -158,7 +159,8 @@ sub generate_resumption_tests {
}
foreach my $ticket ("SessionTicket", "-SessionTicket") {
push @tests, {
# Client is flexible, server upgrades/downgrades.
push @server_tests, {
"name" => "resumption",
"client" => { },
"server" => {
@ -176,11 +178,31 @@ sub generate_resumption_tests {
"ResumptionExpected" => $resumption_expected,
}
};
# Server is flexible, client upgrades/downgrades.
push @client_tests, {
"name" => "resumption",
"client" => {
"MinProtocol" => $protocols[$original_protocol],
"MaxProtocol" => $protocols[$original_protocol],
},
"server" => {
"Options" => $ticket,
},
"resume_client" => {
"MaxProtocol" => $protocols[$resume_protocol],
},
"test" => {
"Protocol" => $protocols[$resume_protocol],
"Method" => $method,
"HandshakeMode" => "Resume",
"ResumptionExpected" => $resumption_expected,
}
};
}
}
}
return @tests;
return (@server_tests, @client_tests);
}
sub expected_result {

View File

@ -215,7 +215,7 @@ static int execute_test(SSL_TEST_FIXTURE fixture)
{
int ret = 0;
SSL_CTX *server_ctx = NULL, *server2_ctx = NULL, *client_ctx = NULL,
*resume_server_ctx = NULL;
*resume_server_ctx = NULL, *resume_client_ctx = NULL;
SSL_TEST_CTX *test_ctx = NULL;
HANDSHAKE_RESULT *result = NULL;
@ -233,7 +233,9 @@ static int execute_test(SSL_TEST_FIXTURE fixture)
client_ctx = SSL_CTX_new(DTLS_client_method());
if (test_ctx->handshake_mode == SSL_TEST_HANDSHAKE_RESUME) {
resume_server_ctx = SSL_CTX_new(DTLS_server_method());
resume_client_ctx = SSL_CTX_new(DTLS_client_method());
OPENSSL_assert(resume_server_ctx != NULL);
OPENSSL_assert(resume_client_ctx != NULL);
}
}
#endif
@ -247,11 +249,14 @@ static int execute_test(SSL_TEST_FIXTURE fixture)
if (test_ctx->handshake_mode == SSL_TEST_HANDSHAKE_RESUME) {
resume_server_ctx = SSL_CTX_new(TLS_server_method());
resume_client_ctx = SSL_CTX_new(TLS_client_method());
OPENSSL_assert(resume_server_ctx != NULL);
OPENSSL_assert(resume_client_ctx != NULL);
}
}
OPENSSL_assert(server_ctx != NULL && client_ctx != NULL);
OPENSSL_assert(server_ctx != NULL);
OPENSSL_assert(client_ctx != NULL);
OPENSSL_assert(CONF_modules_load(conf, fixture.test_app, 0) > 0);
@ -265,9 +270,12 @@ static int execute_test(SSL_TEST_FIXTURE fixture)
if (resume_server_ctx != NULL
&& !SSL_CTX_config(resume_server_ctx, "resume-server"))
goto err;
if (resume_client_ctx != NULL
&& !SSL_CTX_config(resume_client_ctx, "resume-client"))
goto err;
result = do_handshake(server_ctx, server2_ctx, client_ctx,
resume_server_ctx, test_ctx);
resume_server_ctx, resume_client_ctx, test_ctx);
ret = check_test(result, test_ctx);
@ -277,6 +285,7 @@ err:
SSL_CTX_free(server2_ctx);
SSL_CTX_free(client_ctx);
SSL_CTX_free(resume_server_ctx);
SSL_CTX_free(resume_client_ctx);
SSL_TEST_CTX_free(test_ctx);
if (ret != 1)
ERR_print_errors_fp(stderr);

View File

@ -11,6 +11,9 @@ server = {-$testname-}-server{-
if (%resume_server) {
$OUT .= "\nresume-server = $testname-resume-server";
}
if (%resume_client) {
$OUT .= "\nresume-client = $testname-resume-client";
}
-}
client = {-$testname-}-client
@ -37,6 +40,12 @@ client = {-$testname-}-client
foreach my $key (sort keys %client) {
$OUT .= qq{$key} . " = " . qq{$client{$key}\n} if defined $client{$key};
}
if (%resume_client) {
$OUT .= "\n[$testname-resume-client]\n";
foreach my $key (sort keys %resume_client) {
$OUT .= qq{$key} . " = " . qq{$resume_client{$key}\n} if defined $resume_client{$key};
}
}
-}
[test-{-$idx-}]
{-