Enable setting SSL_CERT_FLAG_TLS_STRICT with ssl config

Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17989)

doc/man3/SSL_CONF_cmd.pod

@@ -532,6 +532,9 @@ B<KTLS>: Enables kernel TLS if support has been compiled in, and it is supported
 by the negotiated ciphersuites and extensions. Equivalent to
 B<SSL_OP_ENABLE_KTLS>.
 
+B<StrictCertCheck>: Enable strict certificate checking. Equivalent to
+setting B<SSL_CERT_FLAG_TLS_STRICT> with SSL_CTX_set_cert_flags().
+
 =item B<VerifyMode>
 
 The B<value> argument is a comma separated list of flags to set.

ssl/ssl_conf.c

@@ -396,7 +396,8 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value)
         SSL_FLAG_TBL_INV("AntiReplay", SSL_OP_NO_ANTI_REPLAY),
         SSL_FLAG_TBL_INV("ExtendedMasterSecret", SSL_OP_NO_EXTENDED_MASTER_SECRET),
         SSL_FLAG_TBL_INV("CANames", SSL_OP_DISABLE_TLSEXT_CA_NAMES),
-        SSL_FLAG_TBL("KTLS", SSL_OP_ENABLE_KTLS)
+        SSL_FLAG_TBL("KTLS", SSL_OP_ENABLE_KTLS),
+        SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT)
     };
     if (value == NULL)
         return -3;

test/certs/client-pss-restrict-cert.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDZzCCAk+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdSb290
+IENBMCAXDTIyMDMyNTE1MzcwOFoYDzIxMjIwMzI2MTUzNzA4WjAZMRcwFQYDVQQD
+DA5DbGllbnQtUlNBLVBTUzCCAVIwPQYJKoZIhvcNAQEKMDCgDTALBglghkgBZQME
+AgGhGjAYBgkqhkiG9w0BAQgwCwYJYIZIAWUDBAIBogMCASADggEPADCCAQoCggEB
+ALNFZQLc/LFLrP8cOIdxgbIhx3mQoBfOst3XvfrxjMUHv5a+wouGvEms5431WHM0
+g/aJKArCHnz5M9ljr/xzLhZVyTtrjd4/59V+zUtptcytNeDdjrRBOoLuvAvoUz2B
+HBFmYMMGKWnUTSrp8yttUNirmJ0SpEp058ybo6Z4Tm6kZNojMu7TKLv2mwKdx+WE
+SGrbJ0nR7p9nMbyl0un6ExVduEbobMnnIk/bE49kbdCwDm+mTxF/j/dvW3+sV5c/
+bVVjRUcD0RZGgQD0SMExhex53DyhyjfV3ZNItZ+dcYOgKlo+DNilytczJa3jL28q
+xOpFz/xmU5Oc2k4jx4OSU40CAwEAAaOBjjCBizAdBgNVHQ4EFgQUXcDRXBMxM9Ua
+FdWhAKnZV3ZkbZowHwYDVR0jBBgwFoAUcH8uroNoWZgEIyrN6z4XzSTdAUkwCQYD
+VR0TBAIwADATBgNVHSUEDDAKBggrBgEFBQcDAjApBgNVHREEIjAggh5DbGllbnQg
+UlNBLVBTUyByZXN0cmljdGVkIGNlcnQwDQYJKoZIhvcNAQELBQADggEBAKDXTc7H
+g+o0UxsscFT4cklOFOOPKHGciOtNBylZLcs2K8TlN28sUMHal8bXGyh3tqBIMbLj
+KLfaUUUcysLKruZ+t5ANDJbPvCaF7C6AD53xoYcTTs3+p2XhFp85ivVgpmVU8c6L
+EfUpIr1vhBgUpRE3vdl6sRMB3PveSjBMDfq2f60LSX0mbydZRqeDO0lP5yg/FryH
+VLAtO3YvxQgglqNdtrErdxEAV20mthaSMxJguktTP+volr/3BSbIQfl3yuPnffk/
+hK8EgJeD13fJ9f8Gd4OXMXL98+Lii0gvTyJapw105KtKtZ/2ck2rOFLIKqFN/dk9
+W/mBy7X6U0O32tc=
+-----END CERTIFICATE-----

test/certs/client-pss-restrict-key.pem

@@ -0,0 +1,29 @@
+-----BEGIN PRIVATE KEY-----
+MIIE7QIBADA9BgkqhkiG9w0BAQowMKANMAsGCWCGSAFlAwQCAaEaMBgGCSqGSIb3
+DQEBCDALBglghkgBZQMEAgGiAwIBIASCBKcwggSjAgEAAoIBAQCzRWUC3PyxS6z/
+HDiHcYGyIcd5kKAXzrLd17368YzFB7+WvsKLhrxJrOeN9VhzNIP2iSgKwh58+TPZ
+Y6/8cy4WVck7a43eP+fVfs1LabXMrTXg3Y60QTqC7rwL6FM9gRwRZmDDBilp1E0q
+6fMrbVDYq5idEqRKdOfMm6OmeE5upGTaIzLu0yi79psCncflhEhq2ydJ0e6fZzG8
+pdLp+hMVXbhG6GzJ5yJP2xOPZG3QsA5vpk8Rf4/3b1t/rFeXP21VY0VHA9EWRoEA
+9EjBMYXsedw8oco31d2TSLWfnXGDoCpaPgzYpcrXMyWt4y9vKsTqRc/8ZlOTnNpO
+I8eDklONAgMBAAECggEBAKUMtO0n7HaHR+UwZFM/C7unIfIoV1zT7xYUNVM+5O3a
+LmhphM/U4rGqQR4PzrlyljR7HqSZCFzjSvtQroxstvfVT4ongdwnVhjXv8c4siqZ
+Jku7cFFA5M/7YKJN6aVsoxzZ9yhXGfXXgpyJ/Fn1MUPq6H1k1mG+tFNK0CbKCNwP
+cBFGIRT1dXHJaXjIyo+nfJs3kcN/y2trmtXfYrsOedMxVzAJD/Rn8Gw393wnrSJq
+dCcQ51RcxVjVe59x+mdnU5I+k5oe84uxJpQPT6i6EOoy1y7gNMAv2qncQT8iHM9h
+P/yr+kM96uPZpdELfRUkEWNfghR/bvqNtpfd3DedbPkCgYEA3oVMMYk2oU60pbmc
+Pk68joqJ3fFM3Bk9vVG65a2FbitFq7Fso1e4gwZCoLYCMZLVNMTIOhkKJEdH4I6o
+mxdA9ZaysiAYdDtsP4U/eYxQf/HNsworq7sP9xr0RvnAUixS+sc2B2VJYvyJfanc
+LgBIuqZiyRmbNlYV3GC09xMThw8CgYEAzj5GqBUDeUjcDNCR2ooAMjk6afFSFl8Q
+kvfASMsMxOF+P035k5LaE804rqM/5bsySGOCGNn+xMmxMKNh1UFAzbJXuTTo4Lv6
+r7SEc5i6usvXhk0zr/y083iY8rx9KOgHzWWmntJJr6Ax88wNH4UpPW1EV18D4ng4
+Ax9VmEjPW6MCgYAVzg4XVJDL4cCF9NhAqsqDVUQJQZn8f2SzZozf8M8AarEbD/nL
+T88+16Azy2IPpYnK7/WG6+k4gNukP5Z6DB9LcYb1OXvr9961osMDkCJbR0CW6Mo6
+u8vmtPd29QZJhxpihJ7gvqYgUwrOC5UN1O1LjP5lImM5QdpGjBtvkqj5NQKBgHgl
+K0ALTcS/vwDwF6d5sPeRAwhofmtt4dfb3/YH415mBgeWwwdHCydx681AaJ7J2Fb3
+MPiNNa8p18D/zKRQqRGrDRNlUSxqFXV58ZbtqAndaaZhHvUsf7U90cvGJhtIYBM1
+XkUzN53J8o+VlSeBiS6xkphbT4YEhoy7Gj/mWnWFAoGAU1bDM4GhIThnhk2sFgKn
+vDUBmu2fXiZXPJmrbITrBlpm6ocqNeFerhSmpU3oLFGQ5NZfMxLgvgLF5rRReY+c
+8P5Thav/RIpnFmD+wLxuDtJkpgWuz/4ySEZ7MAD8aLp2u3I1YHu2dFtY1hgeB5x/
+aqfWopW2cxBScbIToCnZnqg=
+-----END PRIVATE KEY-----

test/certs/setup.sh

@@ -413,6 +413,12 @@ openssl req -new -noenc -subj "/CN=localhost" \
     ./mkcert.sh geneenocsr "Server RSA-PSS restricted cert" \
     server-pss-restrict-cert rootkey rootcert
 
+openssl req -new -noenc -subj "/CN=Client-RSA-PSS" \
+    -newkey rsa-pss -keyout client-pss-restrict-key.pem \
+    -pkeyopt rsa_pss_keygen_md:sha256 -pkeyopt rsa_pss_keygen_saltlen:32 | \
+    ./mkcert.sh geneenocsr -p clientAuth "Client RSA-PSS restricted cert" \
+    client-pss-restrict-cert rootkey rootcert
+
 # CT entry
 ./mkcert.sh genct server.example embeddedSCTs1-key embeddedSCTs1 embeddedSCTs1_issuer-key embeddedSCTs1_issuer ct-server-key
 

test/ssl-tests/04-client_auth.cnf.in

@@ -155,6 +155,65 @@ sub generate_tests() {
                 };
                 $tests[-1]{"test"}{"UseSCTP"} = "Yes" if $sctp;
 
+                # Successful handshake with client RSA-PSS cert, StrictCertCheck
+                push @tests, {
+                    name => "client-auth-${protocol_name}-rsa-pss"
+                             .($sctp ? "-sctp" : ""),
+                    server => {
+                        "CipherString" => "DEFAULT:\@SECLEVEL=0",
+                        "MinProtocol" => $protocol,
+                        "MaxProtocol" => $protocol,
+                        "ClientCAFile" => test_pem("rootcert.pem"),
+                        "VerifyCAFile" => test_pem("rootcert.pem"),
+                        "VerifyMode" => "Require",
+                    },
+                    client => {
+                        "CipherString" => "DEFAULT:\@SECLEVEL=0",
+                        "MinProtocol" => $protocol,
+                        "MaxProtocol" => $protocol,
+                        "Certificate" => test_pem("client-pss-restrict-cert.pem"),
+                        "PrivateKey"  => test_pem("client-pss-restrict-key.pem"),
+                        "Options" => "StrictCertCheck",
+                    },
+                    test   => {
+                        "ExpectedResult" => "Success",
+                        "ExpectedClientCertType" => "RSA-PSS",
+                        "ExpectedClientCANames" => test_pem("rootcert.pem"),
+                        "Method" => $method,
+                    },
+                } if $protocol_name eq "TLSv1.2" || $protocol_name eq "flex";
+
+                # Failed handshake with client RSA-PSS cert, StrictCertCheck, bad CA
+                push @tests, {
+                    name => "client-auth-${protocol_name}-rsa-pss-bad"
+                             .($sctp ? "-sctp" : ""),
+                    server => {
+                        "CipherString" => "DEFAULT:\@SECLEVEL=0",
+                        "MinProtocol" => $protocol,
+                        "MaxProtocol" => $protocol,
+                        "ClientCAFile" => test_pem("rootCA.pem"),
+                        "VerifyCAFile" => test_pem("rootCA.pem"),
+                        "VerifyMode" => "Require",
+                    },
+                    client => {
+                        "CipherString" => "DEFAULT:\@SECLEVEL=0",
+                        "MinProtocol" => $protocol,
+                        "MaxProtocol" => $protocol,
+                        "Certificate" => test_pem("client-pss-restrict-cert.pem"),
+                        "PrivateKey"  => test_pem("client-pss-restrict-key.pem"),
+                        "Options" => "StrictCertCheck",
+                    },
+                    test   => {
+                        "ExpectedResult" => "ServerFail",
+                        "ExpectedServerAlert" =>
+                        ($protocol_name eq "flex"
+                            && !disabled("tls1_3")
+                            && (!disabled("ec") || !disabled("dh")))
+                        ? "CertificateRequired" : "HandshakeFailure",
+                        "Method" => $method,
+                    },
+                } if $protocol_name eq "TLSv1.2" || $protocol_name eq "flex";
+
                 # Successful handshake with client authentication non-empty names
                 push @tests, {
                     name => "client-auth-${protocol_name}-require-non-empty-names"