Detect low-level engine and app method based keys

The low-level engine and app method based keys have to be treated
as foreign and must be used with old legacy pmeths.

Fixes #14632

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14859)

crypto/dh/dh_backend.c

@@ -118,6 +118,15 @@ int ossl_dh_key_todata(DH *dh, OSSL_PARAM_BLD *bld, OSSL_PARAM params[])
     return 1;
 }
 
+int ossl_dh_is_foreign(const DH *dh)
+{
+#ifndef FIPS_MODULE
+    if (dh->engine != NULL || ossl_dh_get_method(dh) != DH_OpenSSL())
+        return 1;
+#endif
+    return 0;
+}
+
 static ossl_inline int dh_bn_dup_check(BIGNUM **out, const BIGNUM *f)
 {
     if (f != NULL && (*out = BN_dup(f)) == NULL)
@@ -129,11 +138,9 @@ DH *ossl_dh_dup(const DH *dh, int selection)
 {
     DH *dupkey = NULL;
 
-#ifndef FIPS_MODULE
     /* Do not try to duplicate foreign DH keys */
-    if (ossl_dh_get_method(dh) != DH_OpenSSL())
+    if (ossl_dh_is_foreign(dh))
         return NULL;
-#endif
 
     if ((dupkey = ossl_dh_new_ex(dh->libctx)) == NULL)
         return NULL;

crypto/dsa/dsa_backend.c

@@ -57,6 +57,15 @@ int ossl_dsa_key_fromdata(DSA *dsa, const OSSL_PARAM params[])
     return 0;
 }
 
+int ossl_dsa_is_foreign(const DSA *dsa)
+{
+#ifndef FIPS_MODULE
+    if (dsa->engine != NULL || DSA_get_method((DSA *)dsa) != DSA_OpenSSL())
+        return 1;
+#endif
+    return 0;
+}
+
 static ossl_inline int dsa_bn_dup_check(BIGNUM **out, const BIGNUM *f)
 {
     if (f != NULL && (*out = BN_dup(f)) == NULL)
@@ -68,11 +77,9 @@ DSA *ossl_dsa_dup(const DSA *dsa, int selection)
 {
     DSA *dupkey = NULL;
 
-#ifndef FIPS_MODULE
     /* Do not try to duplicate foreign DSA keys */
-    if (DSA_get_method((DSA *)dsa) != DSA_OpenSSL())
+    if (ossl_dsa_is_foreign(dsa))
         return NULL;
-#endif
 
     if ((dupkey = ossl_dsa_new(dsa->libctx)) == NULL)
         return NULL;

crypto/ec/ec_backend.c

@@ -520,6 +520,16 @@ int ossl_ec_key_otherparams_fromdata(EC_KEY *ec, const OSSL_PARAM params[])
     return 1;
 }
 
+int ossl_ec_key_is_foreign(const EC_KEY *ec)
+{
+#ifndef FIPS_MODULE
+    if (ec->engine != NULL || EC_KEY_get_method(ec) != EC_KEY_OpenSSL())
+        return 1;
+#endif
+    return 0;
+
+}
+
 EC_KEY *ossl_ec_key_dup(const EC_KEY *src, int selection)
 {
     EC_KEY *ret = ossl_ec_key_new_method_int(src->libctx, src->propq,

crypto/evp/p_lib.c

@@ -37,14 +37,15 @@
 #include "internal/ffc.h"
 #include "crypto/asn1.h"
 #include "crypto/evp.h"
+#include "crypto/dh.h"
+#include "crypto/dsa.h"
 #include "crypto/ec.h"
 #include "crypto/ecx.h"
+#include "crypto/rsa.h"
 #include "crypto/x509.h"
 #include "internal/provider.h"
 #include "evp_local.h"
 
-#include "crypto/ec.h"
-
 #include "e_os.h"                /* strcasecmp on Windows */
 
 static int pkey_set_type(EVP_PKEY *pkey, ENGINE *e, int type, const char *str,
@@ -691,6 +692,38 @@ ENGINE *EVP_PKEY_get0_engine(const EVP_PKEY *pkey)
 # endif
 
 # ifndef OPENSSL_NO_DEPRECATED_3_0
+static void detect_foreign_key(EVP_PKEY *pkey)
+{
+    switch (pkey->type) {
+    case EVP_PKEY_RSA:
+        pkey->foreign = pkey->pkey.rsa != NULL
+                        && ossl_rsa_is_foreign(pkey->pkey.rsa);
+        break;
+#  ifndef OPENSSL_NO_EC
+    case EVP_PKEY_SM2:
+    case EVP_PKEY_EC:
+        pkey->foreign = pkey->pkey.ec != NULL
+                        && ossl_ec_key_is_foreign(pkey->pkey.ec);
+        break;
+#  endif
+#  ifndef OPENSSL_NO_DSA
+    case EVP_PKEY_DSA:
+        pkey->foreign = pkey->pkey.dsa != NULL
+                        && ossl_dsa_is_foreign(pkey->pkey.dsa);
+        break;
+#endif
+#  ifndef OPENSSL_NO_DH
+    case EVP_PKEY_DH:
+        pkey->foreign = pkey->pkey.dh != NULL
+                        && ossl_dh_is_foreign(pkey->pkey.dh);
+        break;
+#endif
+    default:
+        pkey->foreign = 0;
+        break;
+    }
+}
+
 int EVP_PKEY_assign(EVP_PKEY *pkey, int type, void *key)
 {
 #  ifndef OPENSSL_NO_EC
@@ -719,6 +752,8 @@ int EVP_PKEY_assign(EVP_PKEY *pkey, int type, void *key)
         return 0;
 
     pkey->pkey.ptr = key;
+    detect_foreign_key(pkey);
+
     return (key != NULL);
 }
 # endif
@@ -1354,7 +1389,6 @@ EVP_PKEY *EVP_PKEY_new(void)
     ret->type = EVP_PKEY_NONE;
     ret->save_type = EVP_PKEY_NONE;
     ret->references = 1;
-    ret->save_parameters = 1;
 
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
@@ -1363,6 +1397,7 @@ EVP_PKEY *EVP_PKEY_new(void)
     }
 
 #ifndef FIPS_MODULE
+    ret->save_parameters = 1;
     if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_EVP_PKEY, ret, &ret->ex_data)) {
         ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE);
         goto err;

crypto/evp/pmeth_lib.c

@@ -224,7 +224,7 @@ static EVP_PKEY_CTX *int_ctx_new(OSSL_LIB_CTX *libctx,
      */
     if (!ossl_assert(e == NULL || keytype == NULL))
         return NULL;
-    if (e == NULL)
+    if (e == NULL && (pkey == NULL || pkey->foreign == 0))
         keytype = OBJ_nid2sn(id);
 
 # ifndef OPENSSL_NO_ENGINE
@@ -246,6 +246,8 @@ static EVP_PKEY_CTX *int_ctx_new(OSSL_LIB_CTX *libctx,
      */
     if (e != NULL)
         pmeth = ENGINE_get_pkey_meth(e, id);
+    else if (pkey != NULL && pkey->foreign)
+        pmeth = EVP_PKEY_meth_find(id);
     else
 # endif
         pmeth = evp_pkey_meth_find_added_by_application(id);

crypto/rsa/rsa_backend.c

@@ -323,6 +323,15 @@ int ossl_rsa_pss_params_30_fromdata(RSA_PSS_PARAMS_30 *pss_params,
     return ret;
 }
 
+int ossl_rsa_is_foreign(const RSA *rsa)
+{
+#ifndef FIPS_MODULE
+    if (rsa->engine != NULL || RSA_get_method(rsa) != RSA_PKCS1_OpenSSL())
+        return 1;
+#endif
+    return 0;
+}
+
 static ossl_inline int rsa_bn_dup_check(BIGNUM **out, const BIGNUM *f)
 {
     if (f != NULL && (*out = BN_dup(f)) == NULL)
@@ -335,11 +344,11 @@ RSA *ossl_rsa_dup(const RSA *rsa, int selection)
     RSA *dupkey = NULL;
 #ifndef FIPS_MODULE
     int pnum, i;
+#endif
 
     /* Do not try to duplicate foreign RSA keys */
-    if (RSA_get_method(rsa) != RSA_PKCS1_OpenSSL())
+    if (ossl_rsa_is_foreign(rsa))
         return NULL;
-#endif
 
     if ((dupkey = ossl_rsa_new_with_ctx(rsa->libctx)) == NULL)
         return NULL;

include/crypto/dh.h

@@ -56,6 +56,7 @@ int ossl_dh_kdf_X9_42_asn1(unsigned char *out, size_t outlen,
                            const unsigned char *ukm, size_t ukmlen,
                            const EVP_MD *md,
                            OSSL_LIB_CTX *libctx, const char *propq);
+int ossl_dh_is_foreign(const DH *dh);
 DH *ossl_dh_dup(const DH *dh, int selection);
 
 #endif  /* OSSL_CRYPTO_DH_H */

include/crypto/dsa.h

@@ -43,6 +43,7 @@ int ossl_dsa_check_pub_key_partial(const DSA *dsa, const BIGNUM *pub_key,
                                    int *ret);
 int ossl_dsa_check_priv_key(const DSA *dsa, const BIGNUM *priv_key, int *ret);
 int ossl_dsa_check_pairwise(const DSA *dsa);
+int ossl_dsa_is_foreign(const DSA *dsa);
 DSA *ossl_dsa_dup(const DSA *dsa, int selection);
 
 #endif

include/crypto/ec.h

@@ -79,6 +79,7 @@ int ossl_ec_group_set_params(EC_GROUP *group, const OSSL_PARAM params[]);
 int ossl_ec_key_fromdata(EC_KEY *ecx, const OSSL_PARAM params[],
                          int include_private);
 int ossl_ec_key_otherparams_fromdata(EC_KEY *ec, const OSSL_PARAM params[]);
+int ossl_ec_key_is_foreign(const EC_KEY *ec);
 EC_KEY *ossl_ec_key_dup(const EC_KEY *key, int selection);
 EC_KEY *ossl_ec_key_param_from_x509_algor(const X509_ALGOR *palg,
                                           OSSL_LIB_CTX *libctx,

include/crypto/evp.h

@@ -663,9 +663,10 @@ struct evp_pkey_st {
     /* == Common attributes == */
     CRYPTO_REF_COUNT references;
     CRYPTO_RWLOCK *lock;
+#ifndef FIPS_MODULE
     STACK_OF(X509_ATTRIBUTE) *attributes; /* [ 0 ] */
     int save_parameters;
-#ifndef FIPS_MODULE
+    int foreign:1; /* the low-level key is using an engine or an app-method */
     CRYPTO_EX_DATA ex_data;
 #endif
 

include/crypto/rsa.h

@@ -63,6 +63,7 @@ int ossl_rsa_set0_all_params(RSA *r, const STACK_OF(BIGNUM) *primes,
 int ossl_rsa_get0_all_params(RSA *r, STACK_OF(BIGNUM_const) *primes,
                              STACK_OF(BIGNUM_const) *exps,
                              STACK_OF(BIGNUM_const) *coeffs);
+int ossl_rsa_is_foreign(const RSA *rsa);
 RSA *ossl_rsa_dup(const RSA *rsa, int selection);
 
 int ossl_rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[]);