signature: add FIPS error state handling

The functions that check for the provider being runnable are: newctx, dupctx,
sign init, sign, verify init, verify, verify recover init, verify recover,
digest sign init, digest sign final, digest verify init and digest verify final.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12801)

providers/implementations/signature/dsa.c

@@ -26,7 +26,7 @@
 #include "internal/nelem.h"
 #include "internal/sizes.h"
 #include "internal/cryptlib.h"
-#include "prov/providercommonerr.h"
+#include "prov/providercommon.h"
 #include "prov/implementations.h"
 #include "prov/providercommonerr.h"
 #include "prov/provider_ctx.h"
@@ -134,8 +134,12 @@ static int dsa_get_md_nid(const EVP_MD *md)
 
 static void *dsa_newctx(void *provctx, const char *propq)
 {
-    PROV_DSA_CTX *pdsactx = OPENSSL_zalloc(sizeof(PROV_DSA_CTX));
+    PROV_DSA_CTX *pdsactx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
+    pdsactx = OPENSSL_zalloc(sizeof(PROV_DSA_CTX));
     if (pdsactx == NULL)
         return NULL;
 
@@ -196,7 +200,10 @@ static int dsa_signature_init(void *vpdsactx, void *vdsa)
 {
     PROV_DSA_CTX *pdsactx = (PROV_DSA_CTX *)vpdsactx;
 
-    if (pdsactx == NULL || vdsa == NULL || !DSA_up_ref(vdsa))
+    if (!ossl_prov_is_running()
+            || pdsactx == NULL
+            || vdsa == NULL
+            || !DSA_up_ref(vdsa))
         return 0;
     DSA_free(pdsactx->dsa);
     pdsactx->dsa = vdsa;
@@ -212,6 +219,9 @@ static int dsa_sign(void *vpdsactx, unsigned char *sig, size_t *siglen,
     size_t dsasize = DSA_size(pdsactx->dsa);
     size_t mdsize = dsa_get_md_size(pdsactx);
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (sig == NULL) {
         *siglen = dsasize;
         return 1;
@@ -237,7 +247,7 @@ static int dsa_verify(void *vpdsactx, const unsigned char *sig, size_t siglen,
     PROV_DSA_CTX *pdsactx = (PROV_DSA_CTX *)vpdsactx;
     size_t mdsize = dsa_get_md_size(pdsactx);
 
-    if (mdsize != 0 && tbslen != mdsize)
+    if (!ossl_prov_is_running() || (mdsize != 0 && tbslen != mdsize))
         return 0;
 
     return DSA_verify(0, tbs, tbslen, sig, siglen, pdsactx->dsa);
@@ -248,6 +258,9 @@ static int dsa_digest_signverify_init(void *vpdsactx, const char *mdname,
 {
     PROV_DSA_CTX *pdsactx = (PROV_DSA_CTX *)vpdsactx;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     pdsactx->flag_allow_md = 0;
     if (!dsa_signature_init(vpdsactx, vdsa))
         return 0;
@@ -290,7 +303,7 @@ int dsa_digest_sign_final(void *vpdsactx, unsigned char *sig, size_t *siglen,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
-    if (pdsactx == NULL || pdsactx->mdctx == NULL)
+    if (!ossl_prov_is_running() || pdsactx == NULL || pdsactx->mdctx == NULL)
         return 0;
 
     /*
@@ -320,7 +333,7 @@ int dsa_digest_verify_final(void *vpdsactx, const unsigned char *sig,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
-    if (pdsactx == NULL || pdsactx->mdctx == NULL)
+    if (!ossl_prov_is_running() || pdsactx == NULL || pdsactx->mdctx == NULL)
         return 0;
 
     /*
@@ -356,6 +369,9 @@ static void *dsa_dupctx(void *vpdsactx)
     PROV_DSA_CTX *srcctx = (PROV_DSA_CTX *)vpdsactx;
     PROV_DSA_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL)
         return NULL;

providers/implementations/signature/ecdsa.c

@@ -24,6 +24,7 @@
 #include "internal/nelem.h"
 #include "internal/sizes.h"
 #include "internal/cryptlib.h"
+#include "prov/providercommon.h"
 #include "prov/providercommonerr.h"
 #include "prov/implementations.h"
 #include "prov/provider_ctx.h"
@@ -95,8 +96,12 @@ typedef struct {
 
 static void *ecdsa_newctx(void *provctx, const char *propq)
 {
-    PROV_ECDSA_CTX *ctx = OPENSSL_zalloc(sizeof(PROV_ECDSA_CTX));
+    PROV_ECDSA_CTX *ctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
+    ctx = OPENSSL_zalloc(sizeof(PROV_ECDSA_CTX));
     if (ctx == NULL)
         return NULL;
 
@@ -113,7 +118,10 @@ static int ecdsa_signature_init(void *vctx, void *ec)
 {
     PROV_ECDSA_CTX *ctx = (PROV_ECDSA_CTX *)vctx;
 
-    if (ctx == NULL || ec == NULL || !EC_KEY_up_ref(ec))
+    if (!ossl_prov_is_running()
+            || ctx == NULL
+            || ec == NULL
+            || !EC_KEY_up_ref(ec))
         return 0;
     EC_KEY_free(ctx->ec);
     ctx->ec = ec;
@@ -128,6 +136,9 @@ static int ecdsa_sign(void *vctx, unsigned char *sig, size_t *siglen,
     unsigned int sltmp;
     size_t ecsize = ECDSA_size(ctx->ec);
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (sig == NULL) {
         *siglen = ecsize;
         return 1;
@@ -157,7 +168,7 @@ static int ecdsa_verify(void *vctx, const unsigned char *sig, size_t siglen,
 {
     PROV_ECDSA_CTX *ctx = (PROV_ECDSA_CTX *)vctx;
 
-    if (ctx->mdsize != 0 && tbslen != ctx->mdsize)
+    if (!ossl_prov_is_running() || (ctx->mdsize != 0 && tbslen != ctx->mdsize))
         return 0;
 
     return ECDSA_verify(0, tbs, tbslen, sig, siglen, ctx->ec);
@@ -221,6 +232,9 @@ static int ecdsa_digest_signverify_init(void *vctx, const char *mdname,
     int md_nid = NID_undef;
     WPACKET pkt;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     free_md(ctx);
 
     if (!ecdsa_signature_init(vctx, ec))
@@ -277,7 +291,7 @@ int ecdsa_digest_sign_final(void *vctx, unsigned char *sig, size_t *siglen,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
-    if (ctx == NULL || ctx->mdctx == NULL)
+    if (!ossl_prov_is_running() || ctx == NULL || ctx->mdctx == NULL)
         return 0;
 
     /*
@@ -304,7 +318,7 @@ int ecdsa_digest_verify_final(void *vctx, const unsigned char *sig,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
-    if (ctx == NULL || ctx->mdctx == NULL)
+    if (!ossl_prov_is_running() || ctx == NULL || ctx->mdctx == NULL)
         return 0;
 
     /*
@@ -334,6 +348,9 @@ static void *ecdsa_dupctx(void *vctx)
     PROV_ECDSA_CTX *srcctx = (PROV_ECDSA_CTX *)vctx;
     PROV_ECDSA_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL)
         return NULL;

providers/implementations/signature/eddsa.c

@@ -16,7 +16,7 @@
 #include <openssl/err.h>
 #include "internal/nelem.h"
 #include "internal/sizes.h"
-#include "prov/providercommonerr.h"
+#include "prov/providercommon.h"
 #include "prov/implementations.h"
 #include "prov/providercommonerr.h"
 #include "prov/provider_ctx.h"
@@ -38,8 +38,12 @@ typedef struct {
 
 static void *eddsa_newctx(void *provctx, const char *propq_unused)
 {
-    PROV_EDDSA_CTX *peddsactx = OPENSSL_zalloc(sizeof(PROV_EDDSA_CTX));
+    PROV_EDDSA_CTX *peddsactx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
+    peddsactx = OPENSSL_zalloc(sizeof(PROV_EDDSA_CTX));
     if (peddsactx == NULL) {
         PROVerr(0, ERR_R_MALLOC_FAILURE);
         return NULL;
@@ -56,6 +60,9 @@ static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname,
     PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx;
     ECX_KEY *edkey = (ECX_KEY *)vedkey;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (mdname != NULL && mdname[0] != '\0') {
         PROVerr(0, PROV_R_INVALID_DIGEST);
         return 0;
@@ -78,6 +85,9 @@ int ed25519_digest_sign(void *vpeddsactx, unsigned char *sigret,
     PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx;
     const ECX_KEY *edkey = peddsactx->key;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (sigret == NULL) {
         *siglen = ED25519_SIGSIZE;
         return 1;
@@ -103,6 +113,9 @@ int ed448_digest_sign(void *vpeddsactx, unsigned char *sigret,
     PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx;
     const ECX_KEY *edkey = peddsactx->key;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (sigret == NULL) {
         *siglen = ED448_SIGSIZE;
         return 1;
@@ -128,7 +141,7 @@ int ed25519_digest_verify(void *vpeddsactx, const unsigned char *sig,
     PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx;
     const ECX_KEY *edkey = peddsactx->key;
 
-    if (siglen != ED25519_SIGSIZE)
+    if (!ossl_prov_is_running() || siglen != ED25519_SIGSIZE)
         return 0;
 
     return ED25519_verify(tbs, tbslen, sig, edkey->pubkey, peddsactx->libctx,
@@ -142,7 +155,7 @@ int ed448_digest_verify(void *vpeddsactx, const unsigned char *sig,
     PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx;
     const ECX_KEY *edkey = peddsactx->key;
 
-    if (siglen != ED448_SIGSIZE)
+    if (!ossl_prov_is_running() || siglen != ED448_SIGSIZE)
         return 0;
 
     return ED448_verify(peddsactx->libctx, tbs, tbslen, sig, edkey->pubkey,
@@ -163,6 +176,9 @@ static void *eddsa_dupctx(void *vpeddsactx)
     PROV_EDDSA_CTX *srcctx = (PROV_EDDSA_CTX *)vpeddsactx;
     PROV_EDDSA_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL)
         return NULL;

providers/implementations/signature/mac_legacy.c

@@ -19,6 +19,7 @@
 #include "prov/implementations.h"
 #include "prov/provider_ctx.h"
 #include "prov/macsignature.h"
+#include "prov/providercommon.h"
 
 static OSSL_FUNC_signature_newctx_fn mac_hmac_newctx;
 static OSSL_FUNC_signature_newctx_fn mac_siphash_newctx;
@@ -44,9 +45,13 @@ typedef struct {
 
 static void *mac_newctx(void *provctx, const char *propq, const char *macname)
 {
-    PROV_MAC_CTX *pmacctx = OPENSSL_zalloc(sizeof(PROV_MAC_CTX));
+    PROV_MAC_CTX *pmacctx;
     EVP_MAC *mac = NULL;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
+    pmacctx = OPENSSL_zalloc(sizeof(PROV_MAC_CTX));
     if (pmacctx == NULL)
         return NULL;
 
@@ -90,7 +95,10 @@ static int mac_digest_sign_init(void *vpmacctx, const char *mdname, void *vkey)
     PROV_MAC_CTX *pmacctx = (PROV_MAC_CTX *)vpmacctx;
     const char *ciphername = NULL, *engine = NULL;
 
-    if (pmacctx == NULL || vkey == NULL || !mac_key_up_ref(vkey))
+    if (!ossl_prov_is_running()
+            || pmacctx == NULL
+            || vkey == NULL
+            || !mac_key_up_ref(vkey))
         return 0;
 
     mac_key_free(pmacctx->key);
@@ -134,7 +142,7 @@ int mac_digest_sign_final(void *vpmacctx, unsigned char *mac, size_t *maclen,
 {
     PROV_MAC_CTX *pmacctx = (PROV_MAC_CTX *)vpmacctx;
 
-    if (pmacctx == NULL || pmacctx->macctx == NULL)
+    if (!ossl_prov_is_running() || pmacctx == NULL || pmacctx->macctx == NULL)
         return 0;
 
     return EVP_MAC_final(pmacctx->macctx, mac, maclen, macsize);
@@ -155,6 +163,9 @@ static void *mac_dupctx(void *vpmacctx)
     PROV_MAC_CTX *srcctx = (PROV_MAC_CTX *)vpmacctx;
     PROV_MAC_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL)
         return NULL;

providers/implementations/signature/rsa.c

@@ -25,6 +25,7 @@
 #include "internal/nelem.h"
 #include "internal/sizes.h"
 #include "crypto/rsa.h"
+#include "prov/providercommon.h"
 #include "prov/providercommonerr.h"
 #include "prov/implementations.h"
 #include "prov/provider_ctx.h"
@@ -198,6 +199,9 @@ static void *rsa_newctx(void *provctx, const char *propq)
     PROV_RSA_CTX *prsactx = NULL;
     char *propq_copy = NULL;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     if ((prsactx = OPENSSL_zalloc(sizeof(PROV_RSA_CTX))) == NULL
         || (propq != NULL
             && (propq_copy = OPENSSL_strdup(propq)) == NULL)) {
@@ -303,6 +307,9 @@ static int rsa_signature_init(void *vprsactx, void *vrsa, int operation)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (prsactx == NULL || vrsa == NULL || !RSA_up_ref(vrsa))
         return 0;
 
@@ -404,6 +411,8 @@ static void free_tbuf(PROV_RSA_CTX *ctx)
 
 static int rsa_sign_init(void *vprsactx, void *vrsa)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_SIGN);
 }
 
@@ -415,6 +424,9 @@ static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen,
     size_t rsasize = RSA_size(prsactx->rsa);
     size_t mdsize = rsa_get_md_size(prsactx);
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (sig == NULL) {
         *siglen = rsasize;
         return 1;
@@ -552,6 +564,8 @@ static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen,
 
 static int rsa_verify_recover_init(void *vprsactx, void *vrsa)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFYRECOVER);
 }
 
@@ -565,6 +579,9 @@ static int rsa_verify_recover(void *vprsactx,
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
     int ret;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (rout == NULL) {
         *routlen = RSA_size(prsactx->rsa);
         return 1;
@@ -638,6 +655,8 @@ static int rsa_verify_recover(void *vprsactx,
 
 static int rsa_verify_init(void *vprsactx, void *vrsa)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFY);
 }
 
@@ -647,6 +666,8 @@ static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
     size_t rslen;
 
+    if (!ossl_prov_is_running())
+        return 0;
     if (prsactx->md != NULL) {
         switch (prsactx->pad_mode) {
         case RSA_PKCS1_PADDING:
@@ -725,6 +746,9 @@ static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (prsactx != NULL)
         prsactx->flag_allow_md = 0;
     if (!rsa_signature_init(vprsactx, vrsa, operation)
@@ -765,6 +789,8 @@ static int rsa_digest_signverify_update(void *vprsactx,
 static int rsa_digest_sign_init(void *vprsactx, const char *mdname,
                                 void *vrsa)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
                                       EVP_PKEY_OP_SIGN);
 }
@@ -776,7 +802,7 @@ static int rsa_digest_sign_final(void *vprsactx, unsigned char *sig,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
-    if (prsactx == NULL)
+    if (!ossl_prov_is_running() || prsactx == NULL)
         return 0;
     prsactx->flag_allow_md = 1;
     if (prsactx->mdctx == NULL)
@@ -801,6 +827,8 @@ static int rsa_digest_sign_final(void *vprsactx, unsigned char *sig,
 static int rsa_digest_verify_init(void *vprsactx, const char *mdname,
                                   void *vrsa)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     return rsa_digest_signverify_init(vprsactx, mdname, vrsa,
                                       EVP_PKEY_OP_VERIFY);
 }
@@ -812,6 +840,9 @@ int rsa_digest_verify_final(void *vprsactx, const unsigned char *sig,
     unsigned char digest[EVP_MAX_MD_SIZE];
     unsigned int dlen = 0;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (prsactx == NULL)
         return 0;
     prsactx->flag_allow_md = 1;
@@ -851,6 +882,9 @@ static void *rsa_dupctx(void *vprsactx)
     PROV_RSA_CTX *srcctx = (PROV_RSA_CTX *)vprsactx;
     PROV_RSA_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL) {
         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);