diff --git a/crypto/bio/bss_core.c b/crypto/bio/bss_core.c
index 2baabe614e..89b1ef7395 100644
--- a/crypto/bio/bss_core.c
+++ b/crypto/bio/bss_core.c
@@ -30,6 +30,7 @@ static void *bio_core_globals_new(OSSL_LIB_CTX *ctx)
 }
 
 static const OSSL_LIB_CTX_METHOD bio_core_globals_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     bio_core_globals_new,
     bio_core_globals_free,
 };
diff --git a/crypto/context.c b/crypto/context.c
index 4ea949970a..b21fdf077d 100644
--- a/crypto/context.c
+++ b/crypto/context.c
@@ -305,7 +305,8 @@ static int ossl_lib_ctx_init_index(OSSL_LIB_CTX *ctx, int static_index,
     idx = ossl_crypto_get_ex_new_index_ex(ctx, CRYPTO_EX_INDEX_OSSL_LIB_CTX, 0,
                                           (void *)meth,
                                           ossl_lib_ctx_generic_new,
-                                          NULL, ossl_lib_ctx_generic_free);
+                                          NULL, ossl_lib_ctx_generic_free,
+                                          meth->priority);
     if (idx < 0)
         return 0;
 
diff --git a/crypto/core_namemap.c b/crypto/core_namemap.c
index 1009fb1e94..5bb0f09ec7 100644
--- a/crypto/core_namemap.c
+++ b/crypto/core_namemap.c
@@ -87,6 +87,7 @@ static void stored_namemap_free(void *vnamemap)
 }
 
 static const OSSL_LIB_CTX_METHOD stored_namemap_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     stored_namemap_new,
     stored_namemap_free,
 };
diff --git a/crypto/encode_decode/decoder_meth.c b/crypto/encode_decode/decoder_meth.c
index 7f8a365b66..7a271f7408 100644
--- a/crypto/encode_decode/decoder_meth.c
+++ b/crypto/encode_decode/decoder_meth.c
@@ -76,6 +76,7 @@ static void *decoder_store_new(OSSL_LIB_CTX *ctx)
 
 
 static const OSSL_LIB_CTX_METHOD decoder_store_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     decoder_store_new,
     decoder_store_free,
 };
diff --git a/crypto/encode_decode/encoder_meth.c b/crypto/encode_decode/encoder_meth.c
index de0a66578c..bb319460b9 100644
--- a/crypto/encode_decode/encoder_meth.c
+++ b/crypto/encode_decode/encoder_meth.c
@@ -76,6 +76,7 @@ static void *encoder_store_new(OSSL_LIB_CTX *ctx)
 
 
 static const OSSL_LIB_CTX_METHOD encoder_store_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     encoder_store_new,
     encoder_store_free,
 };
diff --git a/crypto/evp/evp_fetch.c b/crypto/evp/evp_fetch.c
index 266f657ff2..b74379031b 100644
--- a/crypto/evp/evp_fetch.c
+++ b/crypto/evp/evp_fetch.c
@@ -35,6 +35,8 @@ static void *evp_method_store_new(OSSL_LIB_CTX *ctx)
 
 
 static const OSSL_LIB_CTX_METHOD evp_method_store_method = {
+    /* We want evp_method_store to be cleaned up before the provider store */
+    OSSL_LIB_CTX_METHOD_HIGH_PRIORITY,
     evp_method_store_new,
     evp_method_store_free,
 };
diff --git a/crypto/ex_data.c b/crypto/ex_data.c
index 4a0efbdb18..40223f06e4 100644
--- a/crypto/ex_data.c
+++ b/crypto/ex_data.c
@@ -7,6 +7,7 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <stdlib.h>
 #include "crypto/cryptlib.h"
 #include "internal/thread_once.h"
 
@@ -141,7 +142,8 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index,
                                     long argl, void *argp,
                                     CRYPTO_EX_new *new_func,
                                     CRYPTO_EX_dup *dup_func,
-                                    CRYPTO_EX_free *free_func)
+                                    CRYPTO_EX_free *free_func,
+                                    int priority)
 {
     int toret = -1;
     EX_CALLBACK *a;
@@ -176,6 +178,7 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index,
     a->new_func = new_func;
     a->dup_func = dup_func;
     a->free_func = free_func;
+    a->priority = priority;
 
     if (!sk_EX_CALLBACK_push(ip->meth, NULL)) {
         ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
@@ -195,7 +198,7 @@ int CRYPTO_get_ex_new_index(int class_index, long argl, void *argp,
                             CRYPTO_EX_free *free_func)
 {
     return ossl_crypto_get_ex_new_index_ex(NULL, class_index, argl, argp,
-                                           new_func, dup_func, free_func);
+                                           new_func, dup_func, free_func, 0);
 }
 
 /*
@@ -331,6 +334,27 @@ int CRYPTO_dup_ex_data(int class_index, CRYPTO_EX_DATA *to,
     return toret;
 }
 
+struct ex_callback_entry {
+    const EX_CALLBACK *excb;
+    int index;
+};
+
+static int ex_callback_compare(const void *a, const void *b)
+{
+    const struct ex_callback_entry *ap = (const struct ex_callback_entry *)a;
+    const struct ex_callback_entry *bp = (const struct ex_callback_entry *)b;
+
+    if (ap->excb == bp->excb)
+        return 0;
+
+    if (ap->excb == NULL)
+        return 1;
+    if (bp->excb == NULL)
+        return -1;
+    if (ap->excb->priority == bp->excb->priority)
+        return 0;
+    return ap->excb->priority > bp->excb->priority ? -1 : 1;
+}
 
 /*
  * Cleanup a CRYPTO_EX_DATA variable - including calling free() callbacks for
@@ -341,9 +365,9 @@ void CRYPTO_free_ex_data(int class_index, void *obj, CRYPTO_EX_DATA *ad)
     int mx, i;
     EX_CALLBACKS *ip;
     void *ptr;
-    EX_CALLBACK *f;
-    EX_CALLBACK *stack[10];
-    EX_CALLBACK **storage = NULL;
+    const EX_CALLBACK *f;
+    struct ex_callback_entry stack[10];
+    struct ex_callback_entry *storage = NULL;
     OSSL_EX_DATA_GLOBAL *global = ossl_lib_ctx_get_ex_data_global(ad->ctx);
 
     if (global == NULL)
@@ -360,23 +384,23 @@ void CRYPTO_free_ex_data(int class_index, void *obj, CRYPTO_EX_DATA *ad)
         else
             storage = OPENSSL_malloc(sizeof(*storage) * mx);
         if (storage != NULL)
-            for (i = 0; i < mx; i++)
-                storage[i] = sk_EX_CALLBACK_value(ip->meth, i);
+            for (i = 0; i < mx; i++) {
+                storage[i].excb = sk_EX_CALLBACK_value(ip->meth, i);
+                storage[i].index = i;
+            }
     }
     CRYPTO_THREAD_unlock(global->ex_data_lock);
 
-    for (i = 0; i < mx; i++) {
-        if (storage != NULL)
-            f = storage[i];
-        else {
-            if (!CRYPTO_THREAD_write_lock(global->ex_data_lock))
-                continue;
-            f = sk_EX_CALLBACK_value(ip->meth, i);
-            CRYPTO_THREAD_unlock(global->ex_data_lock);
-        }
-        if (f != NULL && f->free_func != NULL) {
-            ptr = CRYPTO_get_ex_data(ad, i);
-            f->free_func(obj, ptr, ad, i, f->argl, f->argp);
+    if (storage != NULL) {
+        /* Sort according to priority. High priority first */
+        qsort(storage, mx, sizeof(*storage), ex_callback_compare);
+        for (i = 0; i < mx; i++) {
+            f = storage[i].excb;
+
+            if (f != NULL && f->free_func != NULL) {
+                ptr = CRYPTO_get_ex_data(ad, storage[i].index);
+                f->free_func(obj, ptr, ad, storage[i].index, f->argl, f->argp);
+            }
         }
     }
 
diff --git a/crypto/initthread.c b/crypto/initthread.c
index 0740668071..fec3213047 100644
--- a/crypto/initthread.c
+++ b/crypto/initthread.c
@@ -278,6 +278,7 @@ static void thread_event_ossl_ctx_free(void *tlocal)
 }
 
 static const OSSL_LIB_CTX_METHOD thread_event_ossl_ctx_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     thread_event_ossl_ctx_new,
     thread_event_ossl_ctx_free,
 };
diff --git a/crypto/property/defn_cache.c b/crypto/property/defn_cache.c
index 6c6503bdcc..8007599526 100644
--- a/crypto/property/defn_cache.c
+++ b/crypto/property/defn_cache.c
@@ -63,6 +63,7 @@ static void *property_defns_new(OSSL_LIB_CTX *ctx) {
 }
 
 static const OSSL_LIB_CTX_METHOD property_defns_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     property_defns_new,
     property_defns_free,
 };
diff --git a/crypto/property/property.c b/crypto/property/property.c
index 2b841a2204..da6bc84e27 100644
--- a/crypto/property/property.c
+++ b/crypto/property/property.c
@@ -94,6 +94,7 @@ static void *ossl_ctx_global_properties_new(OSSL_LIB_CTX *ctx)
 
 
 static const OSSL_LIB_CTX_METHOD ossl_ctx_global_properties_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     ossl_ctx_global_properties_new,
     ossl_ctx_global_properties_free,
 };
diff --git a/crypto/property/property_string.c b/crypto/property/property_string.c
index 90bb322faa..9eb55cb461 100644
--- a/crypto/property/property_string.c
+++ b/crypto/property/property_string.c
@@ -105,6 +105,7 @@ err:
 }
 
 static const OSSL_LIB_CTX_METHOD property_string_data_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     property_string_data_new,
     property_string_data_free,
 };
diff --git a/crypto/provider_conf.c b/crypto/provider_conf.c
index a04a7aa553..6223d27cbb 100644
--- a/crypto/provider_conf.c
+++ b/crypto/provider_conf.c
@@ -45,6 +45,8 @@ static void prov_conf_ossl_ctx_free(void *vpcgbl)
 }
 
 static const OSSL_LIB_CTX_METHOD provider_conf_ossl_ctx_method = {
+    /* Must be freed before the provider store is freed */
+    OSSL_LIB_CTX_METHOD_HIGH_PRIORITY,
     prov_conf_ossl_ctx_new,
     prov_conf_ossl_ctx_free,
 };
diff --git a/crypto/provider_core.c b/crypto/provider_core.c
index 3d853eb66b..db8763d50d 100644
--- a/crypto/provider_core.c
+++ b/crypto/provider_core.c
@@ -189,6 +189,7 @@ static void *provider_store_new(OSSL_LIB_CTX *ctx)
 }
 
 static const OSSL_LIB_CTX_METHOD provider_store_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     provider_store_new,
     provider_store_free,
 };
diff --git a/crypto/rand/rand_lib.c b/crypto/rand/rand_lib.c
index f6c5bc15ee..bdf5f71f44 100644
--- a/crypto/rand/rand_lib.c
+++ b/crypto/rand/rand_lib.c
@@ -486,6 +486,7 @@ static void rand_ossl_ctx_free(void *vdgbl)
 }
 
 static const OSSL_LIB_CTX_METHOD rand_drbg_ossl_ctx_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     rand_ossl_ctx_new,
     rand_ossl_ctx_free,
 };
diff --git a/crypto/self_test_core.c b/crypto/self_test_core.c
index a4f6c9ab2a..341af7b194 100644
--- a/crypto/self_test_core.c
+++ b/crypto/self_test_core.c
@@ -46,6 +46,7 @@ static void self_test_set_callback_free(void *stcb)
 }
 
 static const OSSL_LIB_CTX_METHOD self_test_set_callback_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     self_test_set_callback_new,
     self_test_set_callback_free,
 };
diff --git a/crypto/store/store_meth.c b/crypto/store/store_meth.c
index a2ab341fe9..7bf0a329ce 100644
--- a/crypto/store/store_meth.c
+++ b/crypto/store/store_meth.c
@@ -81,6 +81,7 @@ static void *loader_store_new(OSSL_LIB_CTX *ctx)
 
 
 static const OSSL_LIB_CTX_METHOD loader_store_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     loader_store_new,
     loader_store_free,
 };
diff --git a/include/internal/cryptlib.h b/include/internal/cryptlib.h
index d583153b89..94d93abda2 100644
--- a/include/internal/cryptlib.h
+++ b/include/internal/cryptlib.h
@@ -120,6 +120,7 @@ size_t OPENSSL_instrument_bus2(unsigned int *, size_t, size_t);
 struct ex_callback_st {
     long argl;                  /* Arbitrary long */
     void *argp;                 /* Arbitrary void * */
+    int priority;               /* Priority ordering for freeing */
     CRYPTO_EX_new *new_func;
     CRYPTO_EX_free *free_func;
     CRYPTO_EX_dup *dup_func;
@@ -166,7 +167,10 @@ typedef struct ossl_ex_data_global_st {
 # define OSSL_LIB_CTX_BIO_CORE_INDEX                17
 # define OSSL_LIB_CTX_MAX_INDEXES                   18
 
+# define OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY       0
+# define OSSL_LIB_CTX_METHOD_HIGH_PRIORITY          1
 typedef struct ossl_lib_ctx_method {
+    int priority;
     void *(*new_func)(OSSL_LIB_CTX *ctx);
     void (*free_func)(void *);
 } OSSL_LIB_CTX_METHOD;
@@ -196,7 +200,8 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index,
                                     long argl, void *argp,
                                     CRYPTO_EX_new *new_func,
                                     CRYPTO_EX_dup *dup_func,
-                                    CRYPTO_EX_free *free_func);
+                                    CRYPTO_EX_free *free_func,
+                                    int priority);
 int ossl_crypto_free_ex_index_ex(OSSL_LIB_CTX *ctx, int class_index, int idx);
 
 /* Function for simple binary search */
diff --git a/providers/fips/fipsprov.c b/providers/fips/fipsprov.c
index 841c80bab7..7998d55d9a 100644
--- a/providers/fips/fipsprov.c
+++ b/providers/fips/fipsprov.c
@@ -96,6 +96,7 @@ static void fips_prov_ossl_ctx_free(void *fgbl)
 }
 
 static const OSSL_LIB_CTX_METHOD fips_prov_ossl_ctx_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     fips_prov_ossl_ctx_new,
     fips_prov_ossl_ctx_free,
 };
diff --git a/providers/implementations/rands/crngt.c b/providers/implementations/rands/crngt.c
index f1b31df101..87902c995c 100644
--- a/providers/implementations/rands/crngt.c
+++ b/providers/implementations/rands/crngt.c
@@ -83,6 +83,7 @@ static void *rand_crng_ossl_ctx_new(OSSL_LIB_CTX *ctx)
 }
 
 static const OSSL_LIB_CTX_METHOD rand_crng_ossl_ctx_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     rand_crng_ossl_ctx_new,
     rand_crng_ossl_ctx_free,
 };
diff --git a/providers/implementations/rands/drbg.c b/providers/implementations/rands/drbg.c
index 461d641273..81343fbd52 100644
--- a/providers/implementations/rands/drbg.c
+++ b/providers/implementations/rands/drbg.c
@@ -303,6 +303,7 @@ static void prov_drbg_nonce_ossl_ctx_free(void *vdngbl)
 }
 
 static const OSSL_LIB_CTX_METHOD drbg_nonce_ossl_ctx_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     prov_drbg_nonce_ossl_ctx_new,
     prov_drbg_nonce_ossl_ctx_free,
 };
diff --git a/test/context_internal_test.c b/test/context_internal_test.c
index 46afd9f521..4c02f601cc 100644
--- a/test/context_internal_test.c
+++ b/test/context_internal_test.c
@@ -39,6 +39,7 @@ static void foo_free(void *ptr)
     OPENSSL_free(ptr);
 }
 static const OSSL_LIB_CTX_METHOD foo_method = {
+    OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY,
     foo_new,
     foo_free
 };