diff --git a/crypto/passphrase.c b/crypto/passphrase.c
index fb8ea1deb1..d61e249440 100644
--- a/crypto/passphrase.c
+++ b/crypto/passphrase.c
@@ -296,7 +296,8 @@ int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len,
     return ret;
 }
 
-int ossl_pw_pem_password(char *buf, int size, int rwflag, void *userdata)
+static int ossl_pw_get_password(char *buf, int size, int rwflag,
+                                void *userdata, const char *info)
 {
     size_t password_len = 0;
     OSSL_PARAM params[] = {
@@ -304,13 +305,23 @@ int ossl_pw_pem_password(char *buf, int size, int rwflag, void *userdata)
         OSSL_PARAM_END
     };
 
-    params[0].data = "PEM";
+    params[0].data = (void *)info;
     if (ossl_pw_get_passphrase(buf, (size_t)size, &password_len, params,
                                rwflag, userdata))
         return (int)password_len;
     return -1;
 }
 
+int ossl_pw_pem_password(char *buf, int size, int rwflag, void *userdata)
+{
+    return ossl_pw_get_password(buf, size, rwflag, userdata, "PEM");
+}
+
+int ossl_pw_pvk_password(char *buf, int size, int rwflag, void *userdata)
+{
+    return ossl_pw_get_password(buf, size, rwflag, userdata, "PVK");
+}
+
 int ossl_pw_passphrase_callback_enc(char *pass, size_t pass_size,
                                     size_t *pass_len,
                                     const OSSL_PARAM params[], void *arg)
diff --git a/include/internal/passphrase.h b/include/internal/passphrase.h
index ee0be9b128..54d997b0d9 100644
--- a/include/internal/passphrase.h
+++ b/include/internal/passphrase.h
@@ -114,6 +114,7 @@ int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len,
  */
 
 pem_password_cb ossl_pw_pem_password;
+pem_password_cb ossl_pw_pvk_password;
 /* One callback for encoding (verification prompt) and one for decoding */
 OSSL_PASSPHRASE_CALLBACK ossl_pw_passphrase_callback_enc;
 OSSL_PASSPHRASE_CALLBACK ossl_pw_passphrase_callback_dec;
diff --git a/providers/implementations/encode_decode/encode_key2ms.c b/providers/implementations/encode_decode/encode_key2ms.c
index 3933a0d420..81528fefb6 100644
--- a/providers/implementations/encode_decode/encode_key2ms.c
+++ b/providers/implementations/encode_decode/encode_key2ms.c
@@ -47,8 +47,7 @@ static int write_msblob(struct key2ms_ctx_st *ctx, OSSL_CORE_BIO *cout,
 }
 
 static int write_pvk(struct key2ms_ctx_st *ctx, OSSL_CORE_BIO *cout,
-                     EVP_PKEY *pkey,
-                     OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg)
+                     EVP_PKEY *pkey)
 {
     BIO *out = NULL;
     int ret = 0;
@@ -56,7 +55,7 @@ static int write_pvk(struct key2ms_ctx_st *ctx, OSSL_CORE_BIO *cout,
 
     out = ossl_bio_new_from_core_bio(ctx->provctx, cout);
     ret = i2b_PVK_bio_ex(out, pkey, ctx->pvk_encr_level,
-                         ossl_pw_pem_password, &ctx->pwdata, libctx, NULL);
+                         ossl_pw_pvk_password, &ctx->pwdata, libctx, NULL);
     BIO_free(out);
 
     return ret;
@@ -81,6 +80,7 @@ static void key2ms_freectx(void *vctx)
 {
     struct key2ms_ctx_st *ctx = vctx;
 
+    ossl_pw_clear_passphrase_data(&ctx->pwdata);
     OPENSSL_free(ctx);
 }
 
@@ -154,8 +154,10 @@ static int key2pvk_encode(void *vctx, const void *key, int selection,
     if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) == 0)
         return 0;                /* Error */
 
-    if ((pkey = EVP_PKEY_new()) != NULL && set1_key(pkey, key))
-        ok = write_pvk(ctx, cout, pkey, pw_cb, pw_cbarg);
+    if ((pkey = EVP_PKEY_new()) != NULL && set1_key(pkey, key)
+        && (pw_cb == NULL
+            || ossl_pw_set_ossl_passphrase_cb(&ctx->pwdata, pw_cb, pw_cbarg)))
+        ok = write_pvk(ctx, cout, pkey);
     EVP_PKEY_free(pkey);
     return ok;
 }