diff --git a/crypto/build.info b/crypto/build.info
index 07e3dd526f..9e10145d3c 100644
--- a/crypto/build.info
+++ b/crypto/build.info
@@ -71,7 +71,7 @@ $UTIL_COMMON=\
         cryptlib.c params.c params_from_text.c bsearch.c ex_data.c o_str.c \
         ctype.c threads_pthread.c threads_win.c threads_none.c initthread.c \
         context.c sparse_array.c asn1_dsa.c packet.c param_build.c $CPUIDASM \
-        param_build_set.c der_writer.c
+        param_build_set.c der_writer.c passphrase.c
 $UTIL_DEFINE=$CPUIDDEF
 
 SOURCE[../libcrypto]=$UTIL_COMMON \
diff --git a/crypto/encode_decode/build.info b/crypto/encode_decode/build.info
index 4686c4a19d..e2cd846673 100644
--- a/crypto/encode_decode/build.info
+++ b/crypto/encode_decode/build.info
@@ -1,8 +1,5 @@
-SOURCE[../../libcrypto]=endecode_pass.c
-
 SOURCE[../../libcrypto]=encoder_meth.c encoder_lib.c encoder_pkey.c
-SOURCE[../../libcrypto]=decoder_meth.c decoder_lib.c \
-        decoder_pkey.c
+SOURCE[../../libcrypto]=decoder_meth.c decoder_lib.c decoder_pkey.c
 
 SOURCE[../../libcrypto]=encoder_err.c
 SOURCE[../../libcrypto]=decoder_err.c
diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index cd6215e309..5d18ef1eff 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -11,6 +11,7 @@
 #include <openssl/bio.h>
 #include <openssl/params.h>
 #include <openssl/provider.h>
+#include "internal/passphrase.h"
 #include "encoder_local.h"
 #include "e_os.h"
 
@@ -35,13 +36,14 @@ int OSSL_DECODER_from_bio(OSSL_DECODER_CTX *ctx, BIO *in)
     data.ctx = ctx;
     data.bio = in;
 
+    /* Enable passphrase caching */
+    (void)ossl_pw_enable_passphrase_caching(&ctx->pwdata);
+
     ok = decoder_process(NULL, &data);
 
     /* Clear any internally cached passphrase */
-    if (!ctx->flag_user_passphrase) {
-        OSSL_DECODER_CTX_set_passphrase(ctx, NULL, 0);
-        ctx->flag_user_passphrase = 0;
-    }
+    (void)ossl_pw_clear_passphrase_cache(&ctx->pwdata);
+
     return ok;
 }
 
@@ -472,7 +474,8 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
         new_data.current_deser_inst_index = i;
         ok = new_deser->decode(new_deser_inst->deserctx, (OSSL_CORE_BIO *)bio,
                                decoder_process, &new_data,
-                               ctx->passphrase_cb, new_data.ctx);
+                               ossl_pw_passphrase_callback_dec,
+                               &new_data.ctx->pwdata);
         if (ok)
             break;
     }
diff --git a/crypto/encode_decode/decoder_meth.c b/crypto/encode_decode/decoder_meth.c
index 2259c6348a..409bb1aa54 100644
--- a/crypto/encode_decode/decoder_meth.c
+++ b/crypto/encode_decode/decoder_meth.c
@@ -492,7 +492,6 @@ OSSL_DECODER_CTX *OSSL_DECODER_CTX_new(void)
         return NULL;
     }
 
-    ctx->passphrase_cb = ossl_decoder_passphrase_in_cb;
     return ctx;
 }
 
@@ -545,8 +544,7 @@ void OSSL_DECODER_CTX_free(OSSL_DECODER_CTX *ctx)
             ctx->cleanup(ctx->construct_data);
         sk_OSSL_DECODER_INSTANCE_pop_free(ctx->decoder_insts,
                                           OSSL_DECODER_INSTANCE_free);
-        OSSL_DECODER_CTX_set_passphrase_ui(ctx, NULL, NULL);
-        OSSL_DECODER_CTX_set_passphrase(ctx, NULL, 0);
+        ossl_pw_clear_passphrase_data(&ctx->pwdata);
         OPENSSL_free(ctx);
     }
 }
diff --git a/crypto/encode_decode/decoder_pkey.c b/crypto/encode_decode/decoder_pkey.c
index db75041d17..92c0d5a6ea 100644
--- a/crypto/encode_decode/decoder_pkey.c
+++ b/crypto/encode_decode/decoder_pkey.c
@@ -20,79 +20,20 @@ int OSSL_DECODER_CTX_set_passphrase(OSSL_DECODER_CTX *ctx,
                                     const unsigned char *kstr,
                                     size_t klen)
 {
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    OPENSSL_clear_free(ctx->cached_passphrase, ctx->cached_passphrase_len);
-    ctx->cached_passphrase = NULL;
-    ctx->cached_passphrase_len = 0;
-    if (kstr != NULL) {
-        if (klen == 0) {
-            ctx->cached_passphrase = OPENSSL_zalloc(1);
-            ctx->cached_passphrase_len = 0;
-        } else {
-            ctx->cached_passphrase = OPENSSL_memdup(kstr, klen);
-            ctx->cached_passphrase_len = klen;
-        }
-        if (ctx->cached_passphrase == NULL) {
-            ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
-            return 0;
-        }
-    }
-    ctx->flag_user_passphrase = 1;
-    return 1;
-}
-
-static void decoder_ctx_reset_passphrase_ui(OSSL_DECODER_CTX *ctx)
-{
-    UI_destroy_method(ctx->allocated_ui_method);
-    ctx->allocated_ui_method = NULL;
-    ctx->ui_method = NULL;
-    ctx->ui_data = NULL;
+    return ossl_pw_set_passphrase(&ctx->pwdata, kstr, klen);
 }
 
 int OSSL_DECODER_CTX_set_passphrase_ui(OSSL_DECODER_CTX *ctx,
                                        const UI_METHOD *ui_method,
                                        void *ui_data)
 {
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    decoder_ctx_reset_passphrase_ui(ctx);
-    ctx->ui_method = ui_method;
-    ctx->ui_data = ui_data;
-    return 1;
+    return ossl_pw_set_ui_method(&ctx->pwdata, ui_method, ui_data);
 }
 
 int OSSL_DECODER_CTX_set_pem_password_cb(OSSL_DECODER_CTX *ctx,
                                          pem_password_cb *cb, void *cbarg)
 {
-    UI_METHOD *ui_method = NULL;
-
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    /*
-     * If |cb| is NULL, it means the caller wants to reset previous
-     * password callback info.  Otherwise, we only set the new data
-     * if a new UI_METHOD could be created for this sort of callback.
-     */
-    if (cb == NULL
-        || (ui_method = UI_UTIL_wrap_read_pem_callback(cb, 0)) != NULL) {
-        decoder_ctx_reset_passphrase_ui(ctx);
-        ctx->ui_method = ctx->allocated_ui_method = ui_method;
-        ctx->ui_data = cbarg;
-        ctx->passphrase_cb = ossl_decoder_passphrase_in_cb;
-        return 1;
-    }
-
-    return 0;
+    return ossl_pw_set_pem_password_cb(&ctx->pwdata, cb, cbarg);
 }
 
 /*
diff --git a/crypto/encode_decode/encoder_local.h b/crypto/encode_decode/encoder_local.h
index 34931d4e43..749b2688e4 100644
--- a/crypto/encode_decode/encoder_local.h
+++ b/crypto/encode_decode/encoder_local.h
@@ -13,6 +13,7 @@
 #include <openssl/encoder.h>
 #include <openssl/decoder.h>
 #include "internal/cryptlib.h"
+#include "internal/passphrase.h"
 #include "internal/refcount.h"
 
 struct ossl_serdes_base_st {
@@ -64,13 +65,7 @@ struct ossl_encoder_ctx_st {
     int (*do_output)(OSSL_ENCODER_CTX *ctx, BIO *out);
 
     /* For any function that needs a passphrase reader */
-    const UI_METHOD *ui_method;
-    void *ui_data;
-    /*
-     * if caller used OSSL_ENCODER_CTX_set_passphrase_cb(), we need
-     * intermediary storage.
-     */
-    UI_METHOD *allocated_ui_method;
+    struct ossl_passphrase_data_st pwdata;
 };
 
 struct ossl_decoder_instance_st {
@@ -103,38 +98,5 @@ struct ossl_decoder_ctx_st {
     void *construct_data;
 
     /* For any function that needs a passphrase reader */
-    OSSL_PASSPHRASE_CALLBACK *passphrase_cb;
-    const UI_METHOD *ui_method;
-    void *ui_data;
-    /*
-     * if caller used OSSL_ENCODER_CTX_set_pem_password_cb(), we need
-     * intermediary storage.
-     */
-    UI_METHOD *allocated_ui_method;
-    /*
-     * Because the same input may pass through more than one decoder,
-     * we cache any passphrase passed to us.  The desrializing processor
-     * must clear this at the end of a run.
-     */
-    unsigned char *cached_passphrase;
-    size_t cached_passphrase_len;
-
-    /*
-     * Flag section.  Keep these together
-     */
-
-    /*
-     * The passphrase was passed to us by the user.  In that case, it
-     * should only be freed when freeing this context.
-     */
-    unsigned int flag_user_passphrase:1;
+    struct ossl_passphrase_data_st pwdata;
 };
-
-/* Passphrase callbacks, found in serdes_pass.c */
-
-/*
- * Encoders typically want to get an outgoing passphrase, while
- * decoders typically want to get en incoming passphrase.
- */
-OSSL_PASSPHRASE_CALLBACK ossl_encoder_passphrase_out_cb;
-OSSL_PASSPHRASE_CALLBACK ossl_decoder_passphrase_in_cb;
diff --git a/crypto/encode_decode/encoder_meth.c b/crypto/encode_decode/encoder_meth.c
index ef3c24433e..81bc5c47b9 100644
--- a/crypto/encode_decode/encoder_meth.c
+++ b/crypto/encode_decode/encoder_meth.c
@@ -517,7 +517,7 @@ void OSSL_ENCODER_CTX_free(OSSL_ENCODER_CTX *ctx)
         if (ctx->encoder != NULL && ctx->encoder->freectx != NULL)
             ctx->encoder->freectx(ctx->serctx);
         OSSL_ENCODER_free(ctx->encoder);
-        UI_destroy_method(ctx->allocated_ui_method);
+        ossl_pw_clear_passphrase_data(&ctx->pwdata);
         OPENSSL_free(ctx);
     }
 }
diff --git a/crypto/encode_decode/encoder_pkey.c b/crypto/encode_decode/encoder_pkey.c
index dc776c023e..6664f589b4 100644
--- a/crypto/encode_decode/encoder_pkey.c
+++ b/crypto/encode_decode/encoder_pkey.c
@@ -48,45 +48,17 @@ int OSSL_ENCODER_CTX_set_passphrase(OSSL_ENCODER_CTX *ctx,
     return OSSL_ENCODER_CTX_set_params(ctx, params);
 }
 
-static void encoder_ctx_reset_passphrase_ui(OSSL_ENCODER_CTX *ctx)
-{
-    UI_destroy_method(ctx->allocated_ui_method);
-    ctx->allocated_ui_method = NULL;
-    ctx->ui_method = NULL;
-    ctx->ui_data = NULL;
-}
-
 int OSSL_ENCODER_CTX_set_passphrase_ui(OSSL_ENCODER_CTX *ctx,
                                        const UI_METHOD *ui_method,
                                        void *ui_data)
 {
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    encoder_ctx_reset_passphrase_ui(ctx);
-    ctx->ui_method = ui_method;
-    ctx->ui_data = ui_data;
-    return 1;
+    return ossl_pw_set_ui_method(&ctx->pwdata, ui_method, ui_data);
 }
 
 int OSSL_ENCODER_CTX_set_passphrase_cb(OSSL_ENCODER_CTX *ctx,
                                        pem_password_cb *cb, void *cbarg)
 {
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    encoder_ctx_reset_passphrase_ui(ctx);
-    if (cb == NULL)
-        return 1;
-    ctx->ui_method =
-        ctx->allocated_ui_method = UI_UTIL_wrap_read_pem_callback(cb, 1);
-    ctx->ui_data = cbarg;
-
-    return ctx->ui_method != NULL;
+    return ossl_pw_set_pem_password_cb(&ctx->pwdata, cb, cbarg);
 }
 
 /*
@@ -125,7 +97,8 @@ static int encoder_write_cb(const OSSL_PARAM params[], void *arg)
     BIO *out = write_data->out;
 
     return ctx->encoder->encode_data(ctx->serctx, params, (OSSL_CORE_BIO *)out,
-                                     ossl_encoder_passphrase_out_cb, ctx);
+                                     ossl_pw_passphrase_callback_enc,
+                                     &ctx->pwdata);
 }
 
 /*
@@ -164,7 +137,8 @@ static int encoder_EVP_PKEY_to_bio(OSSL_ENCODER_CTX *ctx, BIO *out)
 
     return ctx->encoder->encode_object(ctx->serctx, keydata,
                                        (OSSL_CORE_BIO *)out,
-                                       ossl_encoder_passphrase_out_cb, ctx);
+                                       ossl_pw_passphrase_callback_enc,
+                                       &ctx->pwdata);
 }
 
 /*
diff --git a/crypto/encode_decode/endecode_pass.c b/crypto/encode_decode/endecode_pass.c
deleted file mode 100644
index 113f5eeb84..0000000000
--- a/crypto/encode_decode/endecode_pass.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#include <openssl/err.h>
-#include <openssl/ui.h>
-#include <openssl/core_names.h>
-#include "internal/cryptlib.h"
-#include "encoder_local.h"
-
-/* Passphrase callbacks for any who need it */
-
-/*
- * First, define the generic passphrase function that supports both
- * outgoing (with passphrase verify) and incoming (without passphrase
- * verify) passphrase reading.
- */
-static int do_passphrase(char *pass, size_t pass_size, size_t *pass_len,
-                         const OSSL_PARAM params[], void *arg, int verify,
-                         const UI_METHOD *ui_method, void *ui_data, int errlib)
-{
-    const OSSL_PARAM *p;
-    const char *prompt_info = NULL;
-    char *prompt = NULL, *vpass = NULL;
-    int prompt_idx = -1, verify_idx = -1;
-    UI *ui = NULL;
-    int ret = 0;
-
-    if (!ossl_assert(pass != NULL && pass_size != 0 && pass_len != NULL)) {
-        ERR_raise(errlib, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    if ((p = OSSL_PARAM_locate_const(params,
-                                     OSSL_PASSPHRASE_PARAM_INFO)) != NULL) {
-        if (p->data_type != OSSL_PARAM_UTF8_STRING)
-            return 0;
-        prompt_info = p->data;
-    }
-
-    if ((ui = UI_new()) == NULL) {
-        ERR_raise(errlib, ERR_R_MALLOC_FAILURE);
-        return 0;
-    }
-
-    if (ui_method != NULL) {
-        UI_set_method(ui, ui_method);
-        if (ui_data != NULL)
-            UI_add_user_data(ui, ui_data);
-    }
-
-    /* Get an application constructed prompt */
-    prompt = UI_construct_prompt(ui, "pass phrase", prompt_info);
-    if (prompt == NULL) {
-        ERR_raise(errlib, ERR_R_MALLOC_FAILURE);
-        goto end;
-    }
-
-    prompt_idx = UI_add_input_string(ui, prompt,
-                                     UI_INPUT_FLAG_DEFAULT_PWD,
-                                     pass, 0, pass_size - 1) - 1;
-    if (prompt_idx < 0) {
-        ERR_raise(errlib, ERR_R_UI_LIB);
-        goto end;
-    }
-
-    if (verify) {
-        /* Get a buffer for verification prompt */
-        vpass = OPENSSL_zalloc(pass_size);
-        if (vpass == NULL) {
-            ERR_raise(errlib, ERR_R_MALLOC_FAILURE);
-            goto end;
-        }
-        verify_idx = UI_add_verify_string(ui, prompt,
-                                          UI_INPUT_FLAG_DEFAULT_PWD,
-                                          vpass, 0, pass_size - 1,
-                                          pass) - 1;
-        if (verify_idx < 0) {
-            ERR_raise(errlib, ERR_R_UI_LIB);
-            goto end;
-        }
-    }
-
-    switch (UI_process(ui)) {
-    case -2:
-        ERR_raise(errlib, ERR_R_INTERRUPTED_OR_CANCELLED);
-        break;
-    case -1:
-        ERR_raise(errlib, ERR_R_UI_LIB);
-        break;
-    default:
-        *pass_len = (size_t)UI_get_result_length(ui, prompt_idx);
-        ret = 1;
-        break;
-    }
-
- end:
-    OPENSSL_free(vpass);
-    OPENSSL_free(prompt);
-    UI_free(ui);
-    return ret;
-}
-
-/*
- * Encoders typically want to get an outgoing passphrase, while
- * decoders typically want to get en incoming passphrase.
- */
-int ossl_encoder_passphrase_out_cb(char *pass, size_t pass_size,
-                                   size_t *pass_len,
-                                   const OSSL_PARAM params[], void *arg)
-{
-    OSSL_ENCODER_CTX *ctx = arg;
-
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    return do_passphrase(pass, pass_size, pass_len, params, arg, 1,
-                         ctx->ui_method, ctx->ui_data,
-                         ERR_LIB_OSSL_ENCODER);
-}
-
-int ossl_decoder_passphrase_in_cb(char *pass, size_t pass_size,
-                                  size_t *pass_len,
-                                  const OSSL_PARAM params[], void *arg)
-{
-    OSSL_DECODER_CTX *ctx = arg;
-
-    if (!ossl_assert(ctx != NULL)) {
-        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
-        return 0;
-    }
-
-    if (ctx->cached_passphrase != NULL) {
-        size_t len = ctx->cached_passphrase_len;
-
-        if (len > pass_size)
-            len = pass_size;
-        memcpy(pass, ctx->cached_passphrase, len);
-        *pass_len = len;
-        return 1;
-    } else {
-        if ((ctx->cached_passphrase = OPENSSL_zalloc(pass_size)) == NULL) {
-            ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
-            return 0;
-        }
-    }
-    if (do_passphrase(pass, pass_size, pass_len, params, arg, 0,
-                      ctx->ui_method, ctx->ui_data,
-                      ERR_LIB_OSSL_DECODER)) {
-        memcpy(ctx->cached_passphrase, pass, *pass_len);
-        ctx->cached_passphrase_len = *pass_len;
-        return 1;
-    }
-    return 0;
-}
diff --git a/crypto/passphrase.c b/crypto/passphrase.c
new file mode 100644
index 0000000000..ac352697db
--- /dev/null
+++ b/crypto/passphrase.c
@@ -0,0 +1,324 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <openssl/err.h>
+#include <openssl/ui.h>
+#include <openssl/core_names.h>
+#include "internal/cryptlib.h"
+#include "internal/passphrase.h"
+
+void ossl_pw_clear_passphrase_data(struct ossl_passphrase_data_st *data)
+{
+    if (data != NULL) {
+        if (data->type == is_expl_passphrase)
+            OPENSSL_clear_free(data->_.expl_passphrase.passphrase_copy,
+                               data->_.expl_passphrase.passphrase_len);
+        ossl_pw_clear_passphrase_cache(data);
+        memset(data, 0, sizeof(*data));
+    }
+}
+
+void ossl_pw_clear_passphrase_cache(struct ossl_passphrase_data_st *data)
+{
+    OPENSSL_clear_free(data->cached_passphrase, data->cached_passphrase_len);
+    data->cached_passphrase = NULL;
+}
+
+int ossl_pw_set_passphrase(struct ossl_passphrase_data_st *data,
+                           const unsigned char *passphrase,
+                           size_t passphrase_len)
+{
+    if (!ossl_assert(data != NULL && passphrase != NULL)) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    ossl_pw_clear_passphrase_data(data);
+    data->type = is_expl_passphrase;
+    data->_.expl_passphrase.passphrase_copy =
+        OPENSSL_memdup(passphrase, passphrase_len);
+    if (data->_.expl_passphrase.passphrase_copy == NULL) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
+    data->_.expl_passphrase.passphrase_len = passphrase_len;
+    return 1;
+}
+
+int ossl_pw_set_pem_password_cb(struct ossl_passphrase_data_st *data,
+                                pem_password_cb *cb, void *cbarg)
+{
+    if (!ossl_assert(data != NULL && cb != NULL)) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    ossl_pw_clear_passphrase_data(data);
+    data->type = is_pem_password;
+    data->_.pem_password.password_cb = cb;
+    data->_.pem_password.password_cbarg = cbarg;
+    return 1;
+}
+
+int ossl_pw_set_ossl_passphrase_cb(struct ossl_passphrase_data_st *data,
+                                   OSSL_PASSPHRASE_CALLBACK *cb, void *cbarg)
+{
+    if (!ossl_assert(data != NULL && cb != NULL)) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    ossl_pw_clear_passphrase_data(data);
+    data->type = is_ossl_passphrase;
+    data->_.ossl_passphrase.passphrase_cb = cb;
+    data->_.ossl_passphrase.passphrase_cbarg = cbarg;
+    return 1;
+}
+
+int ossl_pw_set_ui_method(struct ossl_passphrase_data_st *data,
+                          const UI_METHOD *ui_method, void *ui_data)
+{
+    if (!ossl_assert(data != NULL && ui_method != NULL)) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    ossl_pw_clear_passphrase_data(data);
+    data->type = is_ui_method;
+    data->_.ui_method.ui_method = ui_method;
+    data->_.ui_method.ui_method_data = ui_data;
+    return 1;
+}
+
+int ossl_pw_enable_passphrase_caching(struct ossl_passphrase_data_st *data)
+{
+    data->flag_cache_passphrase = 1;
+    return 1;
+}
+
+int ossl_pw_disable_passphrase_caching(struct ossl_passphrase_data_st *data)
+{
+    data->flag_cache_passphrase = 0;
+    return 1;
+}
+
+
+/*-
+ * UI_METHOD processor.  It differs from UI_UTIL_read_pw() like this:
+ *
+ * 1.  It constructs a prompt on its own, based on |prompt_info|.
+ * 2.  It allocates a buffer for verification on its own.
+ * 3.  It raises errors.
+ * 4.  It reports back the length of the prompted pass phrase.
+ */
+static int do_ui_passphrase(char *pass, size_t pass_size, size_t *pass_len,
+                            const char *prompt_info, int verify,
+                            const UI_METHOD *ui_method, void *ui_data)
+{
+    char *prompt = NULL, *vpass = NULL;
+    int prompt_idx = -1, verify_idx = -1;
+    UI *ui = NULL;
+    int ret = 0;
+
+    if (!ossl_assert(pass != NULL && pass_size != 0 && pass_len != NULL)) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    if ((ui = UI_new()) == NULL) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
+
+    if (ui_method != NULL) {
+        UI_set_method(ui, ui_method);
+        if (ui_data != NULL)
+            UI_add_user_data(ui, ui_data);
+    }
+
+    /* Get an application constructed prompt */
+    prompt = UI_construct_prompt(ui, "pass phrase", prompt_info);
+    if (prompt == NULL) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+        goto end;
+    }
+
+    prompt_idx = UI_add_input_string(ui, prompt,
+                                     UI_INPUT_FLAG_DEFAULT_PWD,
+                                     pass, 0, pass_size - 1) - 1;
+    if (prompt_idx < 0) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB);
+        goto end;
+    }
+
+    if (verify) {
+        /* Get a buffer for verification prompt */
+        vpass = OPENSSL_zalloc(pass_size);
+        if (vpass == NULL) {
+            ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+            goto end;
+        }
+        verify_idx = UI_add_verify_string(ui, prompt,
+                                          UI_INPUT_FLAG_DEFAULT_PWD,
+                                          vpass, 0, pass_size - 1,
+                                          pass) - 1;
+        if (verify_idx < 0) {
+            ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB);
+            goto end;
+        }
+    }
+
+    switch (UI_process(ui)) {
+    case -2:
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERRUPTED_OR_CANCELLED);
+        break;
+    case -1:
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB);
+        break;
+    default:
+        *pass_len = (size_t)UI_get_result_length(ui, prompt_idx);
+        ret = 1;
+        break;
+    }
+
+ end:
+    OPENSSL_free(vpass);
+    OPENSSL_free(prompt);
+    UI_free(ui);
+    return ret;
+}
+
+/* Central pw prompting dispatcher */
+int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len,
+                           const OSSL_PARAM params[], int verify,
+                           struct ossl_passphrase_data_st *data)
+{
+    const char *source = NULL;
+    size_t source_len = 0;
+    const char *prompt_info = NULL;
+    const UI_METHOD *ui_method = NULL;
+    UI_METHOD *allocated_ui_method = NULL;
+    void *ui_data = NULL;
+    const OSSL_PARAM *p = NULL;
+    int ret;
+
+    /* Handle explicit and cached passphrases */
+
+    if (data->type == is_expl_passphrase) {
+        source = data->_.expl_passphrase.passphrase_copy;
+        source_len = data->_.expl_passphrase.passphrase_len;
+    } else if (data->flag_cache_passphrase && data->cached_passphrase != NULL) {
+        source = data->cached_passphrase;
+        source_len = data->cached_passphrase_len;
+    }
+
+    if (source != NULL) {
+        if (source_len > pass_size)
+            source_len = pass_size;
+        memcpy(pass, source, source_len);
+        *pass_len = source_len;
+        return 1;
+    }
+
+    /* Handle the is_ossl_passphrase case...  that's pretty direct */
+
+    if (data->type == is_ossl_passphrase) {
+        OSSL_PASSPHRASE_CALLBACK *cb = data->_.ossl_passphrase.passphrase_cb;
+        void *cbarg = data->_.ossl_passphrase.passphrase_cbarg;
+
+        ret = cb(pass, pass_size, pass_len, params, cbarg);
+        goto do_cache;
+    }
+
+    /* Handle the is_pem_password and is_ui_method cases */
+
+    if ((p = OSSL_PARAM_locate_const(params,
+                                     OSSL_PASSPHRASE_PARAM_INFO)) != NULL) {
+        if (p->data_type != OSSL_PARAM_UTF8_STRING) {
+            ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT,
+                           "Prompt info data type incorrect");
+            return 0;
+        }
+        prompt_info = p->data;
+    }
+
+    if (data->type == is_pem_password) {
+        /* We use a UI wrapper for PEM */
+        pem_password_cb *cb = data->_.pem_password.password_cb;
+
+        ui_method = allocated_ui_method =
+            UI_UTIL_wrap_read_pem_callback(cb, verify);
+        ui_data = data->_.pem_password.password_cbarg;
+
+        if (ui_method == NULL) {
+            ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+            return 0;
+        }
+    } else if (data->type == is_ui_method) {
+        ui_method = data->_.ui_method.ui_method;
+        ui_data = data->_.ui_method.ui_method_data;
+    }
+
+    if (ui_method == NULL) {
+        ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR);
+        return 0;
+    }
+
+    ret = do_ui_passphrase(pass, pass_size, pass_len, prompt_info, verify,
+                           ui_method, ui_data);
+
+    UI_destroy_method(allocated_ui_method);
+
+ do_cache:
+    if (ret && data->flag_cache_passphrase) {
+        if (*pass_len > data->cached_passphrase_len) {
+            void *new_cache =
+                OPENSSL_clear_realloc(data->cached_passphrase,
+                                      data->cached_passphrase_len,
+                                      *pass_len + 1);
+
+            if (new_cache == NULL) {
+                OPENSSL_cleanse(pass, *pass_len);
+                ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+                return 0;
+            }
+            data->cached_passphrase = new_cache;
+        }
+        memcpy(data->cached_passphrase, pass, *pass_len);
+        data->cached_passphrase[*pass_len] = '\0';
+        data->cached_passphrase_len = *pass_len;
+    }
+
+    return ret;
+}
+
+int ossl_pw_pem_password(char *buf, int size, int rwflag, void *userdata)
+{
+    size_t password_len = 0;
+    OSSL_PARAM params[] = {
+        OSSL_PARAM_utf8_string(OSSL_PASSPHRASE_PARAM_INFO, NULL, 0),
+        OSSL_PARAM_END
+    };
+
+    params[0].data = "PEM";
+    if (ossl_pw_get_passphrase(buf, (size_t)size, &password_len, params,
+                               rwflag, userdata))
+        return (int)password_len;
+    return -1;
+}
+
+int ossl_pw_passphrase_callback_enc(char *pass, size_t pass_size,
+                                    size_t *pass_len,
+                                    const OSSL_PARAM params[], void *arg)
+{
+    return ossl_pw_get_passphrase(pass, pass_size, pass_len, params, 1, arg);
+}
+
+int ossl_pw_passphrase_callback_dec(char *pass, size_t pass_size,
+                                    size_t *pass_len,
+                                    const OSSL_PARAM params[], void *arg)
+{
+    return ossl_pw_get_passphrase(pass, pass_size, pass_len, params, 0, arg);
+}
diff --git a/include/internal/passphrase.h b/include/internal/passphrase.h
new file mode 100644
index 0000000000..9077907d52
--- /dev/null
+++ b/include/internal/passphrase.h
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OSSL_INTERNAL_PASSPHRASE_H
+# define OSSL_INTERNAL_PASSPHRASE_H
+
+/*
+ * This is a passphrase reader bridge with bells and whistles.
+ *
+ * On one hand, an API may wish to offer all sorts of passphrase callback
+ * possibilities to users, or may have to do so for historical reasons.
+ * On the other hand, that same API may have demands from other interfaces,
+ * notably from the libcrypto <-> provider interface, which uses
+ * OSSL_PASSPHRASE_CALLBACK consistently.
+ *
+ * The structure and functions below are the fundaments for bridging one
+ * passphrase callback form to another.
+ *
+ * In addition, extra features are included (this may be a growing list):
+ *
+ * -   password caching.  This is to be used by APIs where it's likely
+ *     that the same passphrase may be asked for more than once, but the
+ *     user shouldn't get prompted more than once.  For example, this is
+ *     useful for OSSL_DECODER, which may have to use a passphrase while
+ *     trying to find out what input it has.
+ */
+
+/*
+ * Structure to hold whatever the calling user may specify.  This structure
+ * is intended to be integrated into API specific structures or to be used
+ * as a local on-stack variable type.  Therefore, no functions to allocate
+ * or freed it on the heap is offered.
+ */
+struct ossl_passphrase_data_st {
+    enum {
+        is_expl_passphrase = 1, /* Explicit passphrase given by user */
+        is_pem_password,        /* pem_password_cb given by user */
+        is_ossl_passphrase,     /* OSSL_PASSPHRASE_CALLBACK given by user */
+        is_ui_method            /* UI_METHOD given by user */
+    } type;
+    union {
+        struct {
+            char *passphrase_copy;
+            size_t passphrase_len;
+        } expl_passphrase;
+
+        struct {
+            pem_password_cb *password_cb;
+            void *password_cbarg;
+        } pem_password;
+
+        struct {
+            OSSL_PASSPHRASE_CALLBACK *passphrase_cb;
+            void *passphrase_cbarg;
+        } ossl_passphrase;
+
+        struct {
+            const UI_METHOD *ui_method;
+            void *ui_method_data;
+        } ui_method;
+    } _;
+
+    /*-
+     * Flags section
+     */
+
+    /* Set to indicate that caching should be done */
+    unsigned int flag_cache_passphrase:1;
+
+    /*-
+     * Misc section: caches and other
+     */
+
+    char *cached_passphrase;
+    size_t cached_passphrase_len;
+};
+
+/* Structure manipulation */
+
+void ossl_pw_clear_passphrase_data(struct ossl_passphrase_data_st *data);
+void ossl_pw_clear_passphrase_cache(struct ossl_passphrase_data_st *data);
+
+int ossl_pw_set_passphrase(struct ossl_passphrase_data_st *data,
+                           const unsigned char *passphrase,
+                           size_t passphrase_len);
+int ossl_pw_set_pem_password_cb(struct ossl_passphrase_data_st *data,
+                                pem_password_cb *cb, void *cbarg);
+int ossl_pw_set_ossl_passphrase_cb(struct ossl_passphrase_data_st *data,
+                                   OSSL_PASSPHRASE_CALLBACK *cb, void *cbarg);
+int ossl_pw_set_ui_method(struct ossl_passphrase_data_st *data,
+                          const UI_METHOD *ui_method, void *ui_data);
+
+int ossl_pw_enable_passphrase_caching(struct ossl_passphrase_data_st *data);
+int ossl_pw_disable_passphrase_caching(struct ossl_passphrase_data_st *data);
+
+/* Central function for direct calls */
+
+int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len,
+                           const OSSL_PARAM params[], int verify,
+                           struct ossl_passphrase_data_st *data);
+
+/* Callback functions */
+
+/*
+ * All of these callback expect that the callback argument is a
+ * struct ossl_passphrase_data_st
+ */
+
+pem_password_cb ossl_pw_pem_password;
+/* One callback for encoding (verification prompt) and one for decoding */
+OSSL_PASSPHRASE_CALLBACK ossl_pw_passphrase_callback_enc;
+OSSL_PASSPHRASE_CALLBACK ossl_pw_passphrase_callback_dec;
+
+#endif
diff --git a/providers/implementations/encode_decode/decode_common.c b/providers/implementations/encode_decode/decode_common.c
index 1b5eea4c8d..2277c150c1 100644
--- a/providers/implementations/encode_decode/decode_common.c
+++ b/providers/implementations/encode_decode/decode_common.c
@@ -16,6 +16,7 @@
 #include "internal/pem.h"        /* For internal PVK and "blob" functions */
 #include "internal/cryptlib.h"
 #include "crypto/asn1.h"
+#include "internal/passphrase.h"
 #include "prov/bio.h"               /* ossl_prov_bio_printf() */
 #include "prov/providercommonerr.h" /* PROV_R_READ_KEY */
 #include "encoder_local.h"
@@ -58,44 +59,22 @@ EVP_PKEY *ossl_prov_read_msblob(PROV_CTX *provctx, OSSL_CORE_BIO *cin,
     return pkey;
 }
 
-struct pwdata_st {
-    OSSL_PASSPHRASE_CALLBACK *pw_cb;
-    void *pw_cbarg;
-};
-
-pem_password_cb pw_pem_password_to_ossl_passhrase;
-int pw_pem_password_to_ossl_passhrase(char *buf, int size, int rwflag,
-                                      void *userdata)
-{
-    struct pwdata_st *data = userdata;
-    size_t pw_len = 0;
-    static char prompt_info[] = "pass phrase";
-    OSSL_PARAM params[] = {
-        OSSL_PARAM_utf8_string(OSSL_PASSPHRASE_PARAM_INFO, prompt_info,
-                               sizeof(prompt_info) - 1),
-        OSSL_PARAM_END
-    };
-    int ok = data->pw_cb(buf, (size_t)size, &pw_len, params, data->pw_cbarg);
-
-    if (ok)
-        return (int)pw_len;
-    else
-        return -1;
-}
-
 # ifndef OPENSSL_NO_RC4
 EVP_PKEY *ossl_prov_read_pvk(PROV_CTX *provctx, OSSL_CORE_BIO *cin,
                              OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg)
 {
-    BIO *in = bio_new_from_core_bio(provctx, cin);
+    BIO *in = NULL;
     EVP_PKEY *pkey = NULL;
-    struct pwdata_st pwdata;
+    struct ossl_passphrase_data_st pwdata;
 
-    pwdata.pw_cb = pw_cb;
-    pwdata.pw_cbarg = pw_cbarg;
-    pkey = b2i_PVK_bio(in, pw_pem_password_to_ossl_passhrase, &pwdata);
+    memset(&pwdata, 0, sizeof(pwdata));
+    if (!ossl_pw_set_ossl_passphrase_cb(&pwdata, pw_cb, pw_cbarg))
+        return NULL;
 
+    in = bio_new_from_core_bio(provctx, cin);
+    pkey = b2i_PVK_bio(in, ossl_pw_pem_password, &pwdata);
     BIO_free(in);
+
     return pkey;
 }
 # endif