From 79040cf29e011c21789563d74da626b7465a0540 Mon Sep 17 00:00:00 2001 From: Juergen Christ Date: Fri, 20 Jan 2023 17:43:59 +0100 Subject: [PATCH] S390x: Support ME and CRT offloading S390x has to ability to offload modular exponentiation and CRT operations to Crypto Express Adapters. This possible performance optimization was not yet used by OpenSSL. Add support for offloading and implement an optimized version of RSA and DH with it. The environment variable OPENSSL_s390xcap now recognizes the token "nocex" to prevent offloading. Signed-off-by: Juergen Christ Reviewed-by: Paul Dale Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/20113) --- crypto/bn/bn_s390x.c | 143 ++++++++++++++++++++++++++++++++++ crypto/bn/build.info | 2 +- crypto/dh/dh_key.c | 4 + crypto/rsa/rsa_ossl.c | 35 +++++++++ crypto/s390x_arch.h | 9 +++ crypto/s390xcap.c | 87 ++++++++++++++++++++- doc/man3/OPENSSL_s390xcap.pod | 7 +- include/crypto/bn.h | 9 +++ 8 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 crypto/bn/bn_s390x.c diff --git a/crypto/bn/bn_s390x.c b/crypto/bn/bn_s390x.c new file mode 100644 index 0000000000..da69b02d90 --- /dev/null +++ b/crypto/bn/bn_s390x.c @@ -0,0 +1,143 @@ +/* + * Copyright 2023-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "crypto/bn.h" +#include "crypto/s390x_arch.h" + +#ifdef S390X_MOD_EXP + +# include +# include +# include +# include +# include +# include +# include + +static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m) +{ + struct ica_rsa_modexpo me; + unsigned char *buffer; + size_t size; + int res = 0; + + if (OPENSSL_s390xcex == -1) + return 0; + size = BN_num_bytes(m); + buffer = OPENSSL_zalloc(4 * size); + if (buffer == NULL) + return 0; + me.inputdata = buffer; + me.inputdatalength = size; + me.outputdata = buffer + size; + me.outputdatalength = size; + me.b_key = buffer + 2 * size; + me.n_modulus = buffer + 3 * size; + if (BN_bn2binpad(a, me.inputdata, size) == -1 + || BN_bn2binpad(p, me.b_key, size) == -1 + || BN_bn2binpad(m, me.n_modulus, size) == -1) + goto dealloc; + if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) { + if (BN_bin2bn(me.outputdata, size, r) != NULL) + res = 1; + } else if (errno == EBADF) { + /*- + * In this cases, someone (e.g. a sandbox) closed the fd. + * Make sure to not further use this hardware acceleration. + */ + OPENSSL_s390xcex = -1; + } + dealloc: + OPENSSL_clear_free(buffer, 4 * size); + return res; +} + +int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) +{ + if (s390x_mod_exp_hw(r, a, p, m) == 1) + return 1; + return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); +} + +int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, + const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp) +{ + struct ica_rsa_modexpo_crt crt; + unsigned char *buffer, *part; + size_t size, plen, qlen; + int res = 0; + + if (OPENSSL_s390xcex == -1) + return 0; + /*- + * Hardware-accelerated CRT can only deal with p>q. Fall back to + * software in the (hopefully rare) other cases. + */ + if (BN_ucmp(p, q) != 1) + return 0; + plen = BN_num_bytes(p); + qlen = BN_num_bytes(q); + size = (plen > qlen ? plen : qlen); + buffer = OPENSSL_zalloc(9 * size + 24); + if (buffer == NULL) + return 0; + part = buffer; + crt.inputdata = part; + crt.inputdatalength = 2 * size; + part += 2 * size; + crt.outputdata = part; + crt.outputdatalength = 2 * size; + part += 2 * size; + crt.bp_key = part; + part += size + 8; + crt.bq_key = part; + part += size; + crt.np_prime = part; + part += size + 8; + crt.nq_prime = part; + part += size; + crt.u_mult_inv = part; + if (BN_bn2binpad(i, crt.inputdata, crt.inputdatalength) == -1 + || BN_bn2binpad(p, crt.np_prime, size + 8) == -1 + || BN_bn2binpad(q, crt.nq_prime, size) == -1 + || BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1 + || BN_bn2binpad(dmq, crt.bq_key, size) == -1 + || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1) + goto dealloc; + if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) { + if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL) + res = 1; + } else if (errno == EBADF) { + /*- + * In this cases, someone (e.g. a sandbox) closed the fd. + * Make sure to not further use this hardware acceleration. + */ + OPENSSL_s390xcex = -1; + } + dealloc: + OPENSSL_clear_free(buffer, 9 * size + 24); + return res; +} + +#else +int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) +{ + return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); +} + +int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, + const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp) +{ + return 0; +} + +#endif diff --git a/crypto/bn/build.info b/crypto/bn/build.info index 0732519a11..77ec6d4721 100644 --- a/crypto/bn/build.info +++ b/crypto/bn/build.info @@ -58,7 +58,7 @@ IF[{- !$disabled{asm} -}] IF[{- ($target{perlasm_scheme} // '') eq '31' -}] $BNASM_s390x=bn_asm.c s390x-mont.S ELSE - $BNASM_s390x=asm/s390x.S s390x-mont.S + $BNASM_s390x=asm/s390x.S s390x-mont.S bn_s390x.c ENDIF $BNDEF_s390x=OPENSSL_BN_ASM_MONT $BNASM_s390x_ec2m=s390x-gf2m.s diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c index 0854395fc5..328780b9a9 100644 --- a/crypto/dh/dh_key.c +++ b/crypto/dh/dh_key.c @@ -184,7 +184,11 @@ static int dh_bn_mod_exp(const DH *dh, BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) { +#ifdef S390X_MOD_EXP + return s390x_mod_exp(r, a, p, m, ctx, m_ctx); +#else return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); +#endif } static int dh_init(DH *dh) diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c index 10d08ebc35..ec69076e63 100644 --- a/crypto/rsa/rsa_ossl.c +++ b/crypto/rsa/rsa_ossl.c @@ -33,6 +33,27 @@ static int rsa_ossl_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, BN_CTX *ctx); static int rsa_ossl_init(RSA *rsa); static int rsa_ossl_finish(RSA *rsa); +#ifdef S390X_MOD_EXP +static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, + BN_CTX *ctx); +static RSA_METHOD rsa_pkcs1_ossl_meth = { + "OpenSSL PKCS#1 RSA", + rsa_ossl_public_encrypt, + rsa_ossl_public_decrypt, /* signature verification */ + rsa_ossl_private_encrypt, /* signing */ + rsa_ossl_private_decrypt, + rsa_ossl_s390x_mod_exp, + s390x_mod_exp, + rsa_ossl_init, + rsa_ossl_finish, + RSA_FLAG_FIPS_METHOD, /* flags */ + NULL, + 0, /* rsa_sign */ + 0, /* rsa_verify */ + NULL, /* rsa_keygen */ + NULL /* rsa_multi_prime_keygen */ +}; +#else static RSA_METHOD rsa_pkcs1_ossl_meth = { "OpenSSL PKCS#1 RSA", rsa_ossl_public_encrypt, @@ -51,6 +72,7 @@ static RSA_METHOD rsa_pkcs1_ossl_meth = { NULL, /* rsa_keygen */ NULL /* rsa_multi_prime_keygen */ }; +#endif static const RSA_METHOD *default_RSA_meth = &rsa_pkcs1_ossl_meth; @@ -1118,3 +1140,16 @@ static int rsa_ossl_finish(RSA *rsa) BN_MONT_CTX_free(rsa->_method_mod_q); return 1; } + +#ifdef S390X_MOD_EXP +static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, + BN_CTX *ctx) +{ + if (rsa->version != RSA_ASN1_VERSION_MULTI) { + if (s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp) == 1) + return 1; + } + return rsa_ossl_mod_exp(r0, i, rsa, ctx); +} + +#endif diff --git a/crypto/s390x_arch.h b/crypto/s390x_arch.h index c48cde8d9d..cfb3460d5b 100644 --- a/crypto/s390x_arch.h +++ b/crypto/s390x_arch.h @@ -12,6 +12,8 @@ # ifndef __ASSEMBLER__ +#include "crypto/bn.h" + void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc, void *param); void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out, @@ -77,6 +79,13 @@ __attribute__ ((visibility("hidden"))) #endif extern struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P; +#ifdef S390X_MOD_EXP +# if defined(__GNUC__) && defined(__linux) +__attribute__ ((visibility("hidden"))) +# endif +extern int OPENSSL_s390xcex; +#endif + /* Max number of 64-bit words currently returned by STFLE */ # define S390X_STFLE_MAX 3 diff --git a/crypto/s390xcap.c b/crypto/s390xcap.c index 59f0f3ef45..5d2e96805c 100644 --- a/crypto/s390xcap.c +++ b/crypto/s390xcap.c @@ -16,6 +16,15 @@ #include "crypto/ctype.h" #include "s390x_arch.h" +#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE) +# include +# include +# include +# include +# include +# include +#endif + #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) # if __GLIBC_PREREQ(2, 16) # include @@ -67,19 +76,41 @@ void OPENSSL_vx_probe(void); #endif static const char *env; -static int parse_env(struct OPENSSL_s390xcap_st *cap); +static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex); void OPENSSL_s390x_facilities(void); void OPENSSL_s390x_functions(void); struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P; +#ifdef S390X_MOD_EXP +static int probe_cex(void); +int OPENSSL_s390xcex; + +#if defined(__GNUC__) +__attribute__ ((visibility("hidden"))) +#endif +void OPENSSL_s390x_cleanup(void); + +#if defined(__GNUC__) +__attribute__ ((visibility("hidden"))) +#endif +void OPENSSL_s390x_cleanup(void) +{ + if (OPENSSL_s390xcex != -1) { + (void)close(OPENSSL_s390xcex); + OPENSSL_s390xcex = -1; + } +} +#endif + #if defined(__GNUC__) && defined(__linux) __attribute__ ((visibility("hidden"))) #endif void OPENSSL_cpuid_setup(void) { struct OPENSSL_s390xcap_st cap; + int cex = 1; if (OPENSSL_s390xcap_P.stfle[0]) return; @@ -140,7 +171,7 @@ void OPENSSL_cpuid_setup(void) env = getenv("OPENSSL_s390xcap"); if (env != NULL) { - if (!parse_env(&cap)) + if (!parse_env(&cap, &cex)) env = NULL; } @@ -178,9 +209,52 @@ void OPENSSL_cpuid_setup(void) OPENSSL_s390xcap_P.kdsa[0] &= cap.kdsa[0]; OPENSSL_s390xcap_P.kdsa[1] &= cap.kdsa[1]; } + +#ifdef S390X_MOD_EXP + if (cex == 0) { + OPENSSL_s390xcex = -1; + } else { + OPENSSL_s390xcex = open("/dev/z90crypt", O_RDWR | O_CLOEXEC); + if (probe_cex() == 1) + OPENSSL_atexit(OPENSSL_s390x_cleanup); + } +#endif } -static int parse_env(struct OPENSSL_s390xcap_st *cap) +#ifdef S390X_MOD_EXP +static int probe_cex(void) +{ + struct ica_rsa_modexpo me; + const unsigned char inval[16] = { + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,2 + }; + const unsigned char modulus[16] = { + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,3 + }; + unsigned char res[16]; + int olderrno; + int rc = 1; + + me.inputdata = (unsigned char *)inval; + me.inputdatalength = sizeof(inval); + me.outputdata = (unsigned char *)res; + me.outputdatalength = sizeof(res); + me.b_key = (unsigned char *)inval; + me.n_modulus = (unsigned char *)modulus; + olderrno = errno; + if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) == -1) { + (void)close(OPENSSL_s390xcex); + OPENSSL_s390xcex = -1; + rc = 0; + } + errno = olderrno; + return rc; +} +#endif + +static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex) { /*- * CPU model data @@ -732,6 +806,13 @@ static int parse_env(struct OPENSSL_s390xcap_st *cap) else if TOK_CPU(z15) else if TOK_CPU(z16) + /* nocex to deactivate cex support */ + else if (sscanf(tok_begin, " %" STR(LEN) "s %" STR(LEN) "s ", + tok[0], tok[1]) == 1 + && !strcmp(tok[0], "nocex")) { + *cex = 0; + } + /* whitespace(ignored) or invalid tokens */ else { while (*tok_begin != '\0') { diff --git a/doc/man3/OPENSSL_s390xcap.pod b/doc/man3/OPENSSL_s390xcap.pod index 2cddec5eaf..a8ade1b612 100644 --- a/doc/man3/OPENSSL_s390xcap.pod +++ b/doc/man3/OPENSSL_s390xcap.pod @@ -25,7 +25,7 @@ processed from left to right (whitespace is ignored): OPENSSL_s390xcap=";;..." -There are three types of tokens: +There are four types of tokens: =over 4 @@ -51,6 +51,11 @@ Store-facility-list-extended (stfle) followed by three 64-bit masks. The part of the environment variable's mask corresponding to the stfle instruction is set to the specified 192-bit mask. +=item nocex + +Deactivate modular exponentiation and CRT operation offloading to +Crypto Express Adapters. + =back The 64-bit masks are specified in hexadecimal notation. The 0x prefix is diff --git a/include/crypto/bn.h b/include/crypto/bn.h index 2453cb814e..b354578ee0 100644 --- a/include/crypto/bn.h +++ b/include/crypto/bn.h @@ -122,4 +122,13 @@ int ossl_bn_rsa_do_unblind(const BIGNUM *intermediate, const BIGNUM *to_mod, BN_CTX *ctx, unsigned char *buf, int num); +#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE) && defined (__s390x__) +# define S390X_MOD_EXP +#endif + +int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); +int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, + const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp); + #endif