From 82741e9c8980f96e4b3bba9e54e5904bc3e385bd Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Fri, 6 Jun 2014 20:48:35 +0200 Subject: [PATCH] Engage GHASH for ARMv8. --- Configure | 6 +++--- TABLE | 6 +++--- crypto/modes/Makefile | 4 +++- crypto/modes/gcm128.c | 25 +++++++++++++++++++++---- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Configure b/Configure index 5d2bb63ef1..7d4ed2c321 100755 --- a/Configure +++ b/Configure @@ -136,8 +136,8 @@ my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o:::::sha1-alpha.o:::::::ghash-a my $mips64_asm=":bn-mips.o mips-mont.o::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::"; my $mips32_asm=$mips64_asm; $mips32_asm =~ s/\s*sha512\-mips\.o//; my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o:"; -my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::void"; -my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o:::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o::::::::"; +my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o::void"; +my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o:::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o:"; my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::32"; my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::64"; my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o::::::::"; @@ -1617,7 +1617,7 @@ else { $wp_obj="wp_block.o"; } $cmll_obj=$cmll_enc unless ($cmll_obj =~ /.o$/); -if ($modes_obj =~ /ghash/) +if ($modes_obj =~ /ghash\-/) { $cflags.=" -DGHASH_ASM"; } diff --git a/TABLE b/TABLE index 519342a6d6..108ba87c02 100644 --- a/TABLE +++ b/TABLE @@ -1110,7 +1110,7 @@ $rmd160_obj = $rc5_obj = $wp_obj = $cmll_obj = -$modes_obj = ghash-armv4.o +$modes_obj = ghash-armv4.o ghashv8-armx.o $engines_obj = $perlasm_scheme = void $dso_scheme = dlfcn @@ -4014,7 +4014,7 @@ $rmd160_obj = $rc5_obj = $wp_obj = $cmll_obj = -$modes_obj = +$modes_obj = ghashv8-armx.o $engines_obj = $perlasm_scheme = linux64 $dso_scheme = dlfcn @@ -4212,7 +4212,7 @@ $rmd160_obj = $rc5_obj = $wp_obj = $cmll_obj = -$modes_obj = ghash-armv4.o +$modes_obj = ghash-armv4.o ghashv8-armx.o $engines_obj = $perlasm_scheme = void $dso_scheme = dlfcn diff --git a/crypto/modes/Makefile b/crypto/modes/Makefile index b8cfa71fa3..d03284e869 100644 --- a/crypto/modes/Makefile +++ b/crypto/modes/Makefile @@ -58,14 +58,16 @@ ghash-alpha.s: asm/ghash-alpha.pl (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \ $(PERL) asm/ghash-alpha.pl > $$preproc && \ $(CC) -E $$preproc > $@ && rm $$preproc) - ghash-parisc.s: asm/ghash-parisc.pl $(PERL) asm/ghash-parisc.pl $(PERLASM_SCHEME) $@ +ghashv8-armx.S: asm/ghashv8-armx.pl + $(PERL) asm/ghashv8-armx.pl $(PERLASM_SCHEME) $@ # GNU make "catch all" ghash-%.S: asm/ghash-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ ghash-armv4.o: ghash-armv4.S +ghashv8-armx.o: ghashv8-armx.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c index 9afef1cf4e..19cbcf5572 100644 --- a/crypto/modes/gcm128.c +++ b/crypto/modes/gcm128.c @@ -645,7 +645,7 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2]) #endif -#if TABLE_BITS==4 && defined(GHASH_ASM) +#if TABLE_BITS==4 && (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ)) # if !defined(I386_ONLY) && \ (defined(__i386) || defined(__i386__) || \ defined(__x86_64) || defined(__x86_64__) || \ @@ -676,14 +676,21 @@ void gcm_ghash_4bit_mmx(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len void gcm_gmult_4bit_x86(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif -# elif defined(__arm__) || defined(__arm) +# elif defined(__arm__) || defined(__arm) || defined(__aarch64__) # include "arm_arch.h" # if __ARM_ARCH__>=7 # define GHASH_ASM_ARM # define GCM_FUNCREF_4BIT +# define PMULL_CAPABLE (OPENSSL_armcap_P & ARMV8_PMULL) +# if defined(__arm__) || defined(__arm) +# define NEON_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) +# endif void gcm_init_neon(u128 Htable[16],const u64 Xi[2]); void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); +void gcm_init_v8(u128 Htable[16],const u64 Xi[2]); +void gcm_gmult_v8(u64 Xi[2],const u128 Htable[16]); +void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif # elif defined(__sparc__) || defined(__sparc) # include "sparc_arch.h" @@ -767,11 +774,21 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) ctx->ghash = gcm_ghash_4bit; # endif # elif defined(GHASH_ASM_ARM) - if (OPENSSL_armcap_P & ARMV7_NEON) { +# ifdef PMULL_CAPABLE + if (PMULL_CAPABLE) { + gcm_init_v8(ctx->Htable,ctx->H.u); + ctx->gmult = gcm_gmult_v8; + ctx->ghash = gcm_ghash_v8; + } else +# endif +# ifdef NEON_CAPABLE + if (NEON_CAPABLE) { gcm_init_neon(ctx->Htable,ctx->H.u); ctx->gmult = gcm_gmult_neon; ctx->ghash = gcm_ghash_neon; - } else { + } else +# endif + { gcm_init_4bit(ctx->Htable,ctx->H.u); ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit;