mirror/glibc
2
0
Fork 0
mirror of git://sourceware.org/git/glibc.git synced 2024-11-21 01:12:26 +08:00
Code Issues Packages Projects Releases Wiki Activity
b868b45f67
glibc/sysdeps/aarch64/strchrnul.S

98 lines
2.6 KiB
ArmAsm
Raw Normal View History

[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
/* strchrnul - find a character or nul in a string
Update copyright dates with scripts/update-copyrights I used these shell commands: ../glibc/scripts/update-copyrights $PWD/../gnulib/build-aux/update-copyright (cd ../glibc && git commit -am"[this commit message]") and then ignored the output, which consisted lines saying "FOO: warning: copyright statement not found" for each of 6694 files FOO. I then removed trailing white space from benchtests/bench-pthread-locks.c and iconvdata/tst-iconv-big5-hkscs-to-2ucs4.c, to work around this diagnostic from Savannah: remote: *** pre-commit check failed ... remote: *** error: lines with trailing whitespace found remote: error: hook declined to update refs/heads/master
2021-01-03 03:32:25 +08:00
Copyright (C) 2014-2021 Free Software Foundation, Inc.
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library. If not, see
Prefer https to http for gnu.org and fsf.org URLs Also, change sources.redhat.com to sourceware.org. This patch was automatically generated by running the following shell script, which uses GNU sed, and which avoids modifying files imported from upstream: sed -ri ' s,(http|ftp)(://(.*\.)?(gnu|fsf|sourceware)\.org($|[^.]|\.[^a-z])),https\2,g s,(http|ftp)(://(.*\.)?)sources\.redhat\.com($|[^.]|\.[^a-z]),https\2sourceware.org\4,g ' \ $(find $(git ls-files) -prune -type f \ ! -name '*.po' \ ! -name 'ChangeLog*' \ ! -path COPYING ! -path COPYING.LIB \ ! -path manual/fdl-1.3.texi ! -path manual/lgpl-2.1.texi \ ! -path manual/texinfo.tex ! -path scripts/config.guess \ ! -path scripts/config.sub ! -path scripts/install-sh \ ! -path scripts/mkinstalldirs ! -path scripts/move-if-change \ ! -path INSTALL ! -path locale/programs/charmap-kw.h \ ! -path po/libc.pot ! -path sysdeps/gnu/errlist.c \ ! '(' -name configure \ -execdir test -f configure.ac -o -f configure.in ';' ')' \ ! '(' -name preconfigure \ -execdir test -f preconfigure.ac ';' ')' \ -print) and then by running 'make dist-prepare' to regenerate files built from the altered files, and then executing the following to cleanup: chmod a+x sysdeps/unix/sysv/linux/riscv/configure # Omit irrelevant whitespace and comment-only changes, # perhaps from a slightly-different Autoconf version. git checkout -f \ sysdeps/csky/configure \ sysdeps/hppa/configure \ sysdeps/riscv/configure \ sysdeps/unix/sysv/linux/csky/configure # Omit changes that caused a pre-commit check to fail like this: # remote: *** error: sysdeps/powerpc/powerpc64/ppc-mcount.S: trailing lines git checkout -f \ sysdeps/powerpc/powerpc64/ppc-mcount.S \ sysdeps/unix/sysv/linux/s390/s390-64/syscall.S # Omit change that caused a pre-commit check to fail like this: # remote: *** error: sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: last line does not end in newline git checkout -f sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
2019-09-07 13:40:42 +08:00
<https://www.gnu.org/licenses/>. */
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
#include <sysdep.h>
/* Assumptions:
*
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
* ARMv8-a, AArch64, Advanced SIMD.
* MTE compatible.
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
*/
#define srcin x0
#define chrin w1
#define result x0
#define src x2
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
#define tmp1 x1
#define tmp2 x3
#define tmp2w w3
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
#define vrepchr v0
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
#define vdata v1
#define qdata q1
#define vhas_nul v2
#define vhas_chr v3
#define vrepmask v4
#define vend v5
#define dend d5
/* Core algorithm:
For each 16-byte chunk we calculate a 64-bit syndrome value with four bits
per byte. For even bytes, bits 0-3 are set if the relevant byte matched the
requested character or the byte is NUL. Bits 4-7 must be zero. Bits 4-7 are
set likewise for odd bytes so that adjacent bytes can be merged. Since the
bits in the syndrome reflect the order in which things occur in the original
string, counting trailing zeros identifies exactly which byte matched. */
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
ENTRY (__strchrnul)
aarch64: use PTR_ARG and SIZE_ARG instead of DELOUSE DELOUSE was added to asm code to make them compatible with non-LP64 ABIs, but it is an unfortunate name and the code was not compatible with ABIs where pointer and size_t are different. Glibc currently only supports the LP64 ABI so these macros are not really needed or tested, but for now the name is changed to be more meaningful instead of removing them completely. Some DELOUSE macros were dropped: clone, strlen and strnlen used it unnecessarily. The out of tree ILP32 patches are currently not maintained and will likely need a rework to rebase them on top of the time64 changes.
2020-12-17 18:03:05 +08:00
PTR_ARG (0)
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
bic src, srcin, 15
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
dup vrepchr.16b, chrin
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
ld1 {vdata.16b}, [src]
mov tmp2w, 0xf00f
dup vrepmask.8h, tmp2w
cmeq vhas_chr.16b, vdata.16b, vrepchr.16b
cmhs vhas_chr.16b, vhas_chr.16b, vdata.16b
lsl tmp2, srcin, 2
and vhas_chr.16b, vhas_chr.16b, vrepmask.16b
addp vend.16b, vhas_chr.16b, vhas_chr.16b /* 128->64 */
fmov tmp1, dend
lsr tmp1, tmp1, tmp2 /* Mask padding bits. */
cbz tmp1, L(loop)
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
rbit tmp1, tmp1
clz tmp1, tmp1
add result, srcin, tmp1, lsr 2
ret
.p2align 4
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
L(loop):
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
ldr qdata, [src, 16]!
cmeq vhas_chr.16b, vdata.16b, vrepchr.16b
cmhs vhas_chr.16b, vhas_chr.16b, vdata.16b
umaxp vend.16b, vhas_chr.16b, vhas_chr.16b
fmov tmp1, dend
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
cbz tmp1, L(loop)
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
and vhas_chr.16b, vhas_chr.16b, vrepmask.16b
addp vend.16b, vhas_chr.16b, vhas_chr.16b /* 128->64 */
fmov tmp1, dend
#ifndef __AARCH64EB__
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
rbit tmp1, tmp1
aarch64: MTE compatible strchrnul Introduce an Arm MTE compatible strchrnul implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra <wilco.dijkstra@arm.com>
2020-06-05 23:18:49 +08:00
#endif
clz tmp1, tmp1
add result, src, tmp1, lsr 2
[AArch64] Add optimized strchrnul. Here is an optimized implementation of __strchrnul. The simplification that we don't have to track precisely why the loop terminates (match or end-of-string) means we have to do less work in both setup and the core inner loop. That means this should never be slower than strchr. As with strchr, the use of LD1 means we do not need different versions for big-/little-endian.
2014-11-05 21:51:56 +08:00
ret
END(__strchrnul)
weak_alias (__strchrnul, strchrnul)
Reference in New Issue Copy Permalink