x86: Add COND_VZEROUPPER that can replace vzeroupper if no ret

The RTM vzeroupper mitigation has no way of replacing inline
vzeroupper not before a return.

This can be useful when hoisting a vzeroupper to save code size
for example:

```
L(foo):
	cmpl	%eax, %edx
	jz	L(bar)
	tzcntl	%eax, %eax
	addq	%rdi, %rax
	VZEROUPPER_RETURN

L(bar):
	xorl	%eax, %eax
	VZEROUPPER_RETURN
```

Can become:

```
L(foo):
	COND_VZEROUPPER
	cmpl	%eax, %edx
	jz	L(bar)
	tzcntl	%eax, %eax
	addq	%rdi, %rax
	ret

L(bar):
	xorl	%eax, %eax
	ret
```

This code does not change any existing functionality.

There is no difference in the objdump of libc.so before and after this
patch.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
This commit is contained in:
Noah Goldstein 2022-06-06 21:11:28 -07:00
parent 8a780a6b91
commit dd5c483b25
2 changed files with 19 additions and 0 deletions

View File

@ -20,6 +20,7 @@
#ifndef _AVX_RTM_VECS_H
#define _AVX_RTM_VECS_H 1
#define COND_VZEROUPPER COND_VZEROUPPER_XTEST
#define ZERO_UPPER_VEC_REGISTERS_RETURN \
ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST

View File

@ -106,6 +106,24 @@ lose: \
vzeroupper; \
ret
/* Can be used to replace vzeroupper that is not directly before a
return. This is useful when hoisting a vzeroupper from multiple
return paths to decrease the total number of vzerouppers and code
size. */
#define COND_VZEROUPPER_XTEST \
xtest; \
jz 1f; \
vzeroall; \
jmp 2f; \
1: \
vzeroupper; \
2:
/* In RTM define this as COND_VZEROUPPER_XTEST. */
#ifndef COND_VZEROUPPER
# define COND_VZEROUPPER vzeroupper
#endif
/* Zero upper vector registers and return. */
#ifndef ZERO_UPPER_VEC_REGISTERS_RETURN
# define ZERO_UPPER_VEC_REGISTERS_RETURN \