mirror of
git://gcc.gnu.org/git/gcc.git
synced 2025-04-16 13:11:24 +08:00
aarch64: Add support for unpacked SVE ASRD
This patch adds support for both conditional and unconditional unpacked ASRD. This meant adding a new define_insn for the unconditional form, instead of reusing the conditional instructions. It also meant extending the current conditional patterns to support merging with any independent value, not just zero. gcc/ * config/aarch64/aarch64-sve.md (sdiv_pow2<mode>3): Extend from SVE_FULL_I to SVE_I. Generate an UNSPEC_PRED_X. (*sdiv_pow2<mode>3): New pattern. (@cond_<sve_int_op><mode>): Extend from SVE_FULL_I to SVE_I. Wrap the ASRD in an UNSPEC_PRED_X. (*cond_<sve_int_op><mode>_2): Likewise. Replace the UNSPEC_PRED_X predicate with a constant PTRUE, if it isn't already. (*cond_<sve_int_op><mode>_z): Replace with... (*cond_<sve_int_op><mode>_any): ...this new pattern. gcc/testsuite/ * gcc.target/aarch64/sve/asrdiv_4.c: New test. * gcc.target/aarch64/sve/cond_asrd_1.c: Likewise. * gcc.target/aarch64/sve/cond_asrd_1_run.c: Likewise. * gcc.target/aarch64/sve/cond_asrd_2.c: Likewise. * gcc.target/aarch64/sve/cond_asrd_2_run.c: Likewise. * gcc.target/aarch64/sve/cond_asrd_3.c: Likewise. * gcc.target/aarch64/sve/cond_asrd_3_run.c: Likewise.
This commit is contained in:
Richard Sandiford
parent
37426e0f06
commit
a958b2fc6d
@ -4724,68 +4724,111 @@
|
||||
;; - URSHR (SVE2)
|
||||
;; -------------------------------------------------------------------------
|
||||
|
||||
;; Unpredicated <SVE_INT_OP>.
|
||||
;; Unpredicated ASRD.
|
||||
(define_expand "sdiv_pow2<mode>3"
|
||||
[(set (match_operand:SVE_FULL_I 0 "register_operand")
|
||||
(unspec:SVE_FULL_I
|
||||
[(set (match_operand:SVE_I 0 "register_operand")
|
||||
(unspec:SVE_I
|
||||
[(match_dup 3)
|
||||
(unspec:SVE_FULL_I
|
||||
[(match_operand:SVE_FULL_I 1 "register_operand")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:SVE_I 1 "register_operand")
|
||||
(match_operand 2 "aarch64_simd_rshift_imm")]
|
||||
UNSPEC_ASRD)
|
||||
(match_dup 1)]
|
||||
UNSPEC_SEL))]
|
||||
UNSPEC_ASRD)]
|
||||
UNSPEC_PRED_X))]
|
||||
"TARGET_SVE"
|
||||
{
|
||||
operands[3] = aarch64_ptrue_reg (<VPRED>mode);
|
||||
}
|
||||
)
|
||||
|
||||
;; Predicated right shift with merging.
|
||||
;; Predicated ASRD.
|
||||
(define_insn "*sdiv_pow2<mode>3"
|
||||
[(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:SVE_I 2 "register_operand" "0, w")
|
||||
(match_operand:SVE_I 3 "aarch64_simd_rshift_imm")]
|
||||
UNSPEC_ASRD)]
|
||||
UNSPEC_PRED_X))]
|
||||
"TARGET_SVE"
|
||||
"@
|
||||
asrd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
|
||||
movprfx\t%0, %2\;asrd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
|
||||
[(set_attr "movprfx" "*,yes")])
|
||||
|
||||
;; Predicated shift with merging.
|
||||
(define_expand "@cond_<sve_int_op><mode>"
|
||||
[(set (match_operand:SVE_FULL_I 0 "register_operand")
|
||||
(unspec:SVE_FULL_I
|
||||
[(set (match_operand:SVE_I 0 "register_operand")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:<VPRED> 1 "register_operand")
|
||||
(unspec:SVE_FULL_I
|
||||
[(match_operand:SVE_FULL_I 2 "register_operand")
|
||||
(match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)
|
||||
(match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")]
|
||||
(unspec:SVE_I
|
||||
[(match_dup 5)
|
||||
(unspec:SVE_I
|
||||
[(match_operand:SVE_I 2 "register_operand")
|
||||
(match_operand:SVE_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)]
|
||||
UNSPEC_PRED_X)
|
||||
(match_operand:SVE_I 4 "aarch64_simd_reg_or_zero")]
|
||||
UNSPEC_SEL))]
|
||||
"TARGET_SVE"
|
||||
{
|
||||
operands[5] = aarch64_ptrue_reg (<VPRED>mode);
|
||||
}
|
||||
)
|
||||
|
||||
;; Predicated right shift, merging with the first input.
|
||||
(define_insn "*cond_<sve_int_op><mode>_2"
|
||||
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
|
||||
(unspec:SVE_FULL_I
|
||||
;; Predicated shift, merging with the first input.
|
||||
(define_insn_and_rewrite "*cond_<sve_int_op><mode>_2"
|
||||
[(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
|
||||
(unspec:SVE_FULL_I
|
||||
[(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
|
||||
(match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)
|
||||
(unspec:SVE_I
|
||||
[(match_operand 4)
|
||||
(unspec:SVE_I
|
||||
[(match_operand:SVE_I 2 "register_operand" "0, w")
|
||||
(match_operand:SVE_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)]
|
||||
UNSPEC_PRED_X)
|
||||
(match_dup 2)]
|
||||
UNSPEC_SEL))]
|
||||
"TARGET_SVE"
|
||||
"@
|
||||
<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
|
||||
movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
|
||||
"&& !CONSTANT_P (operands[4])"
|
||||
{
|
||||
operands[4] = CONSTM1_RTX (<VPRED>mode);
|
||||
}
|
||||
[(set_attr "movprfx" "*,yes")])
|
||||
|
||||
;; Predicated right shift, merging with zero.
|
||||
(define_insn "*cond_<sve_int_op><mode>_z"
|
||||
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=w")
|
||||
(unspec:SVE_FULL_I
|
||||
[(match_operand:<VPRED> 1 "register_operand" "Upl")
|
||||
(unspec:SVE_FULL_I
|
||||
[(match_operand:SVE_FULL_I 2 "register_operand" "w")
|
||||
(match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)
|
||||
(match_operand:SVE_FULL_I 4 "aarch64_simd_imm_zero")]
|
||||
UNSPEC_SEL))]
|
||||
"TARGET_SVE"
|
||||
"movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3"
|
||||
[(set_attr "movprfx" "yes")])
|
||||
;; Predicated shift, merging with an independent value.
|
||||
(define_insn_and_rewrite "*cond_<sve_int_op><mode>_any"
|
||||
[(set (match_operand:SVE_I 0 "register_operand" "=w, &w, ?&w")
|
||||
(unspec:SVE_I
|
||||
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
|
||||
(unspec:SVE_I
|
||||
[(match_operand 5)
|
||||
(unspec:SVE_I
|
||||
[(match_operand:SVE_I 2 "register_operand" "w, w, w")
|
||||
(match_operand:SVE_I 3 "aarch64_simd_<lr>shift_imm")]
|
||||
SVE_INT_SHIFT_IMM)]
|
||||
UNSPEC_PRED_X)
|
||||
(match_operand:SVE_I 4 "aarch64_simd_reg_or_zero" "Dz, 0, w")]
|
||||
UNSPEC_SEL))]
|
||||
"TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
|
||||
"@
|
||||
movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
|
||||
movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
|
||||
#"
|
||||
"&& reload_completed
|
||||
&& register_operand (operands[4], <MODE>mode)
|
||||
&& !rtx_equal_p (operands[0], operands[4])"
|
||||
{
|
||||
emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2],
|
||||
operands[4], operands[1]));
|
||||
operands[4] = operands[2] = operands[0];
|
||||
}
|
||||
[(set_attr "movprfx" "yes")]
|
||||
)
|
||||
|
||||
;; -------------------------------------------------------------------------
|
||||
;; ---- [FP<-INT] General binary arithmetic corresponding to unspecs
|
||||
|
||||
29
gcc/testsuite/gcc.target/aarch64/sve/asrdiv_4.c
Normal file
29
gcc/testsuite/gcc.target/aarch64/sve/asrdiv_4.c
Normal file
@ -0,0 +1,29 @@
|
||||
/* { dg-do compile } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define DEF_LOOP(TYPE1, TYPE2, COUNT) \
|
||||
void __attribute__ ((noipa)) \
|
||||
test_##TYPE1##_##TYPE2##_##TYPE3 (TYPE2 *restrict r, \
|
||||
TYPE1 *restrict pred, \
|
||||
TYPE2 *restrict a) \
|
||||
{ \
|
||||
for (int i = 0; i < COUNT; ++i) \
|
||||
if (pred[i]) \
|
||||
r[i] = a[i] / 16; \
|
||||
}
|
||||
|
||||
#define TEST_ALL(T) \
|
||||
T (int16_t, int8_t, 7) \
|
||||
T (int32_t, int8_t, 3) \
|
||||
T (int32_t, int16_t, 3) \
|
||||
T (int64_t, int8_t, 5) \
|
||||
T (int64_t, int16_t, 5) \
|
||||
T (int64_t, int32_t, 5)
|
||||
|
||||
TEST_ALL (DEF_LOOP)
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, #4\n} 3 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #4\n} 2 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #4\n} 1 } } */
|
||||
32
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_1.c
Normal file
32
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_1.c
Normal file
@ -0,0 +1,32 @@
|
||||
/* { dg-do compile } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define DEF_LOOP(TYPE1, TYPE2, COUNT) \
|
||||
void __attribute__ ((noipa)) \
|
||||
test_##TYPE1##_##TYPE2 (TYPE2 *__restrict r, \
|
||||
TYPE1 *__restrict a, \
|
||||
TYPE2 *__restrict b) \
|
||||
{ \
|
||||
for (int i = 0; i < COUNT; ++i) \
|
||||
r[i] = a[i] == 0 ? b[i] / 16 : b[i]; \
|
||||
}
|
||||
|
||||
#define TEST_ALL(T) \
|
||||
T (int16_t, int8_t, 7) \
|
||||
T (int32_t, int8_t, 3) \
|
||||
T (int32_t, int16_t, 3) \
|
||||
T (int64_t, int8_t, 5) \
|
||||
T (int64_t, int16_t, 5) \
|
||||
T (int64_t, int32_t, 5)
|
||||
|
||||
TEST_ALL (DEF_LOOP)
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, #4\n} 3 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #4\n} 2 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #4\n} 1 } } */
|
||||
|
||||
/* { dg-final { scan-assembler-not {\tmov\tz} } } */
|
||||
/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */
|
||||
/* { dg-final { scan-assembler-not {\tsel\t} } } */
|
||||
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_1_run.c
Normal file
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_1_run.c
Normal file
@ -0,0 +1,26 @@
|
||||
/* { dg-do run { target { aarch64_sve_hw } } } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include "cond_asrd_1.c"
|
||||
|
||||
#define TEST_LOOP(TYPE1, TYPE2, N) \
|
||||
{ \
|
||||
TYPE1 a[N]; \
|
||||
TYPE2 r[N], b[N]; \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
{ \
|
||||
a[i] = i % 3 < 2 ? 0 : i * 13; \
|
||||
b[i] = (i & 1 ? i : -i) * 17; \
|
||||
asm volatile ("" ::: "memory"); \
|
||||
} \
|
||||
test_##TYPE1##_##TYPE2 (r, a, b); \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
if (r[i] != (TYPE2) (a[i] == 0 ? b[i] / 16 : b[i])) \
|
||||
__builtin_abort (); \
|
||||
}
|
||||
|
||||
int main ()
|
||||
{
|
||||
TEST_ALL (TEST_LOOP)
|
||||
return 0;
|
||||
}
|
||||
35
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_2.c
Normal file
35
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_2.c
Normal file
@ -0,0 +1,35 @@
|
||||
/* { dg-do compile } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define DEF_LOOP(TYPE1, TYPE2, COUNT) \
|
||||
void __attribute__ ((noipa)) \
|
||||
test_##TYPE1##_##TYPE2 (TYPE2 *__restrict r, \
|
||||
TYPE1 *__restrict a, \
|
||||
TYPE2 *__restrict b) \
|
||||
{ \
|
||||
for (int i = 0; i < COUNT; ++i) \
|
||||
r[i] = a[i] == 0 ? b[i] / 16 : a[i]; \
|
||||
}
|
||||
|
||||
#define TEST_ALL(T) \
|
||||
T (int16_t, int8_t, 7) \
|
||||
T (int32_t, int8_t, 3) \
|
||||
T (int32_t, int16_t, 3) \
|
||||
T (int64_t, int8_t, 5) \
|
||||
T (int64_t, int16_t, 5) \
|
||||
T (int64_t, int32_t, 5)
|
||||
|
||||
TEST_ALL (DEF_LOOP)
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, #4\n} 3 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #4\n} 2 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #4\n} 1 } } */
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b\n} 3 } } */
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
|
||||
|
||||
/* { dg-final { scan-assembler-not {\tmov\tz} } } */
|
||||
/* { dg-final { scan-assembler-not {\tsel\t} } } */
|
||||
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_2_run.c
Normal file
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_2_run.c
Normal file
@ -0,0 +1,26 @@
|
||||
/* { dg-do run { target { aarch64_sve_hw } } } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include "cond_asrd_2.c"
|
||||
|
||||
#define TEST_LOOP(TYPE1, TYPE2, N) \
|
||||
{ \
|
||||
TYPE1 a[N]; \
|
||||
TYPE2 r[N], b[N]; \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
{ \
|
||||
a[i] = i % 3 < 2 ? 0 : i * 13; \
|
||||
b[i] = (i & 1 ? i : -i) * 17; \
|
||||
asm volatile ("" ::: "memory"); \
|
||||
} \
|
||||
test_##TYPE1##_##TYPE2 (r, a, b); \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
if (r[i] != (TYPE2) (a[i] == 0 ? b[i] / 16 : a[i])) \
|
||||
__builtin_abort (); \
|
||||
}
|
||||
|
||||
int main ()
|
||||
{
|
||||
TEST_ALL (TEST_LOOP)
|
||||
return 0;
|
||||
}
|
||||
35
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_3.c
Normal file
35
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_3.c
Normal file
@ -0,0 +1,35 @@
|
||||
/* { dg-do compile } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define DEF_LOOP(TYPE1, TYPE2, COUNT) \
|
||||
void __attribute__ ((noipa)) \
|
||||
test_##TYPE1##_##TYPE2 (TYPE2 *__restrict r, \
|
||||
TYPE1 *__restrict a, \
|
||||
TYPE2 *__restrict b) \
|
||||
{ \
|
||||
for (int i = 0; i < COUNT; ++i) \
|
||||
r[i] = a[i] == 0 ? b[i] / 16 : 0; \
|
||||
}
|
||||
|
||||
#define TEST_ALL(T) \
|
||||
T (int16_t, int8_t, 7) \
|
||||
T (int32_t, int8_t, 3) \
|
||||
T (int32_t, int16_t, 3) \
|
||||
T (int64_t, int8_t, 5) \
|
||||
T (int64_t, int16_t, 5) \
|
||||
T (int64_t, int32_t, 5)
|
||||
|
||||
TEST_ALL (DEF_LOOP)
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, #4\n} 3 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #4\n} 2 } } */
|
||||
/* { dg-final { scan-assembler-times {\tasrd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #4\n} 1 } } */
|
||||
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.b, p[0-7]/z, z[0-9]+\.b\n} 3 { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 { xfail *-*-* } } } */
|
||||
|
||||
/* { dg-final { scan-assembler-not {\tmov\tz} { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-assembler-not {\tsel\t} { xfail *-*-* } } } */
|
||||
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_3_run.c
Normal file
26
gcc/testsuite/gcc.target/aarch64/sve/cond_asrd_3_run.c
Normal file
@ -0,0 +1,26 @@
|
||||
/* { dg-do run { target { aarch64_sve_hw } } } */
|
||||
/* { dg-options "-O2 -ftree-vectorize" } */
|
||||
|
||||
#include "cond_asrd_3.c"
|
||||
|
||||
#define TEST_LOOP(TYPE1, TYPE2, N) \
|
||||
{ \
|
||||
TYPE1 a[N]; \
|
||||
TYPE2 r[N], b[N]; \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
{ \
|
||||
a[i] = i % 3 < 2 ? 0 : i * 13; \
|
||||
b[i] = (i & 1 ? i : -i) * 17; \
|
||||
asm volatile ("" ::: "memory"); \
|
||||
} \
|
||||
test_##TYPE1##_##TYPE2 (r, a, b); \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
if (r[i] != (TYPE2) (a[i] == 0 ? b[i] / 16 : 0)) \
|
||||
__builtin_abort (); \
|
||||
}
|
||||
|
||||
int main ()
|
||||
{
|
||||
TEST_ALL (TEST_LOOP)
|
||||
return 0;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user