procyberian/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

aarch64: Add vector popcount besides QImode [PR113859]

Browse source 
This patch improves GCC’s vectorization of __builtin_popcount for aarch64 target
by adding popcount patterns for vector modes besides QImode, i.e., HImode,
SImode and DImode.

With this patch, we now generate the following for V8HI:
  cnt     v1.16b, v0.16b
  uaddlp  v2.8h, v1.16b

For V4HI, we generate:
  cnt     v1.8b, v0.8b
  uaddlp  v2.4h, v1.8b

For V4SI, we generate:
  cnt     v1.16b, v0.16b
  uaddlp  v2.8h, v1.16b
  uaddlp  v3.4s, v2.8h

For V4SI with TARGET_DOTPROD, we generate the following instead:
  movi    v0.4s, #0
  movi    v1.16b, #1
  cnt     v3.16b, v2.16b
  udot    v0.4s, v3.16b, v1.16b

For V2SI, we generate:
  cnt     v1.8b, v.8b
  uaddlp  v2.4h, v1.8b
  uaddlp  v3.2s, v2.4h

For V2SI with TARGET_DOTPROD, we generate the following instead:
  movi    v0.8b, #0
  movi    v1.8b, #1
  cnt     v3.8b, v2.8b
  udot    v0.2s, v3.8b, v1.8b

For V2DI, we generate:
  cnt     v1.16b, v.16b
  uaddlp  v2.8h, v1.16b
  uaddlp  v3.4s, v2.8h
  uaddlp  v4.2d, v3.4s

For V4SI with TARGET_DOTPROD, we generate the following instead:
  movi    v0.4s, #0
  movi    v1.16b, #1
  cnt     v3.16b, v2.16b
  udot    v0.4s, v3.16b, v1.16b
  uaddlp  v0.2d, v0.4s

	PR target/113859

gcc/ChangeLog:

	* config/aarch64/aarch64-simd.md (aarch64_<su>addlp<mode>): Rename to...
	(@aarch64_<su>addlp<mode>): ... This.
	(popcount<mode>2): New define_expand.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/popcnt-udot.c: New test.
	* gcc.target/aarch64/popcnt-vec.c: New test.

Signed-off-by: Pengxuan Zheng <quic_pzheng@quicinc.com>
This commit is contained in:
Pengxuan Zheng 2024-06-12 18:23:13 -07:00
parent a7ad9cb813
commit 895bbc08d3
3 changed files with 167 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
gcc/config/aarch64/aarch64-simd.md
View file

@ -3461,7 +3461,7 @@
[(set_attr "type" "neon_reduc_add<VDQV_L:q>")]
)
(define_expand "aarch64_<su>addlp<mode>"
(define_expand "@aarch64_<su>addlp<mode>"
[(set (match_operand:<VDBLW> 0 "register_operand")
(plus:<VDBLW>
(vec_select:<VDBLW>
@ -3517,6 +3517,45 @@
[(set_attr "type" "neon_cnt<q>")]
)
(define_expand "popcount<mode>2"
[(set (match_operand:VDQHSD 0 "register_operand")
(popcount:VDQHSD (match_operand:VDQHSD 1 "register_operand")))]
"TARGET_SIMD"
{
/* Generate a byte popcount. */
machine_mode mode = <bitsize> == 64 ? V8QImode : V16QImode;
rtx tmp = gen_reg_rtx (mode);
auto icode = optab_handler (popcount_optab, mode);
emit_insn (GEN_FCN (icode) (tmp, gen_lowpart (mode, operands[1])));
if (TARGET_DOTPROD
&& (<VEL>mode == SImode || <VEL>mode == DImode))
{
/* For V4SI and V2SI, we can generate a UDOT with a 0 accumulator and a
1 multiplicand. For V2DI, another UAADDLP is needed. */
rtx ones = force_reg (mode, CONST1_RTX (mode));
auto icode = optab_handler (udot_prod_optab, mode);
mode = <bitsize> == 64 ? V2SImode : V4SImode;
rtx dest = mode == <MODE>mode ? operands[0] : gen_reg_rtx (mode);
rtx zeros = force_reg (mode, CONST0_RTX (mode));
emit_insn (GEN_FCN (icode) (dest, tmp, ones, zeros));
tmp = dest;
}
/* Use a sequence of UADDLPs to accumulate the counts. Each step doubles
the element size and halves the number of elements. */
while (mode != <MODE>mode)
{
auto icode = code_for_aarch64_addlp (ZERO_EXTEND, GET_MODE (tmp));
mode = insn_data[icode].operand[0].mode;
rtx dest = mode == <MODE>mode ? operands[0] : gen_reg_rtx (mode);
emit_insn (GEN_FCN (icode) (dest, tmp));
tmp = dest;
}
DONE;
}
)
;; 'across lanes' max and min ops.
;; Template for outputting a scalar, so we can create __builtins which can be

58
gcc/testsuite/gcc.target/aarch64/popcnt-udot.c Normal file
View file

@ -0,0 +1,58 @@
/* { dg-do compile } */
/* { dg-options "-O2 -march=armv8.2-a+dotprod -fno-vect-cost-model -fno-schedule-insns -fno-schedule-insns2" } */
/*
** bar:
** movi v([0-9]+).16b, 0x1
** movi v([0-9]+).4s, 0
** ldr q([0-9]+), \[x0\]
** cnt v([0-9]+).16b, v\3.16b
** udot v\2.4s, v\4.16b, v\1.16b
** str q\2, \[x1\]
** ret
*/
void
bar (unsigned int *__restrict b, unsigned int *__restrict d)
{
d[0] = __builtin_popcount (b[0]);
d[1] = __builtin_popcount (b[1]);
d[2] = __builtin_popcount (b[2]);
d[3] = __builtin_popcount (b[3]);
}
/*
** bar1:
** movi v([0-9]+).8b, 0x1
** movi v([0-9]+).2s, 0
** ldr d([0-9]+), \[x0\]
** cnt v([0-9]+).8b, v\3.8b
** udot v\2.2s, v\4.8b, v\1.8b
** str d\2, \[x1\]
** ret
*/
void
bar1 (unsigned int *__restrict b, unsigned int *__restrict d)
{
d[0] = __builtin_popcount (b[0]);
d[1] = __builtin_popcount (b[1]);
}
/*
** bar2:
** movi v([0-9]+).16b, 0x1
** movi v([0-9]+).4s, 0
** ldr q([0-9]+), \[x0\]
** cnt v([0-9]+).16b, v\3.16b
** udot v\2.4s, v\4.16b, v\1.16b
** uaddlp v\2.2d, v\2.4s
** str q\2, \[x1\]
** ret
*/
void
bar2 (unsigned long long *__restrict b, unsigned long long *__restrict d)
{
d[0] = __builtin_popcountll (b[0]);
d[1] = __builtin_popcountll (b[1]);
}
/* { dg-final { check-function-bodies "**" "" "" } } */

69
gcc/testsuite/gcc.target/aarch64/popcnt-vec.c Normal file
View file

@ -0,0 +1,69 @@
/* { dg-do compile } */
/* { dg-options "-O2 -fno-vect-cost-model" } */
/* This function should produce cnt v.16b. */
void
bar (unsigned char *__restrict b, unsigned char *__restrict d)
{
for (int i = 0; i < 1024; i++)
d[i] = __builtin_popcount (b[i]);
}
/* This function should produce cnt v.16b and uaddlp (Add Long Pairwise). */
void
bar1 (unsigned short *__restrict b, unsigned short *__restrict d)
{
for (int i = 0; i < 1024; i++)
d[i] = __builtin_popcount (b[i]);
}
/* This function should produce cnt v.16b and 2 uaddlp (Add Long Pairwise). */
void
bar2 (unsigned int *__restrict b, unsigned int *__restrict d)
{
for (int i = 0; i < 1024; i++)
d[i] = __builtin_popcount (b[i]);
}
/* This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */
void
bar3 (unsigned long long *__restrict b, unsigned long long *__restrict d)
{
for (int i = 0; i < 1024; i++)
d[i] = __builtin_popcountll (b[i]);
}
/* SLP
This function should produce cnt v.8b and uaddlp (Add Long Pairwise). */
void
bar4 (unsigned short *__restrict b, unsigned short *__restrict d)
{
d[0] = __builtin_popcount (b[0]);
d[1] = __builtin_popcount (b[1]);
d[2] = __builtin_popcount (b[2]);
d[3] = __builtin_popcount (b[3]);
}
/* SLP
This function should produce cnt v.8b and 2 uaddlp (Add Long Pairwise). */
void
bar5 (unsigned int *__restrict b, unsigned int *__restrict d)
{
d[0] = __builtin_popcount (b[0]);
d[1] = __builtin_popcount (b[1]);
}
/* SLP
This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */
void
bar6 (unsigned long long *__restrict b, unsigned long long *__restrict d)
{
d[0] = __builtin_popcountll (b[0]);
d[1] = __builtin_popcountll (b[1]);
}
/* { dg-final { scan-assembler-not {\tbl\tpopcount} } } */
/* { dg-final { scan-assembler-times {cnt\t} 7 } } */
/* { dg-final { scan-assembler-times {uaddlp\t} 12 } } */
/* { dg-final { scan-assembler-times {ldr\tq} 5 } } */
/* { dg-final { scan-assembler-times {ldr\td} 2 } } */