diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 01b084d8ccb..fd0c5e612b5 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -3461,7 +3461,7 @@ [(set_attr "type" "neon_reduc_add")] ) -(define_expand "aarch64_addlp" +(define_expand "@aarch64_addlp" [(set (match_operand: 0 "register_operand") (plus: (vec_select: @@ -3517,6 +3517,45 @@ [(set_attr "type" "neon_cnt")] ) +(define_expand "popcount2" + [(set (match_operand:VDQHSD 0 "register_operand") + (popcount:VDQHSD (match_operand:VDQHSD 1 "register_operand")))] + "TARGET_SIMD" + { + /* Generate a byte popcount. */ + machine_mode mode = == 64 ? V8QImode : V16QImode; + rtx tmp = gen_reg_rtx (mode); + auto icode = optab_handler (popcount_optab, mode); + emit_insn (GEN_FCN (icode) (tmp, gen_lowpart (mode, operands[1]))); + + if (TARGET_DOTPROD + && (mode == SImode || mode == DImode)) + { + /* For V4SI and V2SI, we can generate a UDOT with a 0 accumulator and a + 1 multiplicand. For V2DI, another UAADDLP is needed. */ + rtx ones = force_reg (mode, CONST1_RTX (mode)); + auto icode = optab_handler (udot_prod_optab, mode); + mode = == 64 ? V2SImode : V4SImode; + rtx dest = mode == mode ? operands[0] : gen_reg_rtx (mode); + rtx zeros = force_reg (mode, CONST0_RTX (mode)); + emit_insn (GEN_FCN (icode) (dest, tmp, ones, zeros)); + tmp = dest; + } + + /* Use a sequence of UADDLPs to accumulate the counts. Each step doubles + the element size and halves the number of elements. */ + while (mode != mode) + { + auto icode = code_for_aarch64_addlp (ZERO_EXTEND, GET_MODE (tmp)); + mode = insn_data[icode].operand[0].mode; + rtx dest = mode == mode ? operands[0] : gen_reg_rtx (mode); + emit_insn (GEN_FCN (icode) (dest, tmp)); + tmp = dest; + } + DONE; + } +) + ;; 'across lanes' max and min ops. ;; Template for outputting a scalar, so we can create __builtins which can be diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt-udot.c b/gcc/testsuite/gcc.target/aarch64/popcnt-udot.c new file mode 100644 index 00000000000..f6a968dae95 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/popcnt-udot.c @@ -0,0 +1,58 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8.2-a+dotprod -fno-vect-cost-model -fno-schedule-insns -fno-schedule-insns2" } */ + +/* +** bar: +** movi v([0-9]+).16b, 0x1 +** movi v([0-9]+).4s, 0 +** ldr q([0-9]+), \[x0\] +** cnt v([0-9]+).16b, v\3.16b +** udot v\2.4s, v\4.16b, v\1.16b +** str q\2, \[x1\] +** ret +*/ +void +bar (unsigned int *__restrict b, unsigned int *__restrict d) +{ + d[0] = __builtin_popcount (b[0]); + d[1] = __builtin_popcount (b[1]); + d[2] = __builtin_popcount (b[2]); + d[3] = __builtin_popcount (b[3]); +} + +/* +** bar1: +** movi v([0-9]+).8b, 0x1 +** movi v([0-9]+).2s, 0 +** ldr d([0-9]+), \[x0\] +** cnt v([0-9]+).8b, v\3.8b +** udot v\2.2s, v\4.8b, v\1.8b +** str d\2, \[x1\] +** ret +*/ +void +bar1 (unsigned int *__restrict b, unsigned int *__restrict d) +{ + d[0] = __builtin_popcount (b[0]); + d[1] = __builtin_popcount (b[1]); +} + +/* +** bar2: +** movi v([0-9]+).16b, 0x1 +** movi v([0-9]+).4s, 0 +** ldr q([0-9]+), \[x0\] +** cnt v([0-9]+).16b, v\3.16b +** udot v\2.4s, v\4.16b, v\1.16b +** uaddlp v\2.2d, v\2.4s +** str q\2, \[x1\] +** ret +*/ +void +bar2 (unsigned long long *__restrict b, unsigned long long *__restrict d) +{ + d[0] = __builtin_popcountll (b[0]); + d[1] = __builtin_popcountll (b[1]); +} + +/* { dg-final { check-function-bodies "**" "" "" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c b/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c new file mode 100644 index 00000000000..b3cb8def2d5 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/popcnt-vec.c @@ -0,0 +1,69 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fno-vect-cost-model" } */ + +/* This function should produce cnt v.16b. */ +void +bar (unsigned char *__restrict b, unsigned char *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and uaddlp (Add Long Pairwise). */ +void +bar1 (unsigned short *__restrict b, unsigned short *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and 2 uaddlp (Add Long Pairwise). */ +void +bar2 (unsigned int *__restrict b, unsigned int *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcount (b[i]); +} + +/* This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */ +void +bar3 (unsigned long long *__restrict b, unsigned long long *__restrict d) +{ + for (int i = 0; i < 1024; i++) + d[i] = __builtin_popcountll (b[i]); +} + +/* SLP + This function should produce cnt v.8b and uaddlp (Add Long Pairwise). */ +void +bar4 (unsigned short *__restrict b, unsigned short *__restrict d) +{ + d[0] = __builtin_popcount (b[0]); + d[1] = __builtin_popcount (b[1]); + d[2] = __builtin_popcount (b[2]); + d[3] = __builtin_popcount (b[3]); +} + +/* SLP + This function should produce cnt v.8b and 2 uaddlp (Add Long Pairwise). */ +void +bar5 (unsigned int *__restrict b, unsigned int *__restrict d) +{ + d[0] = __builtin_popcount (b[0]); + d[1] = __builtin_popcount (b[1]); +} + +/* SLP + This function should produce cnt v.16b and 3 uaddlp (Add Long Pairwise). */ +void +bar6 (unsigned long long *__restrict b, unsigned long long *__restrict d) +{ + d[0] = __builtin_popcountll (b[0]); + d[1] = __builtin_popcountll (b[1]); +} + +/* { dg-final { scan-assembler-not {\tbl\tpopcount} } } */ +/* { dg-final { scan-assembler-times {cnt\t} 7 } } */ +/* { dg-final { scan-assembler-times {uaddlp\t} 12 } } */ +/* { dg-final { scan-assembler-times {ldr\tq} 5 } } */ +/* { dg-final { scan-assembler-times {ldr\td} 2 } } */