RISC-V: Enable SLP un-order reduction

This patch is to enable SLP un-order reduction autao-vectorization

Consider this following case:

int __attribute__((noipa))
add_loop (int *x, int n, int res)
{
  for (int i = 0; i < n; ++i)
    {
      res += x[i * 2];
      res += x[i * 2 + 1];
    }
  return res;
}

--param riscv-autovec-preference=scalable -fopt-info-vec-missed:
<source>:4:21: missed: couldn't vectorize loop
<source>:4:21: missed: unsupported SLP instances

After this patch:

add_loop:
	ble	a1,zero,.L5
	csrr	a6,vlenb
	srli	a4,a6,2
	slli	a1,a1,1
	neg	a7,a4
	vsetvli	t1,zero,e32,m1,ta,ma
	vmv.v.i	v2,0
	vslide1up.vx	v1,v2,a2   -----------> generated by VEC_SHL_INSERT
.L4:
	mv	a3,a1
	mv	a5,a1
	bleu	a1,a4,.L3
	mv	a5,a4
.L3:
	vsetvli	zero,a5,e32,m1,tu,ma
	add	a1,a1,a7
	vle32.v	v2,0(a0)
	add	a0,a0,a6
	vadd.vv	v1,v1,v2
	bgtu	a3,a4,.L4
	vsetivli	zero,1,e32,m1,ta,ma
	vmv.v.i	v2,0
	vsetvli	t1,zero,e32,m1,ta,ma
	vredsum.vs	v1,v1,v2
	vmv.x.s	a0,v1
	ret
.L5:
	mv	a0,a2
	ret

gcc/ChangeLog:

	* config/riscv/autovec.md (vec_shl_insert_<mode>): New patterns.
	* config/riscv/riscv-v.cc (shuffle_compress_patterns): Fix bugs.

gcc/testsuite/ChangeLog:

	* gcc.target/riscv/rvv/autovec/reduc/reduc-5.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc-6.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc-7.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc-8.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc-9.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc_run-5.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc_run-6.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc_run-7.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc_run-8.c: New test.
	* gcc.target/riscv/rvv/autovec/reduc/reduc_run-9.c: New test.
This commit is contained in:
Juzhe-Zhong 2023-07-18 16:44:11 +08:00 committed by Lehua Ding
parent 3a407070b6
commit 48b7404707
12 changed files with 542 additions and 0 deletions

View file

@ -372,6 +372,31 @@
}
)
;; Slide an RVV vector left and insert a scalar into element 0.
(define_expand "vec_shl_insert_<mode>"
[(match_operand:VI 0 "register_operand")
(match_operand:VI 1 "register_operand")
(match_operand:<VEL> 2 "reg_or_0_operand")]
"TARGET_VECTOR"
{
insn_code icode = code_for_pred_slide (UNSPEC_VSLIDE1UP, <MODE>mode);
rtx ops[] = {operands[0], RVV_VUNDEF (<MODE>mode), operands[1], operands[2]};
riscv_vector::emit_vlmax_slide_insn (icode, ops);
DONE;
})
(define_expand "vec_shl_insert_<mode>"
[(match_operand:VF 0 "register_operand")
(match_operand:VF 1 "register_operand")
(match_operand:<VEL> 2 "register_operand")]
"TARGET_VECTOR"
{
insn_code icode = code_for_pred_slide (UNSPEC_VFSLIDE1UP, <MODE>mode);
rtx ops[] = {operands[0], RVV_VUNDEF (<MODE>mode), operands[1], operands[2]};
riscv_vector::emit_vlmax_slide_insn (icode, ops);
DONE;
})
;; ========================================================================
;; == Vector operations
;; =========================================================================

View file

@ -2823,6 +2823,10 @@ shuffle_compress_patterns (struct expand_vec_perm_d *d)
if (known_le (d->perm[i], d->perm[i - 1]))
return false;
/* Success! */
if (d->testing_p)
return true;
/* Check whether we need to slideup op1 to apply compress approach.
E.g. For index = { 0, 2, 6, 7}, since d->perm[i - 1] = 7 which

View file

@ -0,0 +1,88 @@
/* { dg-do compile } */
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param=riscv-autovec-preference=fixed-vlmax -ffast-math -fno-vect-cost-model" } */
#include <stdint-gcc.h>
void __attribute__((noipa))
add_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 += x[i * 2];
res1 += x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
min_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 = res0 < x[i * 2] ? res0 : x[i * 2];
res1 = res1 < x[i * 2 + 1] ? res1 : x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
max_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 = res0 > x[i * 2] ? res0 : x[i * 2];
res1 = res1 > x[i * 2 + 1] ? res1 : x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
and_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 &= x[i * 2];
res1 &= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
or_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 |= x[i * 2];
res1 |= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
eor_loop (unsigned int *x, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < 0x7ff; ++i)
{
res0 ^= x[i * 2];
res1 ^= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}

View file

@ -0,0 +1,6 @@
/* { dg-do compile } */
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model -fdump-tree-optimized-details" } */
#include "reduc-5.c"
/* { dg-final { scan-tree-dump-times "VEC_SHL_INSERT" 8 "optimized" } } */

View file

@ -0,0 +1,88 @@
/* { dg-do compile } */
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model -fdump-tree-optimized-details" } */
void __attribute__((noipa))
add_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 += x[i * 2];
res1 += x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
min_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 = res0 < x[i * 2] ? res0 : x[i * 2];
res1 = res1 < x[i * 2 + 1] ? res1 : x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
max_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 = res0 > x[i * 2] ? res0 : x[i * 2];
res1 = res1 > x[i * 2 + 1] ? res1 : x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
and_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 &= x[i * 2];
res1 &= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
or_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 |= x[i * 2];
res1 |= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
void __attribute__((noipa))
eor_loop (unsigned int *x, int n, unsigned int *res)
{
unsigned int res0 = res[0];
unsigned int res1 = res[1];
for (int i = 0; i < n; ++i)
{
res0 ^= x[i * 2];
res1 ^= x[i * 2 + 1];
}
res[0] = res0;
res[1] = res1;
}
/* { dg-final { scan-tree-dump-times "VEC_SHL_INSERT" 8 "optimized" } } */

View file

@ -0,0 +1,16 @@
/* { dg-do compile } */
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model -fdump-tree-optimized-details" } */
int __attribute__((noipa))
add_loop (int *x, int n, int res)
{
for (int i = 0; i < n; ++i)
{
res += x[i * 2];
res += x[i * 2 + 1];
}
return res;
}
/* { dg-final { scan-tree-dump-times "VEC_SHL_INSERT" 1 "optimized" } } */
/* { dg-final { scan-assembler-times {vslide1up\.vx\s+v[0-9]+,\s*v[0-9]+,\s*[a-x0-9]+} 1 } } */

View file

@ -0,0 +1,16 @@
/* { dg-do compile } */
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model -fdump-tree-optimized-details" } */
float __attribute__((noipa))
add_loop (float *x, int n, float res)
{
for (int i = 0; i < n; ++i)
{
res += x[i * 2];
res += x[i * 2 + 1];
}
return res;
}
/* { dg-final { scan-tree-dump-times "VEC_SHL_INSERT" 1 "optimized" } } */
/* { dg-final { scan-assembler-times {vfslide1up\.vf\s+v[0-9]+,\s*v[0-9]+,\s*[a-x0-9]+} 1 } } */

View file

@ -0,0 +1,61 @@
/* { dg-do run { target { riscv_vector } } } */
/* { dg-additional-options "--param=riscv-autovec-preference=fixed-vlmax -ffast-math -fno-vect-cost-model" } */
#define N 0x1100
#include "reduc-5.c"
int
main (void)
{
unsigned int x[N];
for (int i = 0; i < N; ++i)
x[i] = ((i + 1) * (i + 2)) & 0xfffff;
unsigned int add_res[2] = { 42, 1111 };
add_loop (x, add_res);
if (add_res[0] != 968538154
|| add_res[1] != 964340823)
__builtin_abort ();
unsigned int max_res1[2] = { 0, 0 };
max_loop (x, max_res1);
if (max_res1[0] != 1048150
|| max_res1[1] != 1045506)
__builtin_abort ();
unsigned int max_res2[2] = { 1048151, 1045507 };
max_loop (x, max_res2);
if (max_res2[0] != 1048151
|| max_res2[1] != 1045507)
__builtin_abort ();
unsigned int or_res[2] = { 0x1000000, 0x2000000 };
or_loop (x, or_res);
if (or_res[0] != 0x10ffffe
|| or_res[1] != 0x20ffffe)
__builtin_abort ();
unsigned int eor_res[2] = { 0x1000000, 0x2000000 };
eor_loop (x, eor_res);
if (eor_res[0] != 0x1010000
|| eor_res[1] != 0x20b5000)
__builtin_abort ();
for (int i = 0; i < N; ++i)
x[i] = ~x[i] & 0xfffff;
unsigned int min_res1[2] = { 500, 4000 };
min_loop (x, min_res1);
if (min_res1[0] != 425
|| min_res1[1] != 3069)
__builtin_abort ();
unsigned int min_res2[2] = { 424, 3068 };
min_loop (x, min_res2);
if (min_res2[0] != 424
|| min_res2[1] != 3068)
__builtin_abort ();
return 0;
}

View file

@ -0,0 +1,6 @@
/* { dg-do run { target { riscv_vector } } } */
/* { dg-additional-options "--param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model" } */
#define N 0x1100
#include "reduc_run-5.c"

View file

@ -0,0 +1,188 @@
/* { dg-do run { target { riscv_vector } } } */
/* { dg-additional-options "--param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model" } */
#define N 0x1100
#include "reduc-7.c"
int
main (void)
{
unsigned int x[N];
for (int i = 0; i < N; ++i)
x[i] = ((i + 1) * (i + 2)) & 0xfffff;
unsigned int add_res1[2] = { 11, 22 };
add_loop (x, 0, add_res1);
if (add_res1[0] != 11
|| add_res1[1] != 22)
__builtin_abort ();
unsigned int add_res2[2] = { 10, 20 };
add_loop (x, 11, add_res2);
if (add_res2[0] != 1902
|| add_res2[1] != 2176)
__builtin_abort ();
unsigned int add_res3[2] = { 15, 30 };
add_loop (x, 0x100, add_res3);
if (add_res3[0] != 22435087
|| add_res3[1] != 22566686)
__builtin_abort ();
unsigned int add_res4[2] = { 100, 200 };
add_loop (x, 0x11f, add_res4);
if (add_res4[0] != 31602244
|| add_res4[1] != 31767656)
__builtin_abort ();
unsigned int max_res1[2] = { 461, 500 };
max_loop (x, 11, max_res1);
if (max_res1[0] != 462
|| max_res1[1] != 506)
__builtin_abort ();
unsigned int max_res2[2] = { 463, 507 };
max_loop (x, 11, max_res2);
if (max_res2[0] != 463
|| max_res2[1] != 507)
__builtin_abort ();
unsigned int max_res3[2] = { 1000000, 1000000 };
max_loop (x, 0x200, max_res3);
if (max_res3[0] != 1047552
|| max_res3[1] != 1045506)
__builtin_abort ();
unsigned int max_res4[2] = { 1047553, 1045507 };
max_loop (x, 0x200, max_res4);
if (max_res4[0] != 1047553
|| max_res4[1] != 1045507)
__builtin_abort ();
unsigned int max_res5[2] = { 300000, 30000 };
max_loop (x, 0x11f, max_res5);
if (max_res5[0] != 328902
|| max_res5[1] != 330050)
__builtin_abort ();
unsigned int max_res6[2] = { 328903, 330051 };
max_loop (x, 0x11f, max_res6);
if (max_res6[0] != 328903
|| max_res6[1] != 330051)
__builtin_abort ();
unsigned int or_res1[2] = { 11, 22 };
or_loop (x, 0, or_res1);
if (or_res1[0] != 11
|| or_res1[1] != 22)
__builtin_abort ();
unsigned int or_res2[2] = { 0x200000, 0xe00000 };
or_loop (x, 11, or_res2);
if (or_res2[0] != 0x2001fe
|| or_res2[1] != 0xe001fe)
__builtin_abort ();
unsigned int or_res3[2] = { 0x800000, 0x700000 };
or_loop (x, 0x40, or_res3);
if (or_res3[0] != 0x803ffe
|| or_res3[1] != 0x707ffe)
__builtin_abort ();
unsigned int or_res4[2] = { 0x100001, 0x300000 };
or_loop (x, 0x4f, or_res4);
if (or_res4[0] != 0x107fff
|| or_res4[1] != 0x307ffe)
__builtin_abort ();
unsigned int eor_res1[2] = { 11, 22 };
eor_loop (x, 0, eor_res1);
if (eor_res1[0] != 11
|| eor_res1[1] != 22)
__builtin_abort ();
unsigned int eor_res2[2] = { 0x2000ff, 0xe000ff };
eor_loop (x, 11, eor_res2);
if (eor_res2[0] != 0x2001cf
|| eor_res2[1] != 0xe000b7)
__builtin_abort ();
unsigned int eor_res3[2] = { 0x805000, 0x70f000 };
eor_loop (x, 0x100, eor_res3);
if (eor_res3[0] != 0x824200
|| eor_res3[1] != 0x77dc00)
__builtin_abort ();
unsigned int eor_res4[2] = { 0x101201, 0x300f00 };
eor_loop (x, 0x11f, eor_res4);
if (eor_res4[0] != 0x178801
|| eor_res4[1] != 0x337240)
__builtin_abort ();
for (int i = 0; i < N; ++i)
x[i] = ~x[i] & 0xfffff;
unsigned int min_res1[2] = { 1048200, 1048100 };
min_loop (x, 11, min_res1);
if (min_res1[0] != 1048113
|| min_res1[1] != 1048069)
__builtin_abort ();
unsigned int min_res2[2] = { 1048112, 1048068 };
min_loop (x, 11, min_res2);
if (min_res2[0] != 1048112
|| min_res2[1] != 1048068)
__builtin_abort ();
unsigned int min_res3[2] = { 10000, 10000 };
min_loop (x, 0x200, min_res3);
if (min_res3[0] != 1023
|| min_res3[1] != 3069)
__builtin_abort ();
unsigned int min_res4[2] = { 1022, 3068 };
min_loop (x, 0x200, min_res4);
if (min_res4[0] != 1022
|| min_res4[1] != 3068)
__builtin_abort ();
unsigned int min_res5[2] = { 719680, 718530 };
min_loop (x, 0x11f, min_res5);
if (min_res5[0] != 719673
|| min_res5[1] != 718525)
__builtin_abort ();
unsigned int min_res6[2] = { 719672, 718524 };
min_loop (x, 0x11f, min_res6);
if (min_res6[0] != 719672
|| min_res6[1] != 718524)
__builtin_abort ();
unsigned int and_res1[2] = { 11, 22 };
and_loop (x, 0, and_res1);
if (and_res1[0] != 11
|| and_res1[1] != 22)
__builtin_abort ();
unsigned int and_res2[2] = { 0xf5cff, 0xf78ff };
and_loop (x, 11, and_res2);
if (and_res2[0] != 0xf5c01
|| and_res2[1] != 0xf7801)
__builtin_abort ();
unsigned int and_res3[2] = { 0x7efff, 0xecfff };
and_loop (x, 0x40, and_res3);
if (and_res3[0] != 0x7c001
|| and_res3[1] != 0xe8001)
__builtin_abort ();
unsigned int and_res4[2] = { 0xffffff, 0xffffff };
and_loop (x, 0x4f, and_res4);
if (and_res4[0] != 0xf8001
|| and_res4[1] != 0xf8001)
__builtin_abort ();
return 0;
}

View file

@ -0,0 +1,22 @@
/* { dg-do run { target { riscv_vector } } } */
/* { dg-additional-options "--param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model" } */
#define N 0x1100
#include "reduc-8.c"
int
main (void)
{
int x[N];
for (int i = 0; i < N; ++i)
x[i] = ((i + 1) * (i + 2)) & 0xfffff;
if (add_loop (x, 0, 33) != 33
|| add_loop (x, 11, 30) != 4078
|| add_loop (x, 0x100, 45) != 45001773
|| add_loop (x, 0x11f, 300) != 63369900)
__builtin_abort ();
return 0;
}

View file

@ -0,0 +1,22 @@
/* { dg-do run { target { riscv_vector } } } */
/* { dg-additional-options "--param=riscv-autovec-preference=scalable -ffast-math -fno-vect-cost-model" } */
#define N 0x1100
#include "reduc-9.c"
int
main (void)
{
float x[N];
for (int i = 0; i < N; ++i)
x[i] = ((i + 1) * (i + 2)) & 0xfffff;
if (add_loop (x, 0, 33) != 33
|| add_loop (x, 11, 30) != 4078
|| add_loop (x, 0x100, 45) != 45001773
|| add_loop (x, 0x11f, 300) != 63369900)
__builtin_abort ();
return 0;
}