i386: prefer vpermilpd over vpermpd [PR93395]
In Agner Fog's tables, vpermilp[sd] with immediates seem to be much faster than vpermpd with immediate, for a good reason, the former only permute something within the lanes and don't do anything intra-lane, while vpermpd can. So, functionality-wise, vpermilpd is more efficient subset of vpermpd. We use the same RTL for those though (and also for certain broadcast). Now, the problem was that the vpermpd pattern appeared first in sse.md, followed by the broadcast patterns, followed by the vpermilp[sd]. Which means unless -mavx -mno-avx2, we'd emit vpermpd instead of the more efficient alternatives. The following patch reorders them, so that vpermpd comes last, if we can match a broadcast, we do, if we can match a vpermilp[sd] that is not a broadcast, we will, otherwise fall back (of course only if -mavx2) to vpermpd. 2020-01-24 Jakub Jelinek <jakub@redhat.com> PR target/93395 * config/i386/sse.md (*avx_vperm_broadcast_v4sf, *avx_vperm_broadcast_<mode>, <sse2_avx_avx512f>_vpermil<mode><mask_name>, *<sse2_avx_avx512f>_vpermilp<mode><mask_name>): Move before avx2_perm<mode>/avx512f_perm<mode>. * gcc.target/i386/pr93395.c: New test. * gcc.target/i386/avx512vl-vpermilpdi-1.c: Remove xfail.
This commit is contained in:
Jakub Jelinek
parent
14e5881e37
commit
5d782a8d90
5 changed files with 217 additions and 156 deletions
|
|
@ -1,5 +1,12 @@
|
|||
2020-01-24 Jakub Jelinek <jakub@redhat.com>
|
||||
|
||||
PR target/93395
|
||||
* config/i386/sse.md (*avx_vperm_broadcast_v4sf,
|
||||
*avx_vperm_broadcast_<mode>,
|
||||
<sse2_avx_avx512f>_vpermil<mode><mask_name>,
|
||||
*<sse2_avx_avx512f>_vpermilp<mode><mask_name>):
|
||||
Move before avx2_perm<mode>/avx512f_perm<mode>.
|
||||
|
||||
PR target/93376
|
||||
* simplify-rtx.c (simplify_const_unary_operation,
|
||||
simplify_const_binary_operation): Punt for mode precision above
|
||||
|
|
|
|||
|
|
@ -19875,6 +19875,164 @@
|
|||
(set_attr "prefix" "<mask_prefix2>")
|
||||
(set_attr "mode" "<sseinsnmode>")])
|
||||
|
||||
;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
|
||||
;; If it so happens that the input is in memory, use vbroadcast.
|
||||
;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
|
||||
(define_insn "*avx_vperm_broadcast_v4sf"
|
||||
[(set (match_operand:V4SF 0 "register_operand" "=v,v,v")
|
||||
(vec_select:V4SF
|
||||
(match_operand:V4SF 1 "nonimmediate_operand" "m,o,v")
|
||||
(match_parallel 2 "avx_vbroadcast_operand"
|
||||
[(match_operand 3 "const_int_operand" "C,n,n")])))]
|
||||
"TARGET_AVX"
|
||||
{
|
||||
int elt = INTVAL (operands[3]);
|
||||
switch (which_alternative)
|
||||
{
|
||||
case 0:
|
||||
case 1:
|
||||
operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
|
||||
return "vbroadcastss\t{%1, %0|%0, %k1}";
|
||||
case 2:
|
||||
operands[2] = GEN_INT (elt * 0x55);
|
||||
return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
|
||||
default:
|
||||
gcc_unreachable ();
|
||||
}
|
||||
}
|
||||
[(set_attr "type" "ssemov,ssemov,sselog1")
|
||||
(set_attr "prefix_extra" "1")
|
||||
(set_attr "length_immediate" "0,0,1")
|
||||
(set_attr "prefix" "maybe_evex")
|
||||
(set_attr "mode" "SF,SF,V4SF")])
|
||||
|
||||
(define_insn_and_split "*avx_vperm_broadcast_<mode>"
|
||||
[(set (match_operand:VF_256 0 "register_operand" "=v,v,v")
|
||||
(vec_select:VF_256
|
||||
(match_operand:VF_256 1 "nonimmediate_operand" "m,o,?v")
|
||||
(match_parallel 2 "avx_vbroadcast_operand"
|
||||
[(match_operand 3 "const_int_operand" "C,n,n")])))]
|
||||
"TARGET_AVX"
|
||||
"#"
|
||||
"&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
|
||||
[(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
|
||||
{
|
||||
rtx op0 = operands[0], op1 = operands[1];
|
||||
int elt = INTVAL (operands[3]);
|
||||
|
||||
if (REG_P (op1))
|
||||
{
|
||||
int mask;
|
||||
|
||||
if (TARGET_AVX2 && elt == 0)
|
||||
{
|
||||
emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
|
||||
op1)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
/* Shuffle element we care about into all elements of the 128-bit lane.
|
||||
The other lane gets shuffled too, but we don't care. */
|
||||
if (<MODE>mode == V4DFmode)
|
||||
mask = (elt & 1 ? 15 : 0);
|
||||
else
|
||||
mask = (elt & 3) * 0x55;
|
||||
emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
|
||||
|
||||
/* Shuffle the lane we care about into both lanes of the dest. */
|
||||
mask = (elt / (<ssescalarnum> / 2)) * 0x11;
|
||||
if (EXT_REX_SSE_REG_P (op0))
|
||||
{
|
||||
/* There is no EVEX VPERM2F128, but we can use either VBROADCASTSS
|
||||
or VSHUFF128. */
|
||||
gcc_assert (<MODE>mode == V8SFmode);
|
||||
if ((mask & 1) == 0)
|
||||
emit_insn (gen_avx2_vec_dupv8sf (op0,
|
||||
gen_lowpart (V4SFmode, op0)));
|
||||
else
|
||||
emit_insn (gen_avx512vl_shuf_f32x4_1 (op0, op0, op0,
|
||||
GEN_INT (4), GEN_INT (5),
|
||||
GEN_INT (6), GEN_INT (7),
|
||||
GEN_INT (12), GEN_INT (13),
|
||||
GEN_INT (14), GEN_INT (15)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
operands[1] = adjust_address (op1, <ssescalarmode>mode,
|
||||
elt * GET_MODE_SIZE (<ssescalarmode>mode));
|
||||
})
|
||||
|
||||
(define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
|
||||
[(set (match_operand:VF2 0 "register_operand")
|
||||
(vec_select:VF2
|
||||
(match_operand:VF2 1 "nonimmediate_operand")
|
||||
(match_operand:SI 2 "const_0_to_255_operand")))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>"
|
||||
{
|
||||
int mask = INTVAL (operands[2]);
|
||||
rtx perm[<ssescalarnum>];
|
||||
|
||||
int i;
|
||||
for (i = 0; i < <ssescalarnum>; i = i + 2)
|
||||
{
|
||||
perm[i] = GEN_INT (((mask >> i) & 1) + i);
|
||||
perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
|
||||
}
|
||||
|
||||
operands[2]
|
||||
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
|
||||
})
|
||||
|
||||
(define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
|
||||
[(set (match_operand:VF1 0 "register_operand")
|
||||
(vec_select:VF1
|
||||
(match_operand:VF1 1 "nonimmediate_operand")
|
||||
(match_operand:SI 2 "const_0_to_255_operand")))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>"
|
||||
{
|
||||
int mask = INTVAL (operands[2]);
|
||||
rtx perm[<ssescalarnum>];
|
||||
|
||||
int i;
|
||||
for (i = 0; i < <ssescalarnum>; i = i + 4)
|
||||
{
|
||||
perm[i] = GEN_INT (((mask >> 0) & 3) + i);
|
||||
perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
|
||||
perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
|
||||
perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
|
||||
}
|
||||
|
||||
operands[2]
|
||||
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
|
||||
})
|
||||
|
||||
;; This pattern needs to come before the avx2_perm*/avx512f_perm*
|
||||
;; patterns, as they have the same RTL representation (vpermilp*
|
||||
;; being a subset of what vpermp* can do), but vpermilp* has shorter
|
||||
;; latency as it never crosses lanes.
|
||||
(define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
|
||||
[(set (match_operand:VF 0 "register_operand" "=v")
|
||||
(vec_select:VF
|
||||
(match_operand:VF 1 "nonimmediate_operand" "vm")
|
||||
(match_parallel 2 ""
|
||||
[(match_operand 3 "const_int_operand")])))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>
|
||||
&& avx_vpermilp_parallel (operands[2], <MODE>mode)"
|
||||
{
|
||||
int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
|
||||
operands[2] = GEN_INT (mask);
|
||||
return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
|
||||
}
|
||||
[(set_attr "type" "sselog")
|
||||
(set_attr "prefix_extra" "1")
|
||||
(set_attr "length_immediate" "1")
|
||||
(set_attr "prefix" "<mask_prefix>")
|
||||
(set_attr "mode" "<sseinsnmode>")])
|
||||
|
||||
(define_expand "avx2_perm<mode>"
|
||||
[(match_operand:VI8F_256 0 "register_operand")
|
||||
(match_operand:VI8F_256 1 "nonimmediate_operand")
|
||||
|
|
@ -20376,160 +20534,6 @@
|
|||
(set_attr "prefix" "evex")
|
||||
(set_attr "mode" "XI")])
|
||||
|
||||
;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
|
||||
;; If it so happens that the input is in memory, use vbroadcast.
|
||||
;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
|
||||
(define_insn "*avx_vperm_broadcast_v4sf"
|
||||
[(set (match_operand:V4SF 0 "register_operand" "=v,v,v")
|
||||
(vec_select:V4SF
|
||||
(match_operand:V4SF 1 "nonimmediate_operand" "m,o,v")
|
||||
(match_parallel 2 "avx_vbroadcast_operand"
|
||||
[(match_operand 3 "const_int_operand" "C,n,n")])))]
|
||||
"TARGET_AVX"
|
||||
{
|
||||
int elt = INTVAL (operands[3]);
|
||||
switch (which_alternative)
|
||||
{
|
||||
case 0:
|
||||
case 1:
|
||||
operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
|
||||
return "vbroadcastss\t{%1, %0|%0, %k1}";
|
||||
case 2:
|
||||
operands[2] = GEN_INT (elt * 0x55);
|
||||
return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
|
||||
default:
|
||||
gcc_unreachable ();
|
||||
}
|
||||
}
|
||||
[(set_attr "type" "ssemov,ssemov,sselog1")
|
||||
(set_attr "prefix_extra" "1")
|
||||
(set_attr "length_immediate" "0,0,1")
|
||||
(set_attr "prefix" "maybe_evex")
|
||||
(set_attr "mode" "SF,SF,V4SF")])
|
||||
|
||||
(define_insn_and_split "*avx_vperm_broadcast_<mode>"
|
||||
[(set (match_operand:VF_256 0 "register_operand" "=v,v,v")
|
||||
(vec_select:VF_256
|
||||
(match_operand:VF_256 1 "nonimmediate_operand" "m,o,?v")
|
||||
(match_parallel 2 "avx_vbroadcast_operand"
|
||||
[(match_operand 3 "const_int_operand" "C,n,n")])))]
|
||||
"TARGET_AVX"
|
||||
"#"
|
||||
"&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
|
||||
[(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
|
||||
{
|
||||
rtx op0 = operands[0], op1 = operands[1];
|
||||
int elt = INTVAL (operands[3]);
|
||||
|
||||
if (REG_P (op1))
|
||||
{
|
||||
int mask;
|
||||
|
||||
if (TARGET_AVX2 && elt == 0)
|
||||
{
|
||||
emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
|
||||
op1)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
/* Shuffle element we care about into all elements of the 128-bit lane.
|
||||
The other lane gets shuffled too, but we don't care. */
|
||||
if (<MODE>mode == V4DFmode)
|
||||
mask = (elt & 1 ? 15 : 0);
|
||||
else
|
||||
mask = (elt & 3) * 0x55;
|
||||
emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
|
||||
|
||||
/* Shuffle the lane we care about into both lanes of the dest. */
|
||||
mask = (elt / (<ssescalarnum> / 2)) * 0x11;
|
||||
if (EXT_REX_SSE_REG_P (op0))
|
||||
{
|
||||
/* There is no EVEX VPERM2F128, but we can use either VBROADCASTSS
|
||||
or VSHUFF128. */
|
||||
gcc_assert (<MODE>mode == V8SFmode);
|
||||
if ((mask & 1) == 0)
|
||||
emit_insn (gen_avx2_vec_dupv8sf (op0,
|
||||
gen_lowpart (V4SFmode, op0)));
|
||||
else
|
||||
emit_insn (gen_avx512vl_shuf_f32x4_1 (op0, op0, op0,
|
||||
GEN_INT (4), GEN_INT (5),
|
||||
GEN_INT (6), GEN_INT (7),
|
||||
GEN_INT (12), GEN_INT (13),
|
||||
GEN_INT (14), GEN_INT (15)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
|
||||
DONE;
|
||||
}
|
||||
|
||||
operands[1] = adjust_address (op1, <ssescalarmode>mode,
|
||||
elt * GET_MODE_SIZE (<ssescalarmode>mode));
|
||||
})
|
||||
|
||||
(define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
|
||||
[(set (match_operand:VF2 0 "register_operand")
|
||||
(vec_select:VF2
|
||||
(match_operand:VF2 1 "nonimmediate_operand")
|
||||
(match_operand:SI 2 "const_0_to_255_operand")))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>"
|
||||
{
|
||||
int mask = INTVAL (operands[2]);
|
||||
rtx perm[<ssescalarnum>];
|
||||
|
||||
int i;
|
||||
for (i = 0; i < <ssescalarnum>; i = i + 2)
|
||||
{
|
||||
perm[i] = GEN_INT (((mask >> i) & 1) + i);
|
||||
perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
|
||||
}
|
||||
|
||||
operands[2]
|
||||
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
|
||||
})
|
||||
|
||||
(define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
|
||||
[(set (match_operand:VF1 0 "register_operand")
|
||||
(vec_select:VF1
|
||||
(match_operand:VF1 1 "nonimmediate_operand")
|
||||
(match_operand:SI 2 "const_0_to_255_operand")))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>"
|
||||
{
|
||||
int mask = INTVAL (operands[2]);
|
||||
rtx perm[<ssescalarnum>];
|
||||
|
||||
int i;
|
||||
for (i = 0; i < <ssescalarnum>; i = i + 4)
|
||||
{
|
||||
perm[i] = GEN_INT (((mask >> 0) & 3) + i);
|
||||
perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
|
||||
perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
|
||||
perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
|
||||
}
|
||||
|
||||
operands[2]
|
||||
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
|
||||
})
|
||||
|
||||
(define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
|
||||
[(set (match_operand:VF 0 "register_operand" "=v")
|
||||
(vec_select:VF
|
||||
(match_operand:VF 1 "nonimmediate_operand" "vm")
|
||||
(match_parallel 2 ""
|
||||
[(match_operand 3 "const_int_operand")])))]
|
||||
"TARGET_AVX && <mask_mode512bit_condition>
|
||||
&& avx_vpermilp_parallel (operands[2], <MODE>mode)"
|
||||
{
|
||||
int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
|
||||
operands[2] = GEN_INT (mask);
|
||||
return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
|
||||
}
|
||||
[(set_attr "type" "sselog")
|
||||
(set_attr "prefix_extra" "1")
|
||||
(set_attr "length_immediate" "1")
|
||||
(set_attr "prefix" "<mask_prefix>")
|
||||
(set_attr "mode" "<sseinsnmode>")])
|
||||
|
||||
(define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3<mask_name>"
|
||||
[(set (match_operand:VF 0 "register_operand" "=v")
|
||||
(unspec:VF
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
2020-01-24 Jakub Jelinek <jakub@redhat.com>
|
||||
|
||||
PR target/93395
|
||||
* gcc.target/i386/pr93395.c: New test.
|
||||
* gcc.target/i386/avx512vl-vpermilpdi-1.c: Remove xfail.
|
||||
|
||||
2020-01-24 Marek Polacek <polacek@redhat.com>
|
||||
|
||||
PR c++/93299 - ICE in tsubst_copy with parenthesized expression.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* { dg-do compile } */
|
||||
/* { dg-options "-mavx512vl -O2" } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*13\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*13\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*13\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*13\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*3\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\[ \\t\]+\[^\{\n\]*3\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
|
||||
|
||||
|
|
|
|||
44
gcc/testsuite/gcc.target/i386/pr93395.c
Normal file
44
gcc/testsuite/gcc.target/i386/pr93395.c
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/* PR target/93395 */
|
||||
/* { dg-do compile } */
|
||||
/* { dg-options "-O2 -mavx512f -masm=att" } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\t.5, %ymm" 3 } } */
|
||||
/* { dg-final { scan-assembler-times "vpermilpd\t.85, %zmm" 3 } } */
|
||||
/* { dg-final { scan-assembler-not "vpermpd\t" } } */
|
||||
|
||||
#include <immintrin.h>
|
||||
|
||||
__m256d
|
||||
foo1 (__m256d a)
|
||||
{
|
||||
return _mm256_permute4x64_pd (a, 177);
|
||||
}
|
||||
|
||||
__m256d
|
||||
foo2 (__m256d a)
|
||||
{
|
||||
return _mm256_permute_pd (a, 5);
|
||||
}
|
||||
|
||||
__m256d
|
||||
foo3 (__m256d a)
|
||||
{
|
||||
return __builtin_shuffle (a, (__v4di) { 1, 0, 3, 2 });
|
||||
}
|
||||
|
||||
__m512d
|
||||
foo4 (__m512d a)
|
||||
{
|
||||
return _mm512_permutex_pd (a, 177);
|
||||
}
|
||||
|
||||
__m512d
|
||||
foo5 (__m512d a)
|
||||
{
|
||||
return _mm512_permute_pd (a, 85);
|
||||
}
|
||||
|
||||
__m512d
|
||||
foo6 (__m512d a)
|
||||
{
|
||||
return __builtin_shuffle (a, (__v8di) { 1, 0, 3, 2, 5, 4, 7, 6 });
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue