tree-vect-loop-manip.c (vect_build_loop_niters, [...]): Move ...
2013-11-21 Richard Biener <rguenther@suse.de> * tree-vect-loop-manip.c (vect_build_loop_niters, vect_generate_tmps_on_preheader): Move ... * tree-vect-loop.c (vect_build_loop_niters, vect_generate_tmps_on_preheader): ... here and simplify. (vect_transform_loop): Call them here and pass down results to consumers. * tree-vect-loop-manip.c (vect_do_peeling_for_loop_bound): Get niter variables from caller. (vect_do_peeling_for_alignment): Likewise. * tree-vectorizer.h (vect_generate_tmps_on_preheader): Remove. (vect_do_peeling_for_loop_bound, vect_do_peeling_for_alignment): Adjust prototypes. From-SVN: r205199
This commit is contained in:
Richard Biener
Richard Biener
parent
7276921760
commit
f3c92486ed
4 changed files with 153 additions and 170 deletions
|
|
@ -1,3 +1,18 @@
|
|||
2013-11-21 Richard Biener <rguenther@suse.de>
|
||||
|
||||
* tree-vect-loop-manip.c (vect_build_loop_niters,
|
||||
vect_generate_tmps_on_preheader): Move ...
|
||||
* tree-vect-loop.c (vect_build_loop_niters,
|
||||
vect_generate_tmps_on_preheader): ... here and simplify.
|
||||
(vect_transform_loop): Call them here and pass down results
|
||||
to consumers.
|
||||
* tree-vect-loop-manip.c (vect_do_peeling_for_loop_bound):
|
||||
Get niter variables from caller.
|
||||
(vect_do_peeling_for_alignment): Likewise.
|
||||
* tree-vectorizer.h (vect_generate_tmps_on_preheader): Remove.
|
||||
(vect_do_peeling_for_loop_bound, vect_do_peeling_for_alignment):
|
||||
Adjust prototypes.
|
||||
|
||||
2013-11-21 Richard Biener <rguenther@suse.de>
|
||||
|
||||
* tree-ssa-loop-ch.c (copy_loop_headers): Decrement
|
||||
|
|
|
|||
|
|
@ -1400,151 +1400,6 @@ find_loop_location (struct loop *loop)
|
|||
}
|
||||
|
||||
|
||||
/* This function builds ni_name = number of iterations loop executes
|
||||
on the loop preheader. If SEQ is given the stmt is instead emitted
|
||||
there. */
|
||||
|
||||
static tree
|
||||
vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq seq)
|
||||
{
|
||||
tree ni_name, var;
|
||||
gimple_seq stmts = NULL;
|
||||
edge pe;
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
|
||||
|
||||
var = create_tmp_var (TREE_TYPE (ni), "niters");
|
||||
ni_name = force_gimple_operand (ni, &stmts, false, var);
|
||||
|
||||
pe = loop_preheader_edge (loop);
|
||||
if (stmts)
|
||||
{
|
||||
if (seq)
|
||||
gimple_seq_add_seq (&seq, stmts);
|
||||
else
|
||||
{
|
||||
basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
return ni_name;
|
||||
}
|
||||
|
||||
|
||||
/* This function generates the following statements:
|
||||
|
||||
ni_name = number of iterations loop executes
|
||||
ratio = ni_name / vf
|
||||
ratio_mult_vf_name = ratio * vf
|
||||
|
||||
and places them at the loop preheader edge or in COND_EXPR_STMT_LIST
|
||||
if that is non-NULL. */
|
||||
|
||||
void
|
||||
vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
tree *ni_name_ptr,
|
||||
tree *ratio_mult_vf_name_ptr,
|
||||
tree *ratio_name_ptr,
|
||||
gimple_seq cond_expr_stmt_list)
|
||||
{
|
||||
|
||||
edge pe;
|
||||
basic_block new_bb;
|
||||
gimple_seq stmts;
|
||||
tree ni_name, ni_minus_gap_name;
|
||||
tree var;
|
||||
tree ratio_name;
|
||||
tree ratio_mult_vf_name;
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
tree ni = LOOP_VINFO_NITERS (loop_vinfo);
|
||||
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
||||
tree log_vf;
|
||||
|
||||
pe = loop_preheader_edge (loop);
|
||||
|
||||
/* Generate temporary variable that contains
|
||||
number of iterations loop executes. */
|
||||
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, cond_expr_stmt_list);
|
||||
log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
|
||||
|
||||
/* If epilogue loop is required because of data accesses with gaps, we
|
||||
subtract one iteration from the total number of iterations here for
|
||||
correct calculation of RATIO. */
|
||||
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
|
||||
{
|
||||
ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name),
|
||||
ni_name,
|
||||
build_one_cst (TREE_TYPE (ni_name)));
|
||||
if (!is_gimple_val (ni_minus_gap_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "ni_gap");
|
||||
|
||||
stmts = NULL;
|
||||
ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts,
|
||||
true, var);
|
||||
if (cond_expr_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
|
||||
else
|
||||
{
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
ni_minus_gap_name = ni_name;
|
||||
|
||||
/* Create: ratio = ni >> log2(vf) */
|
||||
|
||||
ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name),
|
||||
ni_minus_gap_name, log_vf);
|
||||
if (!is_gimple_val (ratio_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "bnd");
|
||||
|
||||
stmts = NULL;
|
||||
ratio_name = force_gimple_operand (ratio_name, &stmts, true, var);
|
||||
if (cond_expr_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
|
||||
else
|
||||
{
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
/* Create: ratio_mult_vf = ratio << log2 (vf). */
|
||||
|
||||
ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name),
|
||||
ratio_name, log_vf);
|
||||
if (!is_gimple_val (ratio_mult_vf_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
|
||||
|
||||
stmts = NULL;
|
||||
ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts,
|
||||
true, var);
|
||||
if (cond_expr_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
|
||||
else
|
||||
{
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
*ni_name_ptr = ni_name;
|
||||
*ratio_mult_vf_name_ptr = ratio_mult_vf_name;
|
||||
*ratio_name_ptr = ratio_name;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/* Function vect_can_advance_ivs_p
|
||||
|
||||
In case the number of iterations that LOOP iterates is unknown at compile
|
||||
|
|
@ -1762,10 +1617,10 @@ vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, tree niters,
|
|||
test. */
|
||||
|
||||
void
|
||||
vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
|
||||
vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo,
|
||||
tree ni_name, tree ratio_mult_vf_name,
|
||||
unsigned int th, bool check_profitability)
|
||||
{
|
||||
tree ni_name, ratio_mult_vf_name;
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
struct loop *new_loop;
|
||||
edge update_e;
|
||||
|
|
@ -1781,15 +1636,6 @@ vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
|
|||
|
||||
initialize_original_copy_tables ();
|
||||
|
||||
/* Generate the following variables on the preheader of original loop:
|
||||
|
||||
ni_name = number of iteration the original loop executes
|
||||
ratio = ni_name / vf
|
||||
ratio_mult_vf_name = ratio * vf */
|
||||
vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
|
||||
&ratio_mult_vf_name, ratio,
|
||||
cond_expr_stmt_list);
|
||||
|
||||
loop_num = loop->num;
|
||||
|
||||
new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
|
||||
|
|
@ -2025,11 +1871,11 @@ vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters)
|
|||
peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
|
||||
|
||||
void
|
||||
vect_do_peeling_for_alignment (loop_vec_info loop_vinfo,
|
||||
vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, tree ni_name,
|
||||
unsigned int th, bool check_profitability)
|
||||
{
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
tree niters_of_prolog_loop, ni_name;
|
||||
tree niters_of_prolog_loop;
|
||||
tree n_iters;
|
||||
tree wide_prolog_niters;
|
||||
struct loop *new_loop;
|
||||
|
|
@ -2043,7 +1889,8 @@ vect_do_peeling_for_alignment (loop_vec_info loop_vinfo,
|
|||
|
||||
initialize_original_copy_tables ();
|
||||
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, NULL);
|
||||
gimple_seq stmts = NULL;
|
||||
gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
|
||||
niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo,
|
||||
ni_name,
|
||||
&bound);
|
||||
|
|
|
|||
|
|
@ -5572,6 +5572,110 @@ vect_loop_kill_debug_uses (struct loop *loop, gimple stmt)
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/* This function builds ni_name = number of iterations. Statements
|
||||
are queued onto SEQ. */
|
||||
|
||||
static tree
|
||||
vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq *seq)
|
||||
{
|
||||
tree ni_name, var;
|
||||
gimple_seq stmts = NULL;
|
||||
tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
|
||||
|
||||
var = create_tmp_var (TREE_TYPE (ni), "niters");
|
||||
ni_name = force_gimple_operand (ni, &stmts, false, var);
|
||||
|
||||
if (stmts)
|
||||
gimple_seq_add_seq (seq, stmts);
|
||||
|
||||
return ni_name;
|
||||
}
|
||||
|
||||
|
||||
/* This function generates the following statements:
|
||||
|
||||
ni_name = number of iterations loop executes
|
||||
ratio = ni_name / vf
|
||||
ratio_mult_vf_name = ratio * vf
|
||||
|
||||
and places them in COND_EXPR_STMT_LIST. */
|
||||
|
||||
static void
|
||||
vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
tree ni_name,
|
||||
tree *ratio_mult_vf_name_ptr,
|
||||
tree *ratio_name_ptr,
|
||||
gimple_seq *cond_expr_stmt_list)
|
||||
{
|
||||
gimple_seq stmts;
|
||||
tree ni_minus_gap_name;
|
||||
tree var;
|
||||
tree ratio_name;
|
||||
tree ratio_mult_vf_name;
|
||||
tree ni = LOOP_VINFO_NITERS (loop_vinfo);
|
||||
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
||||
tree log_vf;
|
||||
|
||||
log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
|
||||
|
||||
/* If epilogue loop is required because of data accesses with gaps, we
|
||||
subtract one iteration from the total number of iterations here for
|
||||
correct calculation of RATIO. */
|
||||
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
|
||||
{
|
||||
ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name),
|
||||
ni_name,
|
||||
build_one_cst (TREE_TYPE (ni_name)));
|
||||
if (!is_gimple_val (ni_minus_gap_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "ni_gap");
|
||||
|
||||
stmts = NULL;
|
||||
ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts,
|
||||
true, var);
|
||||
gimple_seq_add_seq (cond_expr_stmt_list, stmts);
|
||||
}
|
||||
}
|
||||
else
|
||||
ni_minus_gap_name = ni_name;
|
||||
|
||||
/* Create: ratio = ni >> log2(vf) */
|
||||
|
||||
ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name),
|
||||
ni_minus_gap_name, log_vf);
|
||||
if (!is_gimple_val (ratio_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "bnd");
|
||||
|
||||
stmts = NULL;
|
||||
ratio_name = force_gimple_operand (ratio_name, &stmts, true, var);
|
||||
gimple_seq_add_seq (cond_expr_stmt_list, stmts);
|
||||
}
|
||||
*ratio_name_ptr = ratio_name;
|
||||
|
||||
/* Create: ratio_mult_vf = ratio << log2 (vf). */
|
||||
|
||||
if (ratio_mult_vf_name_ptr)
|
||||
{
|
||||
ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name),
|
||||
ratio_name, log_vf);
|
||||
if (!is_gimple_val (ratio_mult_vf_name))
|
||||
{
|
||||
var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
|
||||
|
||||
stmts = NULL;
|
||||
ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts,
|
||||
true, var);
|
||||
gimple_seq_add_seq (cond_expr_stmt_list, stmts);
|
||||
}
|
||||
*ratio_mult_vf_name_ptr = ratio_mult_vf_name;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
/* Function vect_transform_loop.
|
||||
|
||||
The analysis phase has determined that the loop is vectorizable.
|
||||
|
|
@ -5636,11 +5740,18 @@ vect_transform_loop (loop_vec_info loop_vinfo)
|
|||
}
|
||||
|
||||
/* Peel the loop if there are data refs with unknown alignment.
|
||||
Only one data ref with unknown store is allowed. */
|
||||
Only one data ref with unknown store is allowed.
|
||||
This clobbers LOOP_VINFO_NITERS but retains the original
|
||||
in LOOP_VINFO_NITERS_UNCHANGED. So we cannot avoid re-computing
|
||||
niters. */
|
||||
|
||||
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
||||
{
|
||||
vect_do_peeling_for_alignment (loop_vinfo, th, check_profitability);
|
||||
gimple_seq stmts = NULL;
|
||||
tree ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
|
||||
gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
|
||||
vect_do_peeling_for_alignment (loop_vinfo, ni_name,
|
||||
th, check_profitability);
|
||||
check_profitability = false;
|
||||
}
|
||||
|
||||
|
|
@ -5655,16 +5766,27 @@ vect_transform_loop (loop_vec_info loop_vinfo)
|
|||
if ((int) tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
|
||||
< exact_log2 (vectorization_factor)
|
||||
|| LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
|
||||
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio,
|
||||
th, check_profitability);
|
||||
{
|
||||
tree ni_name, ratio_mult_vf;
|
||||
gimple_seq stmts = NULL;
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
|
||||
vect_generate_tmps_on_preheader (loop_vinfo, ni_name, &ratio_mult_vf,
|
||||
&ratio, &stmts);
|
||||
gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
|
||||
vect_do_peeling_for_loop_bound (loop_vinfo, ni_name, ratio_mult_vf,
|
||||
th, check_profitability);
|
||||
}
|
||||
else if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
|
||||
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
|
||||
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
|
||||
else
|
||||
{
|
||||
tree ni_name, ratio_mult_vf;
|
||||
vect_generate_tmps_on_preheader (loop_vinfo, &ni_name, &ratio_mult_vf,
|
||||
&ratio, NULL);
|
||||
tree ni_name;
|
||||
gimple_seq stmts = NULL;
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
|
||||
vect_generate_tmps_on_preheader (loop_vinfo, ni_name, NULL,
|
||||
&ratio, &stmts);
|
||||
gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
|
||||
}
|
||||
|
||||
/* 1) Make sure the loop header has exactly two entries
|
||||
|
|
|
|||
|
|
@ -937,11 +937,10 @@ extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
|
|||
extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
|
||||
struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, edge);
|
||||
extern void vect_loop_versioning (loop_vec_info, unsigned int, bool);
|
||||
extern void vect_generate_tmps_on_preheader (loop_vec_info, tree *, tree *,
|
||||
tree *, gimple_seq);
|
||||
extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
|
||||
extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree, tree,
|
||||
unsigned int, bool);
|
||||
extern void vect_do_peeling_for_alignment (loop_vec_info, unsigned int, bool);
|
||||
extern void vect_do_peeling_for_alignment (loop_vec_info, tree,
|
||||
unsigned int, bool);
|
||||
extern LOC find_loop_location (struct loop *);
|
||||
extern bool vect_can_advance_ivs_p (loop_vec_info);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue