From f3c92486ed1f5cb336d0f78438ac3a0b03b38f53 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Thu, 21 Nov 2013 09:19:31 +0000 Subject: [PATCH] tree-vect-loop-manip.c (vect_build_loop_niters, [...]): Move ... 2013-11-21 Richard Biener * tree-vect-loop-manip.c (vect_build_loop_niters, vect_generate_tmps_on_preheader): Move ... * tree-vect-loop.c (vect_build_loop_niters, vect_generate_tmps_on_preheader): ... here and simplify. (vect_transform_loop): Call them here and pass down results to consumers. * tree-vect-loop-manip.c (vect_do_peeling_for_loop_bound): Get niter variables from caller. (vect_do_peeling_for_alignment): Likewise. * tree-vectorizer.h (vect_generate_tmps_on_preheader): Remove. (vect_do_peeling_for_loop_bound, vect_do_peeling_for_alignment): Adjust prototypes. From-SVN: r205199 --- gcc/ChangeLog | 15 ++++ gcc/tree-vect-loop-manip.c | 165 ++----------------------------------- gcc/tree-vect-loop.c | 136 ++++++++++++++++++++++++++++-- gcc/tree-vectorizer.h | 7 +- 4 files changed, 153 insertions(+), 170 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 1ececafc827..cecd7578cad 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,18 @@ +2013-11-21 Richard Biener + + * tree-vect-loop-manip.c (vect_build_loop_niters, + vect_generate_tmps_on_preheader): Move ... + * tree-vect-loop.c (vect_build_loop_niters, + vect_generate_tmps_on_preheader): ... here and simplify. + (vect_transform_loop): Call them here and pass down results + to consumers. + * tree-vect-loop-manip.c (vect_do_peeling_for_loop_bound): + Get niter variables from caller. + (vect_do_peeling_for_alignment): Likewise. + * tree-vectorizer.h (vect_generate_tmps_on_preheader): Remove. + (vect_do_peeling_for_loop_bound, vect_do_peeling_for_alignment): + Adjust prototypes. + 2013-11-21 Richard Biener * tree-ssa-loop-ch.c (copy_loop_headers): Decrement diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index 3193f20e5a9..a03b54c1f97 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -1400,151 +1400,6 @@ find_loop_location (struct loop *loop) } -/* This function builds ni_name = number of iterations loop executes - on the loop preheader. If SEQ is given the stmt is instead emitted - there. */ - -static tree -vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq seq) -{ - tree ni_name, var; - gimple_seq stmts = NULL; - edge pe; - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo)); - - var = create_tmp_var (TREE_TYPE (ni), "niters"); - ni_name = force_gimple_operand (ni, &stmts, false, var); - - pe = loop_preheader_edge (loop); - if (stmts) - { - if (seq) - gimple_seq_add_seq (&seq, stmts); - else - { - basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); - gcc_assert (!new_bb); - } - } - - return ni_name; -} - - -/* This function generates the following statements: - - ni_name = number of iterations loop executes - ratio = ni_name / vf - ratio_mult_vf_name = ratio * vf - - and places them at the loop preheader edge or in COND_EXPR_STMT_LIST - if that is non-NULL. */ - -void -vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo, - tree *ni_name_ptr, - tree *ratio_mult_vf_name_ptr, - tree *ratio_name_ptr, - gimple_seq cond_expr_stmt_list) -{ - - edge pe; - basic_block new_bb; - gimple_seq stmts; - tree ni_name, ni_minus_gap_name; - tree var; - tree ratio_name; - tree ratio_mult_vf_name; - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - tree ni = LOOP_VINFO_NITERS (loop_vinfo); - int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - tree log_vf; - - pe = loop_preheader_edge (loop); - - /* Generate temporary variable that contains - number of iterations loop executes. */ - - ni_name = vect_build_loop_niters (loop_vinfo, cond_expr_stmt_list); - log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf)); - - /* If epilogue loop is required because of data accesses with gaps, we - subtract one iteration from the total number of iterations here for - correct calculation of RATIO. */ - if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) - { - ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name), - ni_name, - build_one_cst (TREE_TYPE (ni_name))); - if (!is_gimple_val (ni_minus_gap_name)) - { - var = create_tmp_var (TREE_TYPE (ni), "ni_gap"); - - stmts = NULL; - ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts, - true, var); - if (cond_expr_stmt_list) - gimple_seq_add_seq (&cond_expr_stmt_list, stmts); - else - { - pe = loop_preheader_edge (loop); - new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); - gcc_assert (!new_bb); - } - } - } - else - ni_minus_gap_name = ni_name; - - /* Create: ratio = ni >> log2(vf) */ - - ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name), - ni_minus_gap_name, log_vf); - if (!is_gimple_val (ratio_name)) - { - var = create_tmp_var (TREE_TYPE (ni), "bnd"); - - stmts = NULL; - ratio_name = force_gimple_operand (ratio_name, &stmts, true, var); - if (cond_expr_stmt_list) - gimple_seq_add_seq (&cond_expr_stmt_list, stmts); - else - { - pe = loop_preheader_edge (loop); - new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); - gcc_assert (!new_bb); - } - } - - /* Create: ratio_mult_vf = ratio << log2 (vf). */ - - ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name), - ratio_name, log_vf); - if (!is_gimple_val (ratio_mult_vf_name)) - { - var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf"); - - stmts = NULL; - ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts, - true, var); - if (cond_expr_stmt_list) - gimple_seq_add_seq (&cond_expr_stmt_list, stmts); - else - { - pe = loop_preheader_edge (loop); - new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); - gcc_assert (!new_bb); - } - } - - *ni_name_ptr = ni_name; - *ratio_mult_vf_name_ptr = ratio_mult_vf_name; - *ratio_name_ptr = ratio_name; - - return; -} - /* Function vect_can_advance_ivs_p In case the number of iterations that LOOP iterates is unknown at compile @@ -1762,10 +1617,10 @@ vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, tree niters, test. */ void -vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio, +vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, + tree ni_name, tree ratio_mult_vf_name, unsigned int th, bool check_profitability) { - tree ni_name, ratio_mult_vf_name; struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *new_loop; edge update_e; @@ -1781,15 +1636,6 @@ vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio, initialize_original_copy_tables (); - /* Generate the following variables on the preheader of original loop: - - ni_name = number of iteration the original loop executes - ratio = ni_name / vf - ratio_mult_vf_name = ratio * vf */ - vect_generate_tmps_on_preheader (loop_vinfo, &ni_name, - &ratio_mult_vf_name, ratio, - cond_expr_stmt_list); - loop_num = loop->num; new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop), @@ -2025,11 +1871,11 @@ vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters) peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */ void -vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, +vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, tree ni_name, unsigned int th, bool check_profitability) { struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - tree niters_of_prolog_loop, ni_name; + tree niters_of_prolog_loop; tree n_iters; tree wide_prolog_niters; struct loop *new_loop; @@ -2043,7 +1889,8 @@ vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, initialize_original_copy_tables (); - ni_name = vect_build_loop_niters (loop_vinfo, NULL); + gimple_seq stmts = NULL; + gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts); niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name, &bound); diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index a97d413f5cd..36393da8ad3 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -5572,6 +5572,110 @@ vect_loop_kill_debug_uses (struct loop *loop, gimple stmt) } } + +/* This function builds ni_name = number of iterations. Statements + are queued onto SEQ. */ + +static tree +vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq *seq) +{ + tree ni_name, var; + gimple_seq stmts = NULL; + tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo)); + + var = create_tmp_var (TREE_TYPE (ni), "niters"); + ni_name = force_gimple_operand (ni, &stmts, false, var); + + if (stmts) + gimple_seq_add_seq (seq, stmts); + + return ni_name; +} + + +/* This function generates the following statements: + + ni_name = number of iterations loop executes + ratio = ni_name / vf + ratio_mult_vf_name = ratio * vf + + and places them in COND_EXPR_STMT_LIST. */ + +static void +vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo, + tree ni_name, + tree *ratio_mult_vf_name_ptr, + tree *ratio_name_ptr, + gimple_seq *cond_expr_stmt_list) +{ + gimple_seq stmts; + tree ni_minus_gap_name; + tree var; + tree ratio_name; + tree ratio_mult_vf_name; + tree ni = LOOP_VINFO_NITERS (loop_vinfo); + int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + tree log_vf; + + log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf)); + + /* If epilogue loop is required because of data accesses with gaps, we + subtract one iteration from the total number of iterations here for + correct calculation of RATIO. */ + if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) + { + ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name), + ni_name, + build_one_cst (TREE_TYPE (ni_name))); + if (!is_gimple_val (ni_minus_gap_name)) + { + var = create_tmp_var (TREE_TYPE (ni), "ni_gap"); + + stmts = NULL; + ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts, + true, var); + gimple_seq_add_seq (cond_expr_stmt_list, stmts); + } + } + else + ni_minus_gap_name = ni_name; + + /* Create: ratio = ni >> log2(vf) */ + + ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name), + ni_minus_gap_name, log_vf); + if (!is_gimple_val (ratio_name)) + { + var = create_tmp_var (TREE_TYPE (ni), "bnd"); + + stmts = NULL; + ratio_name = force_gimple_operand (ratio_name, &stmts, true, var); + gimple_seq_add_seq (cond_expr_stmt_list, stmts); + } + *ratio_name_ptr = ratio_name; + + /* Create: ratio_mult_vf = ratio << log2 (vf). */ + + if (ratio_mult_vf_name_ptr) + { + ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name), + ratio_name, log_vf); + if (!is_gimple_val (ratio_mult_vf_name)) + { + var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf"); + + stmts = NULL; + ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts, + true, var); + gimple_seq_add_seq (cond_expr_stmt_list, stmts); + } + *ratio_mult_vf_name_ptr = ratio_mult_vf_name; + } + + return; +} + + /* Function vect_transform_loop. The analysis phase has determined that the loop is vectorizable. @@ -5636,11 +5740,18 @@ vect_transform_loop (loop_vec_info loop_vinfo) } /* Peel the loop if there are data refs with unknown alignment. - Only one data ref with unknown store is allowed. */ + Only one data ref with unknown store is allowed. + This clobbers LOOP_VINFO_NITERS but retains the original + in LOOP_VINFO_NITERS_UNCHANGED. So we cannot avoid re-computing + niters. */ if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo)) { - vect_do_peeling_for_alignment (loop_vinfo, th, check_profitability); + gimple_seq stmts = NULL; + tree ni_name = vect_build_loop_niters (loop_vinfo, &stmts); + gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts); + vect_do_peeling_for_alignment (loop_vinfo, ni_name, + th, check_profitability); check_profitability = false; } @@ -5655,16 +5766,27 @@ vect_transform_loop (loop_vec_info loop_vinfo) if ((int) tree_ctz (LOOP_VINFO_NITERS (loop_vinfo)) < exact_log2 (vectorization_factor) || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) - vect_do_peeling_for_loop_bound (loop_vinfo, &ratio, - th, check_profitability); + { + tree ni_name, ratio_mult_vf; + gimple_seq stmts = NULL; + ni_name = vect_build_loop_niters (loop_vinfo, &stmts); + vect_generate_tmps_on_preheader (loop_vinfo, ni_name, &ratio_mult_vf, + &ratio, &stmts); + gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts); + vect_do_peeling_for_loop_bound (loop_vinfo, ni_name, ratio_mult_vf, + th, check_profitability); + } else if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)) ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)), LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor); else { - tree ni_name, ratio_mult_vf; - vect_generate_tmps_on_preheader (loop_vinfo, &ni_name, &ratio_mult_vf, - &ratio, NULL); + tree ni_name; + gimple_seq stmts = NULL; + ni_name = vect_build_loop_niters (loop_vinfo, &stmts); + vect_generate_tmps_on_preheader (loop_vinfo, ni_name, NULL, + &ratio, &stmts); + gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts); } /* 1) Make sure the loop header has exactly two entries diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 1634d7fa7f5..1297fab6cfe 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -937,11 +937,10 @@ extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree); extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge); struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, edge); extern void vect_loop_versioning (loop_vec_info, unsigned int, bool); -extern void vect_generate_tmps_on_preheader (loop_vec_info, tree *, tree *, - tree *, gimple_seq); -extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *, +extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree, tree, unsigned int, bool); -extern void vect_do_peeling_for_alignment (loop_vec_info, unsigned int, bool); +extern void vect_do_peeling_for_alignment (loop_vec_info, tree, + unsigned int, bool); extern LOC find_loop_location (struct loop *); extern bool vect_can_advance_ivs_p (loop_vec_info);