mirror of git://gcc.gnu.org/git/gcc.git
re PR tree-optimization/80928 (SLP vectorization does not handle induction in outer loop vectorization)
2017-06-06 Richard Biener <rguenther@suse.de> PR tree-optimization/80928 * tree-vect-loop.c (vect_update_vf_for_slp): Amend dumps. (vect_analyze_loop_operations): Properly guard analysis for pure SLP case. (vect_transform_loop): Likewise. (vect_analyze_loop_2): Also reset SLP type on PHIs. (vect_model_induction_cost): Do not cost for pure SLP. (vectorizable_induction): Pass in SLP node, implement SLP vectorization of induction in inner loop vectorization. * tree-vect-slp.c (vect_create_new_slp_node): Handle PHIs. (vect_get_and_check_slp_defs): Handle vect_induction_def. (vect_build_slp_tree): Likewise. Handle PHIs as terminating the recursion. (vect_analyze_slp_cost_1): Cost induction. (vect_detect_hybrid_slp_stmts): Handle PHIs. (vect_get_slp_vect_defs): Likewise. * tree-vect-stmts.c (vect_analyze_stmt): Handle induction. (vect_transform_stmt): Handle SLP reductions. * tree-vectorizer.h (vectorizable_induction): Adjust. * gcc.dg/vect/pr80928.c: New testcase. * gcc.dg/vect/slp-13-big-array.c: Remove XFAILs. * gcc.dg/vect/slp-13.c: Likewise. * gcc.dg/vect/slp-perm-9.c: Prevent vectorization of check loop. From-SVN: r248909
This commit is contained in:
Richard Biener
Richard Biener
parent
09a04a5e5e
commit
e7baeb396d
|
|
@ -1,3 +1,25 @@
|
|||
2017-06-06 Richard Biener <rguenther@suse.de>
|
||||
|
||||
PR tree-optimization/80928
|
||||
* tree-vect-loop.c (vect_update_vf_for_slp): Amend dumps.
|
||||
(vect_analyze_loop_operations): Properly guard analysis for
|
||||
pure SLP case.
|
||||
(vect_transform_loop): Likewise.
|
||||
(vect_analyze_loop_2): Also reset SLP type on PHIs.
|
||||
(vect_model_induction_cost): Do not cost for pure SLP.
|
||||
(vectorizable_induction): Pass in SLP node, implement SLP vectorization
|
||||
of induction in inner loop vectorization.
|
||||
* tree-vect-slp.c (vect_create_new_slp_node): Handle PHIs.
|
||||
(vect_get_and_check_slp_defs): Handle vect_induction_def.
|
||||
(vect_build_slp_tree): Likewise. Handle PHIs as terminating the
|
||||
recursion.
|
||||
(vect_analyze_slp_cost_1): Cost induction.
|
||||
(vect_detect_hybrid_slp_stmts): Handle PHIs.
|
||||
(vect_get_slp_vect_defs): Likewise.
|
||||
* tree-vect-stmts.c (vect_analyze_stmt): Handle induction.
|
||||
(vect_transform_stmt): Handle SLP reductions.
|
||||
* tree-vectorizer.h (vectorizable_induction): Adjust.
|
||||
|
||||
2017-06-05 Michael Meissner <meissner@linux.vnet.ibm.com>
|
||||
|
||||
* config/rs6000/rs6000.c (make_resolver_func): Update
|
||||
|
|
|
|||
|
|
@ -1,3 +1,11 @@
|
|||
2017-06-06 Richard Biener <rguenther@suse.de>
|
||||
|
||||
PR tree-optimization/80928
|
||||
* gcc.dg/vect/pr80928.c: New testcase.
|
||||
* gcc.dg/vect/slp-13-big-array.c: Remove XFAILs.
|
||||
* gcc.dg/vect/slp-13.c: Likewise.
|
||||
* gcc.dg/vect/slp-perm-9.c: Prevent vectorization of check loop.
|
||||
|
||||
2017-06-05 David Malcolm <dmalcolm@redhat.com>
|
||||
|
||||
* g++.dg/plugin/comment_plugin.c: New test plugin.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
/* { dg-do compile } */
|
||||
/* { dg-require-effective-target vect_int } */
|
||||
|
||||
#include "tree-vect.h"
|
||||
|
||||
int a[1020];
|
||||
|
||||
void __attribute__((noinline))
|
||||
foo ()
|
||||
{
|
||||
for (int i = 0; i < 1020; i += 5)
|
||||
{
|
||||
a[i] = i;
|
||||
a[i+1] = i;
|
||||
a[i+2] = i;
|
||||
a[i+3] = i;
|
||||
a[i+4] = i;
|
||||
}
|
||||
}
|
||||
|
||||
int main ()
|
||||
{
|
||||
check_vect ();
|
||||
|
||||
foo ();
|
||||
|
||||
/* check results */
|
||||
for (int i = 0; i < 1020; ++i)
|
||||
if (a[i] != ((i + 4) / 5) * 5)
|
||||
abort ();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Make sure we are not triggering hybrid SLP due to the IV update. */
|
||||
/* { dg-final { scan-tree-dump "Loop contains only SLP stmts" "vect" } } */
|
||||
/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" } } */
|
||||
|
|
@ -22,7 +22,7 @@ main1 ()
|
|||
abort ();
|
||||
}
|
||||
|
||||
/* Induction is not SLPable yet. */
|
||||
/* Induction is SLPable. */
|
||||
for (i = 0; i < N; i++)
|
||||
{
|
||||
out[i*8] = in[i*8] + i;
|
||||
|
|
@ -49,8 +49,7 @@ main1 ()
|
|||
abort ();
|
||||
}
|
||||
|
||||
/* Induction is not SLPable yet and strided group size must be a power of 2
|
||||
to get vectorized. */
|
||||
/* Induction is SLPable. */
|
||||
for (i = 0; i < N/2; i++)
|
||||
{
|
||||
out2[i*12] = in2[i*12] + i;
|
||||
|
|
@ -134,7 +133,5 @@ int main (void)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" } } */
|
||||
|
||||
/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ main1 ()
|
|||
unsigned int in2[N*8] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63};
|
||||
unsigned int out2[N*8];
|
||||
|
||||
/* Induction is not SLPable yet. */
|
||||
/* Induction is SLPable. */
|
||||
for (i = 0; i < N; i++)
|
||||
{
|
||||
out[i*8] = in[i*8] + i;
|
||||
|
|
@ -41,8 +41,7 @@ main1 ()
|
|||
abort ();
|
||||
}
|
||||
|
||||
/* Induction is not SLPable yet and strided group size must be a power of 2
|
||||
to get vectorized. */
|
||||
/* Induction is SLPable. */
|
||||
for (i = 0; i < N/2; i++)
|
||||
{
|
||||
out2[i*12] = in2[i*12] + i;
|
||||
|
|
@ -126,7 +125,5 @@ int main (void)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" { xfail *-*-* } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" } } */
|
||||
|
||||
/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */
|
||||
/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ int main (int argc, const char* argv[])
|
|||
check_results[3*i] = 9 * i + 6;
|
||||
check_results[3*i+1] = 9 * i + 15;
|
||||
check_results[3*i+2] = 9 * i + 4;
|
||||
__asm__ volatile ("" : : : "memory");
|
||||
}
|
||||
|
||||
foo (input, output);
|
||||
|
|
|
|||
|
|
@ -1649,11 +1649,19 @@ vect_update_vf_for_slp (loop_vec_info loop_vinfo)
|
|||
}
|
||||
|
||||
if (only_slp_in_loop)
|
||||
{
|
||||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"Loop contains only SLP stmts\n");
|
||||
vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
|
||||
}
|
||||
else
|
||||
{
|
||||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"Loop contains SLP and non-SLP stmts\n");
|
||||
vectorization_factor
|
||||
= least_common_multiple (vectorization_factor,
|
||||
LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
|
||||
}
|
||||
|
||||
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
|
||||
if (dump_enabled_p ())
|
||||
|
|
@ -1765,8 +1773,9 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
|
|||
if (STMT_VINFO_RELEVANT_P (stmt_info))
|
||||
{
|
||||
need_to_vectorize = true;
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
||||
ok = vectorizable_induction (phi, NULL, NULL);
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
|
||||
&& ! PURE_SLP_STMT (stmt_info))
|
||||
ok = vectorizable_induction (phi, NULL, NULL, NULL);
|
||||
}
|
||||
|
||||
if (ok && STMT_VINFO_LIVE_P (stmt_info))
|
||||
|
|
@ -2275,6 +2284,12 @@ again:
|
|||
for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
|
||||
{
|
||||
basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
|
||||
for (gimple_stmt_iterator si = gsi_start_phis (bb);
|
||||
!gsi_end_p (si); gsi_next (&si))
|
||||
{
|
||||
stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si));
|
||||
STMT_SLP_TYPE (stmt_info) = loop_vect;
|
||||
}
|
||||
for (gimple_stmt_iterator si = gsi_start_bb (bb);
|
||||
!gsi_end_p (si); gsi_next (&si))
|
||||
{
|
||||
|
|
@ -3797,6 +3812,9 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies)
|
|||
void *target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
|
||||
unsigned inside_cost, prologue_cost;
|
||||
|
||||
if (PURE_SLP_STMT (stmt_info))
|
||||
return;
|
||||
|
||||
/* loop cost for vec_loop. */
|
||||
inside_cost = add_stmt_cost (target_cost_data, ncopies, vector_stmt,
|
||||
stmt_info, 0, vect_body);
|
||||
|
|
@ -6086,7 +6104,7 @@ vect_min_worthwhile_factor (enum tree_code code)
|
|||
bool
|
||||
vectorizable_induction (gimple *phi,
|
||||
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
||||
gimple **vec_stmt)
|
||||
gimple **vec_stmt, slp_tree slp_node)
|
||||
{
|
||||
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
|
||||
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
||||
|
|
@ -6125,13 +6143,12 @@ vectorizable_induction (gimple *phi,
|
|||
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
|
||||
return false;
|
||||
|
||||
/* FORNOW: SLP not supported. */
|
||||
if (STMT_SLP_TYPE (stmt_info))
|
||||
return false;
|
||||
|
||||
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
||||
unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
||||
|
||||
if (slp_node)
|
||||
ncopies = 1;
|
||||
else
|
||||
ncopies = vf / nunits;
|
||||
gcc_assert (ncopies >= 1);
|
||||
|
||||
|
|
@ -6234,6 +6251,147 @@ vectorizable_induction (gimple *phi,
|
|||
/* Find the first insertion point in the BB. */
|
||||
si = gsi_after_labels (bb);
|
||||
|
||||
/* For SLP induction we have to generate several IVs as for example
|
||||
with group size 3 we need [i, i, i, i + S] [i + S, i + S, i + 2*S, i + 2*S]
|
||||
[i + 2*S, i + 3*S, i + 3*S, i + 3*S]. The step is the same uniform
|
||||
[VF*S, VF*S, VF*S, VF*S] for all. */
|
||||
if (slp_node)
|
||||
{
|
||||
/* Convert the init to the desired type. */
|
||||
stmts = NULL;
|
||||
init_expr = gimple_convert (&stmts, TREE_TYPE (vectype), init_expr);
|
||||
if (stmts)
|
||||
{
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
|
||||
/* Generate [VF*S, VF*S, ... ]. */
|
||||
if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr)))
|
||||
{
|
||||
expr = build_int_cst (integer_type_node, vf);
|
||||
expr = fold_convert (TREE_TYPE (step_expr), expr);
|
||||
}
|
||||
else
|
||||
expr = build_int_cst (TREE_TYPE (step_expr), vf);
|
||||
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
||||
expr, step_expr);
|
||||
if (! CONSTANT_CLASS_P (new_name))
|
||||
new_name = vect_init_vector (phi, new_name,
|
||||
TREE_TYPE (step_expr), NULL);
|
||||
new_vec = build_vector_from_val (vectype, new_name);
|
||||
vec_step = vect_init_vector (phi, new_vec, vectype, NULL);
|
||||
|
||||
/* Now generate the IVs. */
|
||||
unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
|
||||
unsigned nvects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
|
||||
unsigned elts = nunits * nvects;
|
||||
unsigned nivs = least_common_multiple (group_size, nunits) / nunits;
|
||||
gcc_assert (elts % group_size == 0);
|
||||
tree elt = init_expr;
|
||||
unsigned ivn;
|
||||
for (ivn = 0; ivn < nivs; ++ivn)
|
||||
{
|
||||
tree *elts = XALLOCAVEC (tree, nunits);
|
||||
bool constant_p = true;
|
||||
for (unsigned eltn = 0; eltn < nunits; ++eltn)
|
||||
{
|
||||
if (ivn*nunits + eltn >= group_size
|
||||
&& (ivn*nunits + eltn) % group_size == 0)
|
||||
{
|
||||
stmts = NULL;
|
||||
elt = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (elt),
|
||||
elt, step_expr);
|
||||
if (stmts)
|
||||
{
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
if (! CONSTANT_CLASS_P (elt))
|
||||
constant_p = false;
|
||||
elts[eltn] = elt;
|
||||
}
|
||||
if (constant_p)
|
||||
new_vec = build_vector (vectype, elts);
|
||||
else
|
||||
{
|
||||
vec<constructor_elt, va_gc> *v;
|
||||
vec_alloc (v, nunits);
|
||||
for (i = 0; i < nunits; ++i)
|
||||
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, elts[i]);
|
||||
new_vec = build_constructor (vectype, v);
|
||||
}
|
||||
vec_init = vect_init_vector (phi, new_vec, vectype, NULL);
|
||||
|
||||
/* Create the induction-phi that defines the induction-operand. */
|
||||
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
|
||||
induction_phi = create_phi_node (vec_dest, iv_loop->header);
|
||||
set_vinfo_for_stmt (induction_phi,
|
||||
new_stmt_vec_info (induction_phi, loop_vinfo));
|
||||
induc_def = PHI_RESULT (induction_phi);
|
||||
|
||||
/* Create the iv update inside the loop */
|
||||
vec_def = make_ssa_name (vec_dest);
|
||||
new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, vec_step);
|
||||
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
||||
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo));
|
||||
|
||||
/* Set the arguments of the phi node: */
|
||||
add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
|
||||
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
|
||||
UNKNOWN_LOCATION);
|
||||
|
||||
SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi);
|
||||
}
|
||||
|
||||
/* Re-use IVs when we can. */
|
||||
if (ivn < nvects)
|
||||
{
|
||||
unsigned vfp
|
||||
= least_common_multiple (group_size, nunits) / group_size;
|
||||
/* Generate [VF'*S, VF'*S, ... ]. */
|
||||
if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr)))
|
||||
{
|
||||
expr = build_int_cst (integer_type_node, vfp);
|
||||
expr = fold_convert (TREE_TYPE (step_expr), expr);
|
||||
}
|
||||
else
|
||||
expr = build_int_cst (TREE_TYPE (step_expr), vfp);
|
||||
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
||||
expr, step_expr);
|
||||
if (! CONSTANT_CLASS_P (new_name))
|
||||
new_name = vect_init_vector (phi, new_name,
|
||||
TREE_TYPE (step_expr), NULL);
|
||||
new_vec = build_vector_from_val (vectype, new_name);
|
||||
vec_step = vect_init_vector (phi, new_vec, vectype, NULL);
|
||||
for (; ivn < nvects; ++ivn)
|
||||
{
|
||||
gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs];
|
||||
tree def;
|
||||
if (gimple_code (iv) == GIMPLE_PHI)
|
||||
def = gimple_phi_result (iv);
|
||||
else
|
||||
def = gimple_assign_lhs (iv);
|
||||
new_stmt = gimple_build_assign (make_ssa_name (vectype),
|
||||
PLUS_EXPR,
|
||||
def, vec_step);
|
||||
if (gimple_code (iv) == GIMPLE_PHI)
|
||||
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
||||
else
|
||||
{
|
||||
gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
|
||||
gsi_insert_after (&tgsi, new_stmt, GSI_CONTINUE_LINKING);
|
||||
}
|
||||
set_vinfo_for_stmt (new_stmt,
|
||||
new_stmt_vec_info (new_stmt, loop_vinfo));
|
||||
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Create the vector that holds the initial_value of the induction. */
|
||||
if (nested_in_vect_loop)
|
||||
{
|
||||
|
|
@ -6841,7 +6999,8 @@ vect_transform_loop (loop_vec_info loop_vinfo)
|
|||
&& dump_enabled_p ())
|
||||
dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
|
||||
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
|
||||
&& ! PURE_SLP_STMT (stmt_info))
|
||||
{
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_NOTE, vect_location, "transform phi.\n");
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ vect_create_new_slp_node (vec<gimple *> scalar_stmts)
|
|||
if (gimple_assign_rhs_code (stmt) == COND_EXPR)
|
||||
nops++;
|
||||
}
|
||||
else if (gimple_code (stmt) == GIMPLE_PHI)
|
||||
nops = 0;
|
||||
else
|
||||
return NULL;
|
||||
|
||||
|
|
@ -404,6 +406,7 @@ again:
|
|||
case vect_reduction_def:
|
||||
break;
|
||||
|
||||
case vect_induction_def:
|
||||
case vect_internal_def:
|
||||
oprnd_info->def_stmts.quick_push (def_stmt);
|
||||
break;
|
||||
|
|
@ -935,9 +938,23 @@ vect_build_slp_tree (vec_info *vinfo,
|
|||
if (gimple_assign_rhs_code (stmt) == COND_EXPR)
|
||||
nops++;
|
||||
}
|
||||
else if (gimple_code (stmt) == GIMPLE_PHI)
|
||||
nops = 0;
|
||||
else
|
||||
return NULL;
|
||||
|
||||
/* If the SLP node is a PHI (induction), terminate the recursion. */
|
||||
if (gimple_code (stmt) == GIMPLE_PHI)
|
||||
{
|
||||
FOR_EACH_VEC_ELT (stmts, i, stmt)
|
||||
if (stmt != stmts[0])
|
||||
/* Induction from different IVs is not supported. */
|
||||
return NULL;
|
||||
node = vect_create_new_slp_node (stmts);
|
||||
return node;
|
||||
}
|
||||
|
||||
|
||||
bool two_operators = false;
|
||||
unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
|
||||
if (!vect_build_slp_tree_1 (vinfo, swap,
|
||||
|
|
@ -987,7 +1004,8 @@ vect_build_slp_tree (vec_info *vinfo,
|
|||
unsigned old_tree_size = this_tree_size;
|
||||
unsigned int j;
|
||||
|
||||
if (oprnd_info->first_dt != vect_internal_def)
|
||||
if (oprnd_info->first_dt != vect_internal_def
|
||||
&& oprnd_info->first_dt != vect_induction_def)
|
||||
continue;
|
||||
|
||||
if (++this_tree_size > max_tree_size)
|
||||
|
|
@ -1611,6 +1629,28 @@ vect_analyze_slp_cost_1 (slp_instance instance, slp_tree node,
|
|||
return;
|
||||
}
|
||||
}
|
||||
else if (STMT_VINFO_TYPE (stmt_info) == induc_vec_info_type)
|
||||
{
|
||||
/* ncopies_for_cost is the number of IVs we generate. */
|
||||
record_stmt_cost (body_cost_vec, ncopies_for_cost, vector_stmt,
|
||||
stmt_info, 0, vect_body);
|
||||
|
||||
/* Prologue cost for the initial values and step vector. */
|
||||
record_stmt_cost (prologue_cost_vec, ncopies_for_cost,
|
||||
CONSTANT_CLASS_P
|
||||
(STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED
|
||||
(stmt_info))
|
||||
? vector_load : vec_construct,
|
||||
stmt_info, 0, vect_prologue);
|
||||
record_stmt_cost (prologue_cost_vec, 1,
|
||||
CONSTANT_CLASS_P
|
||||
(STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info))
|
||||
? vector_load : vec_construct,
|
||||
stmt_info, 0, vect_prologue);
|
||||
|
||||
/* ??? No easy way to get at the actual number of vector stmts
|
||||
to be geneated and thus the derived IVs. */
|
||||
}
|
||||
else
|
||||
{
|
||||
record_stmt_cost (body_cost_vec, ncopies_for_cost, vector_stmt,
|
||||
|
|
@ -2169,8 +2209,13 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype)
|
|||
if (! STMT_VINFO_IN_PATTERN_P (stmt_vinfo)
|
||||
&& STMT_VINFO_RELATED_STMT (stmt_vinfo))
|
||||
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
|
||||
if (TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
|
||||
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0))
|
||||
tree def;
|
||||
if (gimple_code (stmt) == GIMPLE_PHI)
|
||||
def = gimple_phi_result (stmt);
|
||||
else
|
||||
def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF);
|
||||
if (def)
|
||||
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, def)
|
||||
{
|
||||
if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
|
||||
continue;
|
||||
|
|
@ -3277,6 +3322,9 @@ vect_get_slp_vect_defs (slp_tree slp_node, vec<tree> *vec_oprnds)
|
|||
FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt)
|
||||
{
|
||||
gcc_assert (vec_def_stmt);
|
||||
if (gimple_code (vec_def_stmt) == GIMPLE_PHI)
|
||||
vec_oprnd = gimple_phi_result (vec_def_stmt);
|
||||
else
|
||||
vec_oprnd = gimple_get_lhs (vec_def_stmt);
|
||||
vec_oprnds->quick_push (vec_oprnd);
|
||||
}
|
||||
|
|
@ -3331,8 +3379,13 @@ vect_get_slp_defs (vec<tree> ops, slp_tree slp_node,
|
|||
gimple *first_def = SLP_TREE_SCALAR_STMTS (child)[0];
|
||||
gimple *related
|
||||
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (first_def));
|
||||
tree first_def_op;
|
||||
|
||||
if (operand_equal_p (oprnd, gimple_get_lhs (first_def), 0)
|
||||
if (gimple_code (first_def) == GIMPLE_PHI)
|
||||
first_def_op = gimple_phi_result (first_def);
|
||||
else
|
||||
first_def_op = gimple_get_lhs (first_def);
|
||||
if (operand_equal_p (oprnd, first_def_op, 0)
|
||||
|| (related
|
||||
&& operand_equal_p (oprnd, gimple_get_lhs (related), 0)))
|
||||
{
|
||||
|
|
|
|||
|
|
@ -8520,6 +8520,9 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node)
|
|||
break;
|
||||
|
||||
case vect_induction_def:
|
||||
gcc_assert (!bb_vinfo);
|
||||
break;
|
||||
|
||||
case vect_constant_def:
|
||||
case vect_external_def:
|
||||
case vect_unknown_def_type:
|
||||
|
|
@ -8598,6 +8601,7 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node)
|
|||
|| vectorizable_call (stmt, NULL, NULL, node)
|
||||
|| vectorizable_store (stmt, NULL, NULL, node)
|
||||
|| vectorizable_reduction (stmt, NULL, NULL, node)
|
||||
|| vectorizable_induction (stmt, NULL, NULL, node)
|
||||
|| vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
|
||||
|| vectorizable_comparison (stmt, NULL, NULL, NULL, node));
|
||||
else
|
||||
|
|
@ -8681,8 +8685,7 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
break;
|
||||
|
||||
case induc_vec_info_type:
|
||||
gcc_assert (!slp_node);
|
||||
done = vectorizable_induction (stmt, gsi, &vec_stmt);
|
||||
done = vectorizable_induction (stmt, gsi, &vec_stmt, slp_node);
|
||||
gcc_assert (done);
|
||||
break;
|
||||
|
||||
|
|
|
|||
|
|
@ -1159,7 +1159,8 @@ extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *,
|
|||
slp_tree, int, gimple **);
|
||||
extern bool vectorizable_reduction (gimple *, gimple_stmt_iterator *,
|
||||
gimple **, slp_tree);
|
||||
extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, gimple **);
|
||||
extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *,
|
||||
gimple **, slp_tree);
|
||||
extern tree get_initial_def_for_reduction (gimple *, tree, tree *);
|
||||
extern int vect_min_worthwhile_factor (enum tree_code);
|
||||
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
|
||||
|
|
|
|||
Loading…
Reference in New Issue