git
/
gcc
mirror of git://gcc.gnu.org/git/gcc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[20/46] Make *FIRST_ELEMENT and *NEXT_ELEMENT stmt_vec_infos 

This patch changes {REDUC,DR}_GROUP_{FIRST,NEXT} element from a
gimple stmt to stmt_vec_info.

2018-07-31  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (_stmt_vec_info::first_element): Change from
	a gimple stmt to a stmt_vec_info.
	(_stmt_vec_info::next_element): Likewise.
	* tree-vect-data-refs.c (vect_update_misalignment_for_peel)
	(vect_slp_analyze_and_verify_node_alignment)
	(vect_analyze_group_access_1, vect_analyze_group_access)
	(vect_small_gap_p, vect_prune_runtime_alias_test_list)
	(vect_create_data_ref_ptr, vect_record_grouped_load_vectors)
	(vect_supportable_dr_alignment): Update accordingly.
	* tree-vect-loop.c (vect_fixup_reduc_chain): Likewise.
	(vect_fixup_scalar_cycles_with_patterns, vect_is_slp_reduction)
	(vect_is_simple_reduction, vectorizable_reduction): Likewise.
	* tree-vect-patterns.c (vect_reassociating_reduction_p): Likewise.
	* tree-vect-slp.c (vect_build_slp_tree_1)
	(vect_attempt_slp_rearrange_stmts, vect_supported_load_permutation_p)
	(vect_split_slp_store_group, vect_analyze_slp_instance)
	(vect_analyze_slp, vect_transform_slp_perm_load): Likewise.
	* tree-vect-stmts.c (vect_model_store_cost, vect_model_load_cost)
	(get_group_load_store_type, get_load_store_type)
	(get_group_alias_ptr_type, vectorizable_store, vectorizable_load)
	(vect_transform_stmt, vect_remove_stores): Likewise.

From-SVN: r263135
This commit is contained in:
Richard Sandiford 2018-07-31 14:23:29 +00:00 committed by Richard Sandiford
parent 78e02b3bbb
commit bffb8014d0
7 changed files with 227 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
gcc/ChangeLog
View File

@ -1,3 +1,27 @@
2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
* tree-vectorizer.h (_stmt_vec_info::first_element): Change from
a gimple stmt to a stmt_vec_info.
(_stmt_vec_info::next_element): Likewise.
* tree-vect-data-refs.c (vect_update_misalignment_for_peel)
(vect_slp_analyze_and_verify_node_alignment)
(vect_analyze_group_access_1, vect_analyze_group_access)
(vect_small_gap_p, vect_prune_runtime_alias_test_list)
(vect_create_data_ref_ptr, vect_record_grouped_load_vectors)
(vect_supportable_dr_alignment): Update accordingly.
* tree-vect-loop.c (vect_fixup_reduc_chain): Likewise.
(vect_fixup_scalar_cycles_with_patterns, vect_is_slp_reduction)
(vect_is_simple_reduction, vectorizable_reduction): Likewise.
* tree-vect-patterns.c (vect_reassociating_reduction_p): Likewise.
* tree-vect-slp.c (vect_build_slp_tree_1)
(vect_attempt_slp_rearrange_stmts, vect_supported_load_permutation_p)
(vect_split_slp_store_group, vect_analyze_slp_instance)
(vect_analyze_slp, vect_transform_slp_perm_load): Likewise.
* tree-vect-stmts.c (vect_model_store_cost, vect_model_load_cost)
(get_group_load_store_type, get_load_store_type)
(get_group_alias_ptr_type, vectorizable_store, vectorizable_load)
(vect_transform_stmt, vect_remove_stores): Likewise.
2018-07-31 Richard Sandiford <richard.sandiford@arm.com> 2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
* tree-vectorizer.h (vect_dr_stmt): Return a stmt_vec_info rather * tree-vectorizer.h (vect_dr_stmt): Return a stmt_vec_info rather

112
gcc/tree-vect-data-refs.c
View File

@ -1077,7 +1077,7 @@ vect_update_misalignment_for_peel (struct data_reference *dr,
/* For interleaved data accesses the step in the loop must be multiplied by /* For interleaved data accesses the step in the loop must be multiplied by
the size of the interleaving group. */ the size of the interleaving group. */
if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
dr_size *= DR_GROUP_SIZE (vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (stmt_info))); dr_size *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
if (STMT_VINFO_GROUPED_ACCESS (peel_stmt_info)) if (STMT_VINFO_GROUPED_ACCESS (peel_stmt_info))
dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info); dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info);
@ -2370,12 +2370,11 @@ vect_slp_analyze_and_verify_node_alignment (slp_tree node)
the node is permuted in which case we start from the first the node is permuted in which case we start from the first
element in the group. */ element in the group. */
stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0]; stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
gimple *first_stmt = first_stmt_info->stmt;
data_reference_p first_dr = STMT_VINFO_DATA_REF (first_stmt_info); data_reference_p first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
if (SLP_TREE_LOAD_PERMUTATION (node).exists ()) if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
first_stmt = DR_GROUP_FIRST_ELEMENT (first_stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
data_reference_p dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); data_reference_p dr = STMT_VINFO_DATA_REF (first_stmt_info);
vect_compute_data_ref_alignment (dr); vect_compute_data_ref_alignment (dr);
/* For creating the data-ref pointer we need alignment of the /* For creating the data-ref pointer we need alignment of the
first element anyway. */ first element anyway. */
@ -2520,11 +2519,11 @@ vect_analyze_group_access_1 (struct data_reference *dr)
if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info) if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
{ {
/* First stmt in the interleaving chain. Check the chain. */ /* First stmt in the interleaving chain. Check the chain. */
gimple *next = DR_GROUP_NEXT_ELEMENT (stmt_info); stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
struct data_reference *data_ref = dr; struct data_reference *data_ref = dr;
unsigned int count = 1; unsigned int count = 1;
tree prev_init = DR_INIT (data_ref); tree prev_init = DR_INIT (data_ref);
gimple *prev = stmt_info; stmt_vec_info prev = stmt_info;
HOST_WIDE_INT diff, gaps = 0; HOST_WIDE_INT diff, gaps = 0;
/* By construction, all group members have INTEGER_CST DR_INITs. */ /* By construction, all group members have INTEGER_CST DR_INITs. */
@ -2535,8 +2534,7 @@ vect_analyze_group_access_1 (struct data_reference *dr)
stmt, and the rest get their vectorized loads from the first stmt, and the rest get their vectorized loads from the first
one. */ one. */
if (!tree_int_cst_compare (DR_INIT (data_ref), if (!tree_int_cst_compare (DR_INIT (data_ref),
DR_INIT (STMT_VINFO_DATA_REF ( DR_INIT (STMT_VINFO_DATA_REF (next))))
vinfo_for_stmt (next)))))
{ {
if (DR_IS_WRITE (data_ref)) if (DR_IS_WRITE (data_ref))
{ {
@ -2550,16 +2548,16 @@ vect_analyze_group_access_1 (struct data_reference *dr)
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Two or more load stmts share the same dr.\n"); "Two or more load stmts share the same dr.\n");
/* For load use the same data-ref load. */ /* For load use the same data-ref load. */
DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next)) = prev; DR_GROUP_SAME_DR_STMT (next) = prev;
prev = next; prev = next;
next = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); next = DR_GROUP_NEXT_ELEMENT (next);
continue; continue;
} }
prev = next; prev = next;
data_ref = STMT_VINFO_DATA_REF (vinfo_for_stmt (next)); data_ref = STMT_VINFO_DATA_REF (next);
/* All group members have the same STEP by construction. */ /* All group members have the same STEP by construction. */
gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0)); gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0));
@ -2587,12 +2585,12 @@ vect_analyze_group_access_1 (struct data_reference *dr)
/* Store the gap from the previous member of the group. If there is no /* Store the gap from the previous member of the group. If there is no
gap in the access, DR_GROUP_GAP is always 1. */ gap in the access, DR_GROUP_GAP is always 1. */
DR_GROUP_GAP (vinfo_for_stmt (next)) = diff; DR_GROUP_GAP (next) = diff;
prev_init = DR_INIT (data_ref); prev_init = DR_INIT (data_ref);
next = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); next = DR_GROUP_NEXT_ELEMENT (next);
/* Count the number of data-refs in the chain. */ /* Count the number of data-refs in the chain. */
count++; count++;
} }
if (groupsize == 0) if (groupsize == 0)
@ -2668,15 +2666,13 @@ vect_analyze_group_access (struct data_reference *dr)
if (!vect_analyze_group_access_1 (dr)) if (!vect_analyze_group_access_1 (dr))
{ {
/* Dissolve the group if present. */ /* Dissolve the group if present. */
gimple *next; stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (dr));
gimple *stmt = DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (dr)); while (stmt_info)
while (stmt)
{ {
stmt_vec_info vinfo = vinfo_for_stmt (stmt); stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
next = DR_GROUP_NEXT_ELEMENT (vinfo); DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
DR_GROUP_FIRST_ELEMENT (vinfo) = NULL; DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
DR_GROUP_NEXT_ELEMENT (vinfo) = NULL; stmt_info = next;
stmt = next;
} }
return false; return false;
} }
@ -3281,7 +3277,7 @@ vect_small_gap_p (loop_vec_info loop_vinfo, data_reference *dr, poly_int64 gap)
HOST_WIDE_INT count HOST_WIDE_INT count
= estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo)); = estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
if (DR_GROUP_FIRST_ELEMENT (stmt_info)) if (DR_GROUP_FIRST_ELEMENT (stmt_info))
count *= DR_GROUP_SIZE (vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (stmt_info))); count *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
return estimated_poly_value (gap) <= count * vect_get_scalar_dr_size (dr); return estimated_poly_value (gap) <= count * vect_get_scalar_dr_size (dr);
} }
@ -3379,11 +3375,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
int comp_res; int comp_res;
poly_uint64 lower_bound; poly_uint64 lower_bound;
struct data_reference *dr_a, *dr_b; struct data_reference *dr_a, *dr_b;
gimple *dr_group_first_a, *dr_group_first_b;
tree segment_length_a, segment_length_b; tree segment_length_a, segment_length_b;
unsigned HOST_WIDE_INT access_size_a, access_size_b; unsigned HOST_WIDE_INT access_size_a, access_size_b;
unsigned int align_a, align_b; unsigned int align_a, align_b;
gimple *stmt_a, *stmt_b;
/* Ignore the alias if the VF we chose ended up being no greater /* Ignore the alias if the VF we chose ended up being no greater
than the dependence distance. */ than the dependence distance. */
@ -3409,15 +3403,15 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
} }
dr_a = DDR_A (ddr); dr_a = DDR_A (ddr);
stmt_a = vect_dr_stmt (DDR_A (ddr)); stmt_vec_info stmt_info_a = vect_dr_stmt (DDR_A (ddr));
dr_b = DDR_B (ddr); dr_b = DDR_B (ddr);
stmt_b = vect_dr_stmt (DDR_B (ddr)); stmt_vec_info stmt_info_b = vect_dr_stmt (DDR_B (ddr));
/* Skip the pair if inter-iteration dependencies are irrelevant /* Skip the pair if inter-iteration dependencies are irrelevant
and intra-iteration dependencies are guaranteed to be honored. */ and intra-iteration dependencies are guaranteed to be honored. */
if (ignore_step_p if (ignore_step_p
&& (vect_preserves_scalar_order_p (stmt_a, stmt_b) && (vect_preserves_scalar_order_p (stmt_info_a, stmt_info_b)
|| vectorizable_with_step_bound_p (dr_a, dr_b, &lower_bound))) || vectorizable_with_step_bound_p (dr_a, dr_b, &lower_bound)))
{ {
if (dump_enabled_p ()) if (dump_enabled_p ())
@ -3468,18 +3462,18 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
continue; continue;
} }
dr_group_first_a = DR_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_a)); stmt_vec_info dr_group_first_a = DR_GROUP_FIRST_ELEMENT (stmt_info_a);
if (dr_group_first_a) if (dr_group_first_a)
{ {
stmt_a = dr_group_first_a; stmt_info_a = dr_group_first_a;
dr_a = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_a)); dr_a = STMT_VINFO_DATA_REF (stmt_info_a);
} }
dr_group_first_b = DR_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_b)); stmt_vec_info dr_group_first_b = DR_GROUP_FIRST_ELEMENT (stmt_info_b);
if (dr_group_first_b) if (dr_group_first_b)
{ {
stmt_b = dr_group_first_b; stmt_info_b = dr_group_first_b;
dr_b = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_b)); dr_b = STMT_VINFO_DATA_REF (stmt_info_b);
} }
if (ignore_step_p) if (ignore_step_p)
@ -4734,10 +4728,9 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop,
/* Likewise for any of the data references in the stmt group. */ /* Likewise for any of the data references in the stmt group. */
else if (DR_GROUP_SIZE (stmt_info) > 1) else if (DR_GROUP_SIZE (stmt_info) > 1)
{ {
gimple *orig_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info sinfo = DR_GROUP_FIRST_ELEMENT (stmt_info);
do do
{ {
stmt_vec_info sinfo = vinfo_for_stmt (orig_stmt);
struct data_reference *sdr = STMT_VINFO_DATA_REF (sinfo); struct data_reference *sdr = STMT_VINFO_DATA_REF (sinfo);
if (!alias_sets_conflict_p (get_alias_set (aggr_type), if (!alias_sets_conflict_p (get_alias_set (aggr_type),
get_alias_set (DR_REF (sdr)))) get_alias_set (DR_REF (sdr))))
@ -4745,9 +4738,9 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop,
need_ref_all = true; need_ref_all = true;
break; break;
} }
orig_stmt = DR_GROUP_NEXT_ELEMENT (sinfo); sinfo = DR_GROUP_NEXT_ELEMENT (sinfo);
} }
while (orig_stmt); while (sinfo);
} }
aggr_ptr_type = build_pointer_type_for_mode (aggr_type, ptr_mode, aggr_ptr_type = build_pointer_type_for_mode (aggr_type, ptr_mode,
need_ref_all); need_ref_all);
@ -6345,19 +6338,18 @@ vect_record_grouped_load_vectors (gimple *stmt, vec<tree> result_chain)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt); stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
gimple *first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
gimple *next_stmt;
unsigned int i, gap_count; unsigned int i, gap_count;
tree tmp_data_ref; tree tmp_data_ref;
/* Put a permuted data-ref in the VECTORIZED_STMT field. /* Put a permuted data-ref in the VECTORIZED_STMT field.
Since we scan the chain starting from it's first node, their order Since we scan the chain starting from it's first node, their order
corresponds the order of data-refs in RESULT_CHAIN. */ corresponds the order of data-refs in RESULT_CHAIN. */
next_stmt = first_stmt; stmt_vec_info next_stmt_info = first_stmt_info;
gap_count = 1; gap_count = 1;
FOR_EACH_VEC_ELT (result_chain, i, tmp_data_ref) FOR_EACH_VEC_ELT (result_chain, i, tmp_data_ref)
{ {
if (!next_stmt) if (!next_stmt_info)
break; break;
/* Skip the gaps. Loads created for the gaps will be removed by dead /* Skip the gaps. Loads created for the gaps will be removed by dead
@ -6366,27 +6358,27 @@ vect_record_grouped_load_vectors (gimple *stmt, vec<tree> result_chain)
DR_GROUP_GAP is the number of steps in elements from the previous DR_GROUP_GAP is the number of steps in elements from the previous
access (if there is no gap DR_GROUP_GAP is 1). We skip loads that access (if there is no gap DR_GROUP_GAP is 1). We skip loads that
correspond to the gaps. */ correspond to the gaps. */
if (next_stmt != first_stmt if (next_stmt_info != first_stmt_info
&& gap_count < DR_GROUP_GAP (vinfo_for_stmt (next_stmt))) && gap_count < DR_GROUP_GAP (next_stmt_info))
{ {
gap_count++; gap_count++;
continue; continue;
} }
while (next_stmt) while (next_stmt_info)
{ {
stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref); stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref);
/* We assume that if VEC_STMT is not NULL, this is a case of multiple /* We assume that if VEC_STMT is not NULL, this is a case of multiple
copies, and we put the new vector statement in the first available copies, and we put the new vector statement in the first available
RELATED_STMT. */ RELATED_STMT. */
if (!STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt))) if (!STMT_VINFO_VEC_STMT (next_stmt_info))
STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)) = new_stmt_info; STMT_VINFO_VEC_STMT (next_stmt_info) = new_stmt_info;
else else
{ {
if (!DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) if (!DR_GROUP_SAME_DR_STMT (next_stmt_info))
{ {
stmt_vec_info prev_stmt_info stmt_vec_info prev_stmt_info
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)); = STMT_VINFO_VEC_STMT (next_stmt_info);
stmt_vec_info rel_stmt_info stmt_vec_info rel_stmt_info
= STMT_VINFO_RELATED_STMT (prev_stmt_info); = STMT_VINFO_RELATED_STMT (prev_stmt_info);
while (rel_stmt_info) while (rel_stmt_info)
@ -6399,12 +6391,12 @@ vect_record_grouped_load_vectors (gimple *stmt, vec<tree> result_chain)
} }
} }
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
gap_count = 1; gap_count = 1;
/* If NEXT_STMT accesses the same DR as the previous statement, /* If NEXT_STMT_INFO accesses the same DR as the previous statement,
put the same TMP_DATA_REF as its vectorized statement; otherwise put the same TMP_DATA_REF as its vectorized statement; otherwise
get the next data-ref from RESULT_CHAIN. */ get the next data-ref from RESULT_CHAIN. */
if (!next_stmt || !DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) if (!next_stmt_info || !DR_GROUP_SAME_DR_STMT (next_stmt_info))
break; break;
} }
} }
@ -6545,8 +6537,8 @@ vect_supportable_dr_alignment (struct data_reference *dr,
if (loop_vinfo if (loop_vinfo
&& STMT_SLP_TYPE (stmt_info) && STMT_SLP_TYPE (stmt_info)
&& !multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo) && !multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
* DR_GROUP_SIZE (vinfo_for_stmt * (DR_GROUP_SIZE
(DR_GROUP_FIRST_ELEMENT (stmt_info))), (DR_GROUP_FIRST_ELEMENT (stmt_info))),
TYPE_VECTOR_SUBPARTS (vectype))) TYPE_VECTOR_SUBPARTS (vectype)))
; ;
else if (!loop_vinfo else if (!loop_vinfo

57
gcc/tree-vect-loop.c
View File

@ -661,14 +661,14 @@ vect_fixup_reduc_chain (gimple *stmt)
REDUC_GROUP_SIZE (firstp) = REDUC_GROUP_SIZE (stmt_info); REDUC_GROUP_SIZE (firstp) = REDUC_GROUP_SIZE (stmt_info);
do do
{ {
stmtp = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); stmtp = STMT_VINFO_RELATED_STMT (stmt_info);
REDUC_GROUP_FIRST_ELEMENT (stmtp) = firstp; REDUC_GROUP_FIRST_ELEMENT (stmtp) = firstp;
stmt = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)); stmt_info = REDUC_GROUP_NEXT_ELEMENT (stmt_info);
if (stmt) if (stmt_info)
REDUC_GROUP_NEXT_ELEMENT (stmtp) REDUC_GROUP_NEXT_ELEMENT (stmtp)
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); = STMT_VINFO_RELATED_STMT (stmt_info);
} }
while (stmt); while (stmt_info);
STMT_VINFO_DEF_TYPE (stmtp) = vect_reduction_def; STMT_VINFO_DEF_TYPE (stmtp) = vect_reduction_def;
} }
@ -683,12 +683,12 @@ vect_fixup_scalar_cycles_with_patterns (loop_vec_info loop_vinfo)
FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo), i, first) FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo), i, first)
if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (first))) if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (first)))
{ {
gimple *next = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first)); stmt_vec_info next = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first));
while (next) while (next)
{ {
if (! STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (next))) if (! STMT_VINFO_IN_PATTERN_P (next))
break; break;
next = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); next = REDUC_GROUP_NEXT_ELEMENT (next);
} }
/* If not all stmt in the chain are patterns try to handle /* If not all stmt in the chain are patterns try to handle
the chain without patterns. */ the chain without patterns. */
@ -2188,7 +2188,7 @@ again:
vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0]; vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0];
if (! STMT_VINFO_GROUPED_ACCESS (vinfo)) if (! STMT_VINFO_GROUPED_ACCESS (vinfo))
continue; continue;
vinfo = vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (vinfo)); vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
unsigned int size = DR_GROUP_SIZE (vinfo); unsigned int size = DR_GROUP_SIZE (vinfo);
tree vectype = STMT_VINFO_VECTYPE (vinfo); tree vectype = STMT_VINFO_VECTYPE (vinfo);
if (! vect_store_lanes_supported (vectype, size, false) if (! vect_store_lanes_supported (vectype, size, false)
@ -2198,7 +2198,7 @@ again:
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node) FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node)
{ {
vinfo = SLP_TREE_SCALAR_STMTS (node)[0]; vinfo = SLP_TREE_SCALAR_STMTS (node)[0];
vinfo = vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (vinfo)); vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo); bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
size = DR_GROUP_SIZE (vinfo); size = DR_GROUP_SIZE (vinfo);
vectype = STMT_VINFO_VECTYPE (vinfo); vectype = STMT_VINFO_VECTYPE (vinfo);
@ -2527,7 +2527,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi,
struct loop *loop = (gimple_bb (phi))->loop_father; struct loop *loop = (gimple_bb (phi))->loop_father;
struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
enum tree_code code; enum tree_code code;
gimple *loop_use_stmt = NULL, *first, *next_stmt; gimple *loop_use_stmt = NULL;
stmt_vec_info use_stmt_info, current_stmt_info = NULL; stmt_vec_info use_stmt_info, current_stmt_info = NULL;
tree lhs; tree lhs;
imm_use_iterator imm_iter; imm_use_iterator imm_iter;
@ -2592,12 +2592,12 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi,
use_stmt_info = loop_info->lookup_stmt (loop_use_stmt); use_stmt_info = loop_info->lookup_stmt (loop_use_stmt);
if (current_stmt_info) if (current_stmt_info)
{ {
REDUC_GROUP_NEXT_ELEMENT (current_stmt_info) = loop_use_stmt; REDUC_GROUP_NEXT_ELEMENT (current_stmt_info) = use_stmt_info;
REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) REDUC_GROUP_FIRST_ELEMENT (use_stmt_info)
= REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
} }
else else
REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = loop_use_stmt; REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = use_stmt_info;
lhs = gimple_assign_lhs (loop_use_stmt); lhs = gimple_assign_lhs (loop_use_stmt);
current_stmt_info = use_stmt_info; current_stmt_info = use_stmt_info;
@ -2610,9 +2610,10 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi,
/* Swap the operands, if needed, to make the reduction operand be the second /* Swap the operands, if needed, to make the reduction operand be the second
operand. */ operand. */
lhs = PHI_RESULT (phi); lhs = PHI_RESULT (phi);
next_stmt = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); stmt_vec_info next_stmt_info = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
while (next_stmt) while (next_stmt_info)
{ {
gassign *next_stmt = as_a <gassign *> (next_stmt_info->stmt);
if (gimple_assign_rhs2 (next_stmt) == lhs) if (gimple_assign_rhs2 (next_stmt) == lhs)
{ {
tree op = gimple_assign_rhs1 (next_stmt); tree op = gimple_assign_rhs1 (next_stmt);
@ -2626,7 +2627,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi,
&& vect_valid_reduction_input_p (def_stmt_info)) && vect_valid_reduction_input_p (def_stmt_info))
{ {
lhs = gimple_assign_lhs (next_stmt); lhs = gimple_assign_lhs (next_stmt);
next_stmt = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = REDUC_GROUP_NEXT_ELEMENT (next_stmt_info);
continue; continue;
} }
@ -2663,13 +2664,14 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi,
} }
lhs = gimple_assign_lhs (next_stmt); lhs = gimple_assign_lhs (next_stmt);
next_stmt = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = REDUC_GROUP_NEXT_ELEMENT (next_stmt_info);
} }
/* Save the chain for further analysis in SLP detection. */ /* Save the chain for further analysis in SLP detection. */
first = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info); stmt_vec_info first_stmt_info
LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (first); = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
REDUC_GROUP_SIZE (vinfo_for_stmt (first)) = size; LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (first_stmt_info);
REDUC_GROUP_SIZE (first_stmt_info) = size;
return true; return true;
} }
@ -3254,12 +3256,12 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
} }
/* Dissolve group eventually half-built by vect_is_slp_reduction. */ /* Dissolve group eventually half-built by vect_is_slp_reduction. */
gimple *first = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (def_stmt)); stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (def_stmt));
while (first) while (first)
{ {
gimple *next = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first)); stmt_vec_info next = REDUC_GROUP_NEXT_ELEMENT (first);
REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (first)) = NULL; REDUC_GROUP_FIRST_ELEMENT (first) = NULL;
REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first)) = NULL; REDUC_GROUP_NEXT_ELEMENT (first) = NULL;
first = next; first = next;
} }
@ -6130,7 +6132,8 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
} }
if (REDUC_GROUP_FIRST_ELEMENT (stmt_info)) if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
gcc_assert (slp_node && REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt); gcc_assert (slp_node
&& REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info);
if (gimple_code (stmt) == GIMPLE_PHI) if (gimple_code (stmt) == GIMPLE_PHI)
{ {
@ -6784,8 +6787,8 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
tree neutral_op = NULL_TREE; tree neutral_op = NULL_TREE;
if (slp_node) if (slp_node)
neutral_op = neutral_op_for_slp_reduction neutral_op = neutral_op_for_slp_reduction
(slp_node_instance->reduc_phis, code, (slp_node_instance->reduc_phis, code,
REDUC_GROUP_FIRST_ELEMENT (stmt_info) != NULL); REDUC_GROUP_FIRST_ELEMENT (stmt_info) != NULL_STMT_VEC_INFO);
if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION) if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
{ {

2
gcc/tree-vect-patterns.c
View File

@ -820,7 +820,7 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
{ {
return (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def return (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
? STMT_VINFO_REDUC_TYPE (stmt_vinfo) != FOLD_LEFT_REDUCTION ? STMT_VINFO_REDUC_TYPE (stmt_vinfo) != FOLD_LEFT_REDUCTION
: REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL); : REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL_STMT_VEC_INFO);
} }
/* As above, but also require it to have code CODE and to be a reduction /* As above, but also require it to have code CODE and to be a reduction

74
gcc/tree-vect-slp.c
View File

@ -712,7 +712,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
int icode; int icode;
machine_mode optab_op2_mode; machine_mode optab_op2_mode;
machine_mode vec_mode; machine_mode vec_mode;
gimple *first_load = NULL, *prev_first_load = NULL; stmt_vec_info first_load = NULL, prev_first_load = NULL;
/* For every stmt in NODE find its def stmt/s. */ /* For every stmt in NODE find its def stmt/s. */
stmt_vec_info stmt_info; stmt_vec_info stmt_info;
@ -1692,8 +1692,7 @@ vect_attempt_slp_rearrange_stmts (slp_instance slp_instn)
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node) FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node)
{ {
stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0]; stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
first_stmt_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
= vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (first_stmt_info));
/* But we have to keep those permutations that are required because /* But we have to keep those permutations that are required because
of handling of gaps. */ of handling of gaps. */
if (known_eq (unrolling_factor, 1U) if (known_eq (unrolling_factor, 1U)
@ -1717,7 +1716,6 @@ vect_supported_load_permutation_p (slp_instance slp_instn)
unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_instn); unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_instn);
unsigned int i, j, k, next; unsigned int i, j, k, next;
slp_tree node; slp_tree node;
gimple *next_load;
if (dump_enabled_p ()) if (dump_enabled_p ())
{ {
@ -1766,26 +1764,25 @@ vect_supported_load_permutation_p (slp_instance slp_instn)
if (!SLP_TREE_LOAD_PERMUTATION (node).exists ()) if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
continue; continue;
bool subchain_p = true; bool subchain_p = true;
next_load = NULL; stmt_vec_info next_load_info = NULL;
stmt_vec_info load_info; stmt_vec_info load_info;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info) FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
{ {
if (j != 0 if (j != 0
&& (next_load != load_info && (next_load_info != load_info
|| DR_GROUP_GAP (load_info) != 1)) || DR_GROUP_GAP (load_info) != 1))
{ {
subchain_p = false; subchain_p = false;
break; break;
} }
next_load = DR_GROUP_NEXT_ELEMENT (load_info); next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
} }
if (subchain_p) if (subchain_p)
SLP_TREE_LOAD_PERMUTATION (node).release (); SLP_TREE_LOAD_PERMUTATION (node).release ();
else else
{ {
stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (node)[0]; stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (node)[0];
group_info group_info = DR_GROUP_FIRST_ELEMENT (group_info);
= vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (group_info));
unsigned HOST_WIDE_INT nunits; unsigned HOST_WIDE_INT nunits;
unsigned k, maxk = 0; unsigned k, maxk = 0;
FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), j, k) FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), j, k)
@ -1868,33 +1865,33 @@ static gimple *
vect_split_slp_store_group (gimple *first_stmt, unsigned group1_size) vect_split_slp_store_group (gimple *first_stmt, unsigned group1_size)
{ {
stmt_vec_info first_vinfo = vinfo_for_stmt (first_stmt); stmt_vec_info first_vinfo = vinfo_for_stmt (first_stmt);
gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_stmt); gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
gcc_assert (group1_size > 0); gcc_assert (group1_size > 0);
int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size; int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
gcc_assert (group2_size > 0); gcc_assert (group2_size > 0);
DR_GROUP_SIZE (first_vinfo) = group1_size; DR_GROUP_SIZE (first_vinfo) = group1_size;
gimple *stmt = first_stmt; stmt_vec_info stmt_info = first_vinfo;
for (unsigned i = group1_size; i > 1; i--) for (unsigned i = group1_size; i > 1; i--)
{ {
stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)); stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
gcc_assert (DR_GROUP_GAP (vinfo_for_stmt (stmt)) == 1); gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
} }
/* STMT is now the last element of the first group. */ /* STMT is now the last element of the first group. */
gimple *group2 = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)); stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)) = 0; DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
DR_GROUP_SIZE (vinfo_for_stmt (group2)) = group2_size; DR_GROUP_SIZE (group2) = group2_size;
for (stmt = group2; stmt; stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt))) for (stmt_info = group2; stmt_info;
stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
{ {
DR_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = group2; DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
gcc_assert (DR_GROUP_GAP (vinfo_for_stmt (stmt)) == 1); gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
} }
/* For the second group, the DR_GROUP_GAP is that before the original group, /* For the second group, the DR_GROUP_GAP is that before the original group,
plus skipping over the first vector. */ plus skipping over the first vector. */
DR_GROUP_GAP (vinfo_for_stmt (group2)) DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
= DR_GROUP_GAP (first_vinfo) + group1_size;
/* DR_GROUP_GAP of the first group now has to skip over the second group too. */ /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
DR_GROUP_GAP (first_vinfo) += group2_size; DR_GROUP_GAP (first_vinfo) += group2_size;
@ -1928,8 +1925,6 @@ vect_analyze_slp_instance (vec_info *vinfo,
slp_tree node; slp_tree node;
unsigned int group_size; unsigned int group_size;
tree vectype, scalar_type = NULL_TREE; tree vectype, scalar_type = NULL_TREE;
gimple *next;
stmt_vec_info next_info;
unsigned int i; unsigned int i;
vec<slp_tree> loads; vec<slp_tree> loads;
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
@ -1970,34 +1965,32 @@ vect_analyze_slp_instance (vec_info *vinfo,
/* Create a node (a root of the SLP tree) for the packed grouped stores. */ /* Create a node (a root of the SLP tree) for the packed grouped stores. */
scalar_stmts.create (group_size); scalar_stmts.create (group_size);
next = stmt; stmt_vec_info next_info = stmt_info;
if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
{ {
/* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */ /* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
while (next) while (next_info)
{ {
next_info = vinfo_for_stmt (next);
if (STMT_VINFO_IN_PATTERN_P (next_info) if (STMT_VINFO_IN_PATTERN_P (next_info)
&& STMT_VINFO_RELATED_STMT (next_info)) && STMT_VINFO_RELATED_STMT (next_info))
scalar_stmts.safe_push (STMT_VINFO_RELATED_STMT (next_info)); scalar_stmts.safe_push (STMT_VINFO_RELATED_STMT (next_info));
else else
scalar_stmts.safe_push (next_info); scalar_stmts.safe_push (next_info);
next = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); next_info = DR_GROUP_NEXT_ELEMENT (next_info);
} }
} }
else if (!dr && REDUC_GROUP_FIRST_ELEMENT (stmt_info)) else if (!dr && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
{ {
/* Collect the reduction stmts and store them in /* Collect the reduction stmts and store them in
SLP_TREE_SCALAR_STMTS. */ SLP_TREE_SCALAR_STMTS. */
while (next) while (next_info)
{ {
next_info = vinfo_for_stmt (next);
if (STMT_VINFO_IN_PATTERN_P (next_info) if (STMT_VINFO_IN_PATTERN_P (next_info)
&& STMT_VINFO_RELATED_STMT (next_info)) && STMT_VINFO_RELATED_STMT (next_info))
scalar_stmts.safe_push (STMT_VINFO_RELATED_STMT (next_info)); scalar_stmts.safe_push (STMT_VINFO_RELATED_STMT (next_info));
else else
scalar_stmts.safe_push (next_info); scalar_stmts.safe_push (next_info);
next = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); next_info = REDUC_GROUP_NEXT_ELEMENT (next_info);
} }
/* Mark the first element of the reduction chain as reduction to properly /* Mark the first element of the reduction chain as reduction to properly
transform the node. In the reduction analysis phase only the last transform the node. In the reduction analysis phase only the last
@ -2067,15 +2060,14 @@ vect_analyze_slp_instance (vec_info *vinfo,
vec<unsigned> load_permutation; vec<unsigned> load_permutation;
int j; int j;
stmt_vec_info load_info; stmt_vec_info load_info;
gimple *first_stmt;
bool this_load_permuted = false; bool this_load_permuted = false;
load_permutation.create (group_size); load_permutation.create (group_size);
first_stmt = DR_GROUP_FIRST_ELEMENT stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT
(SLP_TREE_SCALAR_STMTS (load_node)[0]); (SLP_TREE_SCALAR_STMTS (load_node)[0]);
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (load_node), j, load_info) FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (load_node), j, load_info)
{ {
int load_place = vect_get_place_in_interleaving_chain int load_place = vect_get_place_in_interleaving_chain
(load_info, first_stmt); (load_info, first_stmt_info);
gcc_assert (load_place != -1); gcc_assert (load_place != -1);
if (load_place != j) if (load_place != j)
this_load_permuted = true; this_load_permuted = true;
@ -2086,8 +2078,8 @@ vect_analyze_slp_instance (vec_info *vinfo,
a gap either because the group is larger than the SLP a gap either because the group is larger than the SLP
group-size or because there is a gap between the groups. */ group-size or because there is a gap between the groups. */
&& (known_eq (unrolling_factor, 1U) && (known_eq (unrolling_factor, 1U)
|| (group_size == DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)) || (group_size == DR_GROUP_SIZE (first_stmt_info)
&& DR_GROUP_GAP (vinfo_for_stmt (first_stmt)) == 0))) && DR_GROUP_GAP (first_stmt_info) == 0)))
{ {
load_permutation.release (); load_permutation.release ();
continue; continue;
@ -2122,11 +2114,9 @@ vect_analyze_slp_instance (vec_info *vinfo,
slp_tree load_node; slp_tree load_node;
FOR_EACH_VEC_ELT (loads, i, load_node) FOR_EACH_VEC_ELT (loads, i, load_node)
{ {
gimple *first_stmt = DR_GROUP_FIRST_ELEMENT stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
(SLP_TREE_SCALAR_STMTS (load_node)[0]); (SLP_TREE_SCALAR_STMTS (load_node)[0]);
stmt_vec_info stmt_vinfo = vinfo_for_stmt (first_stmt); /* Use SLP for strided accesses (or if we can't load-lanes). */
/* Use SLP for strided accesses (or if we
can't load-lanes). */
if (STMT_VINFO_STRIDED_P (stmt_vinfo) if (STMT_VINFO_STRIDED_P (stmt_vinfo)
|| ! vect_load_lanes_supported || ! vect_load_lanes_supported
(STMT_VINFO_VECTYPE (stmt_vinfo), (STMT_VINFO_VECTYPE (stmt_vinfo),
@ -2230,11 +2220,11 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size)
max_tree_size)) max_tree_size))
{ {
/* Dissolve reduction chain group. */ /* Dissolve reduction chain group. */
gimple *next, *stmt = first_element; gimple *stmt = first_element;
while (stmt) while (stmt)
{ {
stmt_vec_info vinfo = vinfo_for_stmt (stmt); stmt_vec_info vinfo = vinfo_for_stmt (stmt);
next = REDUC_GROUP_NEXT_ELEMENT (vinfo); stmt_vec_info next = REDUC_GROUP_NEXT_ELEMENT (vinfo);
REDUC_GROUP_FIRST_ELEMENT (vinfo) = NULL; REDUC_GROUP_FIRST_ELEMENT (vinfo) = NULL;
REDUC_GROUP_NEXT_ELEMENT (vinfo) = NULL; REDUC_GROUP_NEXT_ELEMENT (vinfo) = NULL;
stmt = next; stmt = next;
@ -3698,7 +3688,7 @@ vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
if (!STMT_VINFO_GROUPED_ACCESS (stmt_info)) if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
return false; return false;
stmt_info = vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (stmt_info)); stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
mode = TYPE_MODE (vectype); mode = TYPE_MODE (vectype);

173
gcc/tree-vect-stmts.c
View File

@ -978,7 +978,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
unsigned int inside_cost = 0, prologue_cost = 0; unsigned int inside_cost = 0, prologue_cost = 0;
gimple *first_stmt = STMT_VINFO_STMT (stmt_info); stmt_vec_info first_stmt_info = stmt_info;
bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info); bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
/* ??? Somehow we need to fix this at the callers. */ /* ??? Somehow we need to fix this at the callers. */
@ -998,12 +998,12 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
/* Grouped stores update all elements in the group at once, /* Grouped stores update all elements in the group at once,
so we want the DR for the first statement. */ so we want the DR for the first statement. */
if (!slp_node && grouped_access_p) if (!slp_node && grouped_access_p)
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
/* True if we should include any once-per-group costs as well as /* True if we should include any once-per-group costs as well as
the cost of the statement itself. For SLP we only get called the cost of the statement itself. For SLP we only get called
once per group anyhow. */ once per group anyhow. */
bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info)); bool first_stmt_p = (first_stmt_info == stmt_info);
/* We assume that the cost of a single store-lanes instruction is /* We assume that the cost of a single store-lanes instruction is
equivalent to the cost of DR_GROUP_SIZE separate stores. If a grouped equivalent to the cost of DR_GROUP_SIZE separate stores. If a grouped
@ -1014,7 +1014,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
{ {
/* Uses a high and low interleave or shuffle operations for each /* Uses a high and low interleave or shuffle operations for each
needed permute. */ needed permute. */
int group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); int group_size = DR_GROUP_SIZE (first_stmt_info);
int nstmts = ncopies * ceil_log2 (group_size) * group_size; int nstmts = ncopies * ceil_log2 (group_size) * group_size;
inside_cost = record_stmt_cost (cost_vec, nstmts, vec_perm, inside_cost = record_stmt_cost (cost_vec, nstmts, vec_perm,
stmt_info, 0, vect_body); stmt_info, 0, vect_body);
@ -1122,7 +1122,6 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies,
slp_tree slp_node, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
gimple *first_stmt = STMT_VINFO_STMT (stmt_info);
unsigned int inside_cost = 0, prologue_cost = 0; unsigned int inside_cost = 0, prologue_cost = 0;
bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info); bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
@ -1136,28 +1135,27 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies,
{ {
/* If the load is permuted then the alignment is determined by /* If the load is permuted then the alignment is determined by
the first group element not by the first scalar stmt DR. */ the first group element not by the first scalar stmt DR. */
gimple *stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
/* Record the cost for the permutation. */ /* Record the cost for the permutation. */
unsigned n_perms; unsigned n_perms;
unsigned assumed_nunits unsigned assumed_nunits
= vect_nunits_for_cost (STMT_VINFO_VECTYPE (stmt_info)); = vect_nunits_for_cost (STMT_VINFO_VECTYPE (first_stmt_info));
unsigned slp_vf = (ncopies * assumed_nunits) / instance->group_size; unsigned slp_vf = (ncopies * assumed_nunits) / instance->group_size;
vect_transform_slp_perm_load (slp_node, vNULL, NULL, vect_transform_slp_perm_load (slp_node, vNULL, NULL,
slp_vf, instance, true, slp_vf, instance, true,
&n_perms); &n_perms);
inside_cost += record_stmt_cost (cost_vec, n_perms, vec_perm, inside_cost += record_stmt_cost (cost_vec, n_perms, vec_perm,
stmt_info, 0, vect_body); first_stmt_info, 0, vect_body);
/* And adjust the number of loads performed. This handles /* And adjust the number of loads performed. This handles
redundancies as well as loads that are later dead. */ redundancies as well as loads that are later dead. */
auto_sbitmap perm (DR_GROUP_SIZE (stmt_info)); auto_sbitmap perm (DR_GROUP_SIZE (first_stmt_info));
bitmap_clear (perm); bitmap_clear (perm);
for (unsigned i = 0; for (unsigned i = 0;
i < SLP_TREE_LOAD_PERMUTATION (slp_node).length (); ++i) i < SLP_TREE_LOAD_PERMUTATION (slp_node).length (); ++i)
bitmap_set_bit (perm, SLP_TREE_LOAD_PERMUTATION (slp_node)[i]); bitmap_set_bit (perm, SLP_TREE_LOAD_PERMUTATION (slp_node)[i]);
ncopies = 0; ncopies = 0;
bool load_seen = false; bool load_seen = false;
for (unsigned i = 0; i < DR_GROUP_SIZE (stmt_info); ++i) for (unsigned i = 0; i < DR_GROUP_SIZE (first_stmt_info); ++i)
{ {
if (i % assumed_nunits == 0) if (i % assumed_nunits == 0)
{ {
@ -1171,19 +1169,21 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies,
if (load_seen) if (load_seen)
ncopies++; ncopies++;
gcc_assert (ncopies gcc_assert (ncopies
<= (DR_GROUP_SIZE (stmt_info) - DR_GROUP_GAP (stmt_info) <= (DR_GROUP_SIZE (first_stmt_info)
- DR_GROUP_GAP (first_stmt_info)
+ assumed_nunits - 1) / assumed_nunits); + assumed_nunits - 1) / assumed_nunits);
} }
/* Grouped loads read all elements in the group at once, /* Grouped loads read all elements in the group at once,
so we want the DR for the first statement. */ so we want the DR for the first statement. */
stmt_vec_info first_stmt_info = stmt_info;
if (!slp_node && grouped_access_p) if (!slp_node && grouped_access_p)
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
/* True if we should include any once-per-group costs as well as /* True if we should include any once-per-group costs as well as
the cost of the statement itself. For SLP we only get called the cost of the statement itself. For SLP we only get called
once per group anyhow. */ once per group anyhow. */
bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info)); bool first_stmt_p = (first_stmt_info == stmt_info);
/* We assume that the cost of a single load-lanes instruction is /* We assume that the cost of a single load-lanes instruction is
equivalent to the cost of DR_GROUP_SIZE separate loads. If a grouped equivalent to the cost of DR_GROUP_SIZE separate loads. If a grouped
@ -1194,7 +1194,7 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies,
{ {
/* Uses an even and odd extract operations or shuffle operations /* Uses an even and odd extract operations or shuffle operations
for each needed permute. */ for each needed permute. */
int group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); int group_size = DR_GROUP_SIZE (first_stmt_info);
int nstmts = ncopies * ceil_log2 (group_size) * group_size; int nstmts = ncopies * ceil_log2 (group_size) * group_size;
inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm, inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm,
stmt_info, 0, vect_body); stmt_info, 0, vect_body);
@ -2183,12 +2183,12 @@ get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
gimple *first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
data_reference *first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); data_reference *first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
unsigned int group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); unsigned int group_size = DR_GROUP_SIZE (first_stmt_info);
bool single_element_p = (stmt == first_stmt bool single_element_p = (stmt_info == first_stmt_info
&& !DR_GROUP_NEXT_ELEMENT (stmt_info)); && !DR_GROUP_NEXT_ELEMENT (stmt_info));
unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (vinfo_for_stmt (first_stmt)); unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
/* True if the vectorized statements would access beyond the last /* True if the vectorized statements would access beyond the last
@ -2315,14 +2315,14 @@ get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
*memory_access_type = VMAT_GATHER_SCATTER; *memory_access_type = VMAT_GATHER_SCATTER;
} }
if (vls_type != VLS_LOAD && first_stmt == stmt) if (vls_type != VLS_LOAD && first_stmt_info == stmt_info)
{ {
/* STMT is the leader of the group. Check the operands of all the /* STMT is the leader of the group. Check the operands of all the
stmts of the group. */ stmts of the group. */
gimple *next_stmt = DR_GROUP_NEXT_ELEMENT (stmt_info); stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
while (next_stmt) while (next_stmt_info)
{ {
tree op = vect_get_store_rhs (next_stmt); tree op = vect_get_store_rhs (next_stmt_info);
enum vect_def_type dt; enum vect_def_type dt;
if (!vect_is_simple_use (op, vinfo, &dt)) if (!vect_is_simple_use (op, vinfo, &dt))
{ {
@ -2331,7 +2331,7 @@ get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
"use not simple.\n"); "use not simple.\n");
return false; return false;
} }
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
} }
} }
@ -2482,7 +2482,7 @@ get_load_store_type (gimple *stmt, tree vectype, bool slp, bool masked_p,
traditional behavior until that can be fixed. */ traditional behavior until that can be fixed. */
if (*memory_access_type == VMAT_ELEMENTWISE if (*memory_access_type == VMAT_ELEMENTWISE
&& !STMT_VINFO_STRIDED_P (stmt_info) && !STMT_VINFO_STRIDED_P (stmt_info)
&& !(stmt == DR_GROUP_FIRST_ELEMENT (stmt_info) && !(stmt_info == DR_GROUP_FIRST_ELEMENT (stmt_info)
&& !DR_GROUP_NEXT_ELEMENT (stmt_info) && !DR_GROUP_NEXT_ELEMENT (stmt_info)
&& !pow2p_hwi (DR_GROUP_SIZE (stmt_info)))) && !pow2p_hwi (DR_GROUP_SIZE (stmt_info))))
{ {
@ -6195,13 +6195,13 @@ static tree
get_group_alias_ptr_type (gimple *first_stmt) get_group_alias_ptr_type (gimple *first_stmt)
{ {
struct data_reference *first_dr, *next_dr; struct data_reference *first_dr, *next_dr;
gimple *next_stmt;
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first_stmt)); stmt_vec_info next_stmt_info
while (next_stmt) = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (first_stmt));
while (next_stmt_info)
{ {
next_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (next_stmt)); next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
if (get_alias_set (DR_REF (first_dr)) if (get_alias_set (DR_REF (first_dr))
!= get_alias_set (DR_REF (next_dr))) != get_alias_set (DR_REF (next_dr)))
{ {
@ -6210,7 +6210,7 @@ get_group_alias_ptr_type (gimple *first_stmt)
"conflicting alias set types.\n"); "conflicting alias set types.\n");
return ptr_type_node; return ptr_type_node;
} }
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
} }
return reference_alias_ptr_type (DR_REF (first_dr)); return reference_alias_ptr_type (DR_REF (first_dr));
} }
@ -6248,7 +6248,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
gimple *ptr_incr = NULL; gimple *ptr_incr = NULL;
int ncopies; int ncopies;
int j; int j;
gimple *next_stmt, *first_stmt; stmt_vec_info first_stmt_info;
bool grouped_store; bool grouped_store;
unsigned int group_size, i; unsigned int group_size, i;
vec<tree> oprnds = vNULL; vec<tree> oprnds = vNULL;
@ -6400,13 +6400,13 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
&& (slp || memory_access_type != VMAT_CONTIGUOUS)); && (slp || memory_access_type != VMAT_CONTIGUOUS));
if (grouped_store) if (grouped_store)
{ {
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); group_size = DR_GROUP_SIZE (first_stmt_info);
} }
else else
{ {
first_stmt = stmt; first_stmt_info = stmt_info;
first_dr = dr; first_dr = dr;
group_size = vec_num = 1; group_size = vec_num = 1;
} }
@ -6584,10 +6584,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
} }
if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
{ DR_GROUP_STORE_COUNT (DR_GROUP_FIRST_ELEMENT (stmt_info))++;
gimple *group_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info);
DR_GROUP_STORE_COUNT (vinfo_for_stmt (group_stmt))++;
}
if (grouped_store) if (grouped_store)
{ {
@ -6596,8 +6593,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
/* We vectorize all the stmts of the interleaving group when we /* We vectorize all the stmts of the interleaving group when we
reach the last stmt in the group. */ reach the last stmt in the group. */
if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt)) if (DR_GROUP_STORE_COUNT (first_stmt_info)
< DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)) < DR_GROUP_SIZE (first_stmt_info)
&& !slp) && !slp)
{ {
*vec_stmt = NULL; *vec_stmt = NULL;
@ -6610,17 +6607,18 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
/* VEC_NUM is the number of vect stmts to be created for this /* VEC_NUM is the number of vect stmts to be created for this
group. */ group. */
vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0]; first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
gcc_assert (DR_GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt)) == first_stmt); gcc_assert (DR_GROUP_FIRST_ELEMENT (first_stmt_info)
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); == first_stmt_info);
op = vect_get_store_rhs (first_stmt); first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
op = vect_get_store_rhs (first_stmt_info);
} }
else else
/* VEC_NUM is the number of vect stmts to be created for this /* VEC_NUM is the number of vect stmts to be created for this
group. */ group. */
vec_num = group_size; vec_num = group_size;
ref_type = get_group_alias_ptr_type (first_stmt); ref_type = get_group_alias_ptr_type (first_stmt_info);
} }
else else
ref_type = reference_alias_ptr_type (DR_REF (first_dr)); ref_type = reference_alias_ptr_type (DR_REF (first_dr));
@ -6759,7 +6757,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
prev_stmt_info = NULL; prev_stmt_info = NULL;
alias_off = build_int_cst (ref_type, 0); alias_off = build_int_cst (ref_type, 0);
next_stmt = first_stmt; stmt_vec_info next_stmt_info = first_stmt_info;
for (g = 0; g < group_size; g++) for (g = 0; g < group_size; g++)
{ {
running_off = offvar; running_off = offvar;
@ -6780,7 +6778,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
for (j = 0; j < ncopies; j++) for (j = 0; j < ncopies; j++)
{ {
/* We've set op and dt above, from vect_get_store_rhs, /* We've set op and dt above, from vect_get_store_rhs,
and first_stmt == stmt. */ and first_stmt_info == stmt_info. */
if (j == 0) if (j == 0)
{ {
if (slp) if (slp)
@ -6791,8 +6789,9 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
} }
else else
{ {
op = vect_get_store_rhs (next_stmt); op = vect_get_store_rhs (next_stmt_info);
vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt); vec_oprnd = vect_get_vec_def_for_operand
(op, next_stmt_info);
} }
} }
else else
@ -6866,7 +6865,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
} }
} }
} }
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
if (slp) if (slp)
break; break;
} }
@ -6985,19 +6984,20 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and
OPRNDS are of size 1. */ OPRNDS are of size 1. */
next_stmt = first_stmt; stmt_vec_info next_stmt_info = first_stmt_info;
for (i = 0; i < group_size; i++) for (i = 0; i < group_size; i++)
{ {
/* Since gaps are not supported for interleaved stores, /* Since gaps are not supported for interleaved stores,
DR_GROUP_SIZE is the exact number of stmts in the chain. DR_GROUP_SIZE is the exact number of stmts in the chain.
Therefore, NEXT_STMT can't be NULL_TREE. In case that Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
there is no interleaving, DR_GROUP_SIZE is 1, and only one that there is no interleaving, DR_GROUP_SIZE is 1,
iteration of the loop will be executed. */ and only one iteration of the loop will be executed. */
op = vect_get_store_rhs (next_stmt); op = vect_get_store_rhs (next_stmt_info);
vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt); vec_oprnd = vect_get_vec_def_for_operand
(op, next_stmt_info);
dr_chain.quick_push (vec_oprnd); dr_chain.quick_push (vec_oprnd);
oprnds.quick_push (vec_oprnd); oprnds.quick_push (vec_oprnd);
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
} }
if (mask) if (mask)
vec_mask = vect_get_vec_def_for_operand (mask, stmt, vec_mask = vect_get_vec_def_for_operand (mask, stmt,
@ -7029,7 +7029,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
} }
else else
dataref_ptr dataref_ptr
= vect_create_data_ref_ptr (first_stmt, aggr_type, = vect_create_data_ref_ptr (first_stmt_info, aggr_type,
simd_lane_access_p ? loop : NULL, simd_lane_access_p ? loop : NULL,
offset, &dummy, gsi, &ptr_incr, offset, &dummy, gsi, &ptr_incr,
simd_lane_access_p, &inv_p, simd_lane_access_p, &inv_p,
@ -7132,7 +7132,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
&result_chain); &result_chain);
} }
next_stmt = first_stmt; stmt_vec_info next_stmt_info = first_stmt_info;
for (i = 0; i < vec_num; i++) for (i = 0; i < vec_num; i++)
{ {
unsigned align, misalign; unsigned align, misalign;
@ -7249,8 +7249,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi,
if (slp) if (slp)
continue; continue;
next_stmt = DR_GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
if (!next_stmt) if (!next_stmt_info)
break; break;
} }
} }
@ -7423,7 +7423,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
gphi *phi = NULL; gphi *phi = NULL;
vec<tree> dr_chain = vNULL; vec<tree> dr_chain = vNULL;
bool grouped_load = false; bool grouped_load = false;
gimple *first_stmt; stmt_vec_info first_stmt_info;
stmt_vec_info first_stmt_info_for_drptr = NULL; stmt_vec_info first_stmt_info_for_drptr = NULL;
bool inv_p; bool inv_p;
bool compute_in_loop = false; bool compute_in_loop = false;
@ -7565,8 +7565,8 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
gcc_assert (!nested_in_vect_loop); gcc_assert (!nested_in_vect_loop);
gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info)); gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); group_size = DR_GROUP_SIZE (first_stmt_info);
if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
slp_perm = true; slp_perm = true;
@ -7696,25 +7696,26 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
if (grouped_load) if (grouped_load)
{ {
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
} }
else else
{ {
first_stmt = stmt; first_stmt_info = stmt_info;
first_dr = dr; first_dr = dr;
} }
if (slp && grouped_load) if (slp && grouped_load)
{ {
group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); group_size = DR_GROUP_SIZE (first_stmt_info);
ref_type = get_group_alias_ptr_type (first_stmt); ref_type = get_group_alias_ptr_type (first_stmt_info);
} }
else else
{ {
if (grouped_load) if (grouped_load)
cst_offset cst_offset
= (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype))) = (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)))
* vect_get_place_in_interleaving_chain (stmt, first_stmt)); * vect_get_place_in_interleaving_chain (stmt,
first_stmt_info));
group_size = 1; group_size = 1;
ref_type = reference_alias_ptr_type (DR_REF (dr)); ref_type = reference_alias_ptr_type (DR_REF (dr));
} }
@ -7924,19 +7925,19 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
if (grouped_load) if (grouped_load)
{ {
first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); group_size = DR_GROUP_SIZE (first_stmt_info);
/* For SLP vectorization we directly vectorize a subchain /* For SLP vectorization we directly vectorize a subchain
without permutation. */ without permutation. */
if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0]; first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
/* For BB vectorization always use the first stmt to base /* For BB vectorization always use the first stmt to base
the data ref pointer on. */ the data ref pointer on. */
if (bb_vinfo) if (bb_vinfo)
first_stmt_info_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0]; first_stmt_info_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0];
/* Check if the chain of loads is already vectorized. */ /* Check if the chain of loads is already vectorized. */
if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt)) if (STMT_VINFO_VEC_STMT (first_stmt_info)
/* For SLP we would need to copy over SLP_TREE_VEC_STMTS. /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
??? But we can only do so if there is exactly one ??? But we can only do so if there is exactly one
as we have no way to get at the rest. Leave the CSE as we have no way to get at the rest. Leave the CSE
@ -7950,7 +7951,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
*vec_stmt = STMT_VINFO_VEC_STMT (stmt_info); *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
return true; return true;
} }
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
group_gap_adj = 0; group_gap_adj = 0;
/* VEC_NUM is the number of vect stmts to be created for this group. */ /* VEC_NUM is the number of vect stmts to be created for this group. */
@ -7979,11 +7980,11 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
else else
vec_num = group_size; vec_num = group_size;
ref_type = get_group_alias_ptr_type (first_stmt); ref_type = get_group_alias_ptr_type (first_stmt_info);
} }
else else
{ {
first_stmt = stmt; first_stmt_info = stmt_info;
first_dr = dr; first_dr = dr;
group_size = vec_num = 1; group_size = vec_num = 1;
group_gap_adj = 0; group_gap_adj = 0;
@ -8120,7 +8121,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
|| alignment_support_scheme == dr_explicit_realign) || alignment_support_scheme == dr_explicit_realign)
&& !compute_in_loop) && !compute_in_loop)
{ {
msq = vect_setup_realignment (first_stmt, gsi, &realignment_token, msq = vect_setup_realignment (first_stmt_info, gsi, &realignment_token,
alignment_support_scheme, NULL_TREE, alignment_support_scheme, NULL_TREE,
&at_loop); &at_loop);
if (alignment_support_scheme == dr_explicit_realign_optimized) if (alignment_support_scheme == dr_explicit_realign_optimized)
@ -8184,7 +8185,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
inv_p = false; inv_p = false;
} }
else if (first_stmt_info_for_drptr else if (first_stmt_info_for_drptr
&& first_stmt != first_stmt_info_for_drptr) && first_stmt_info != first_stmt_info_for_drptr)
{ {
dataref_ptr dataref_ptr
= vect_create_data_ref_ptr (first_stmt_info_for_drptr, = vect_create_data_ref_ptr (first_stmt_info_for_drptr,
@ -8209,7 +8210,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
} }
else else
dataref_ptr dataref_ptr
= vect_create_data_ref_ptr (first_stmt, aggr_type, at_loop, = vect_create_data_ref_ptr (first_stmt_info, aggr_type, at_loop,
offset, &dummy, gsi, &ptr_incr, offset, &dummy, gsi, &ptr_incr,
simd_lane_access_p, &inv_p, simd_lane_access_p, &inv_p,
byte_offset, bump); byte_offset, bump);
@ -8388,7 +8389,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype)); tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
if (compute_in_loop) if (compute_in_loop)
msq = vect_setup_realignment (first_stmt, gsi, msq = vect_setup_realignment (first_stmt_info, gsi,
&realignment_token, &realignment_token,
dr_explicit_realign, dr_explicit_realign,
dataref_ptr, NULL); dataref_ptr, NULL);
@ -9708,8 +9709,7 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
one are skipped, and there vec_stmt_info shouldn't be freed one are skipped, and there vec_stmt_info shouldn't be freed
meanwhile. */ meanwhile. */
*grouped_store = true; *grouped_store = true;
stmt_vec_info group_info stmt_vec_info group_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
= vinfo_for_stmt (DR_GROUP_FIRST_ELEMENT (stmt_info));
if (DR_GROUP_STORE_COUNT (group_info) == DR_GROUP_SIZE (group_info)) if (DR_GROUP_STORE_COUNT (group_info) == DR_GROUP_SIZE (group_info))
is_store = true; is_store = true;
} }
@ -9817,14 +9817,13 @@ void
vect_remove_stores (gimple *first_stmt) vect_remove_stores (gimple *first_stmt)
{ {
gimple *next = first_stmt; gimple *next = first_stmt;
gimple *tmp;
gimple_stmt_iterator next_si; gimple_stmt_iterator next_si;
while (next) while (next)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (next); stmt_vec_info stmt_info = vinfo_for_stmt (next);
tmp = DR_GROUP_NEXT_ELEMENT (stmt_info); stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (stmt_info);
if (is_pattern_stmt_p (stmt_info)) if (is_pattern_stmt_p (stmt_info))
next = STMT_VINFO_RELATED_STMT (stmt_info); next = STMT_VINFO_RELATED_STMT (stmt_info);
/* Free the attached stmt_vec_info and remove the stmt. */ /* Free the attached stmt_vec_info and remove the stmt. */

4
gcc/tree-vectorizer.h
View File

@ -871,9 +871,9 @@ struct _stmt_vec_info {
/* Interleaving and reduction chains info. */ /* Interleaving and reduction chains info. */
/* First element in the group. */ /* First element in the group. */
gimple *first_element; stmt_vec_info first_element;
/* Pointer to the next element in the group. */ /* Pointer to the next element in the group. */
gimple *next_element; stmt_vec_info next_element;
/* For data-refs, in case that two or more stmts share data-ref, this is the /* For data-refs, in case that two or more stmts share data-ref, this is the
pointer to the previously detected stmt with the same dr. */ pointer to the previously detected stmt with the same dr. */
gimple *same_dr_stmt; gimple *same_dr_stmt;