[PATCH] Make {SLP_TREE,STMT_VINFO}_VEC_STMTS a vector of gimple *
Richard Biener
rguenther@suse.de
Wed Jun 10 12:16:05 GMT 2020
This makes {SLP_TREE,STMT_VINFO}_VEC_STMTS a vector of gimple * and
not allocate a stmt_vec_info for vectorizer generated stmts since
this is now possible after removing the only use which was chaining
of vector stmts via STMT_VINFO_RELATED_STMT.
This also removes all stmt_vec_info allocations done for vector
stmts, the remaining ones are for stmts in the scalar IL and for
patterns which are not part of the IL. Thus after this the stmt
UIDs inside a basic-block are suitable for dominance checking
if you ignore (or lazy-fill) UIDs of zero of the vector stmts
inserted during transform. This property is ensured by a new
flag set when pattern analysis is complete.
Bootstrap and regtest running on x86_64-unknown-linux-gnu.
Richard.
2020-06-10 Richard Biener <rguenther@suse.de>
* tree-vectorizer.h (_slp_tree::vec_stmts): Make it a vector
of gimple * stmts.
(_stmt_vec_info::vec_stmts): Likewise.
(vec_info::stmt_vec_info_ro): New flag.
(vect_finish_replace_stmt): Adjust declaration.
(vect_finish_stmt_generation): Likewise.
(vectorizable_induction): Likewise.
(vect_transform_reduction): Likewise.
(vectorizable_lc_phi): Likewise.
* tree-vect-data-refs.c (vect_create_data_ref_ptr): Do not
allocate stmt infos for increments.
(vect_record_grouped_load_vectors): Adjust.
* tree-vect-loop.c (vect_create_epilog_for_reduction): Likewise.
(vectorize_fold_left_reduction): Likewise.
(vect_transform_reduction): Likewise.
(vect_transform_cycle_phi): Likewise.
(vectorizable_lc_phi): Likewise.
(vectorizable_induction): Likewise.
(vectorizable_live_operation): Likewise.
(vect_transform_loop): Likewise.
* tree-vect-patterns.c (vect_pattern_recog): Set stmt_vec_info_ro.
* tree-vect-slp.c (vect_get_slp_vect_def): Adjust.
(vect_get_slp_defs): Likewise.
(vect_transform_slp_perm_load): Likewise.
(vect_schedule_slp_instance): Likewise.
(vectorize_slp_instance_root_stmt): Likewise.
* tree-vect-stmts.c (vect_get_vec_defs_for_operand): Likewise.
(vect_finish_stmt_generation_1): Do not allocate a stmt info.
(vect_finish_replace_stmt): Do not return anything.
(vect_finish_stmt_generation): Likewise.
(vect_build_gather_load_calls): Adjust.
(vectorizable_bswap): Likewise.
(vectorizable_call): Likewise.
(vectorizable_simd_clone_call): Likewise.
(vect_create_vectorized_demotion_stmts): Likewise.
(vectorizable_conversion): Likewise.
(vectorizable_assignment): Likewise.
(vectorizable_shift): Likewise.
(vectorizable_operation): Likewise.
(vectorizable_scan_store): Likewise.
(vectorizable_store): Likewise.
(vectorizable_load): Likewise.
(vectorizable_condition): Likewise.
(vectorizable_comparison): Likewise.
(vect_transform_stmt): Likewise.
* tree-vectorizer.c (vec_info::vec_info): Initialize
stmt_vec_info_ro.
(vec_info::replace_stmt): Copy over stmt UID rather than
unsetting/setting a stmt info allocating a new UID.
(vec_info::set_vinfo_for_stmt): Assert !stmt_vec_info_ro.
---
gcc/tree-vect-data-refs.c | 8 +-
gcc/tree-vect-loop.c | 112 ++++-----
gcc/tree-vect-patterns.c | 3 +
gcc/tree-vect-slp.c | 47 ++--
gcc/tree-vect-stmts.c | 461 ++++++++++++++++----------------------
gcc/tree-vectorizer.c | 5 +-
gcc/tree-vectorizer.h | 26 +--
7 files changed, 282 insertions(+), 380 deletions(-)
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 7edd9ebe3b6..39d5a1b554c 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -4896,7 +4896,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
aggr_ptr, loop, &incr_gsi, insert_after,
&indx_before_incr, &indx_after_incr);
incr = gsi_stmt (incr_gsi);
- loop_vinfo->add_stmt (incr);
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
@@ -4926,7 +4925,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
containing_loop, &incr_gsi, insert_after, &indx_before_incr,
&indx_after_incr);
incr = gsi_stmt (incr_gsi);
- loop_vinfo->add_stmt (incr);
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
@@ -6407,7 +6405,7 @@ vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
for each vector to the associated scalar statement. */
void
-vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
+vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info,
vec<tree> result_chain)
{
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
@@ -6441,10 +6439,10 @@ vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
DR_GROUP_SAME_DR_STMT. */
if (next_stmt_info)
{
- stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref);
+ gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref);
/* We assume that if VEC_STMT is not NULL, this is a case of multiple
copies, and we put the new vector statement last. */
- STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt);
next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
gap_count = 1;
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index cc3d391d7c2..f0b33258ac5 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -4585,7 +4585,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
{
if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
{
- gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]->stmt;
+ gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0];
gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
ccompares.safe_push
(std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt)),
@@ -4636,7 +4636,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
/* Create a vector phi node. */
tree new_phi_tree = make_ssa_name (cr_index_vector_type);
new_phi = create_phi_node (new_phi_tree, loop->header);
- loop_vinfo->add_stmt (new_phi);
add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
loop_preheader_edge (loop), UNKNOWN_LOCATION);
@@ -4663,9 +4662,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
new_phi_tree, indx_before_incr);
}
gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
- stmt_vec_info index_vec_info
- = loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (new_phi_tree));
- STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
/* Update the phi with the vec cond. */
induction_index = new_phi_tree;
@@ -4712,17 +4708,16 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
if (slp_node)
def = vect_get_slp_vect_def (slp_node, i);
else
- def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]->stmt);
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]);
for (j = 0; j < ncopies; j++)
{
tree new_def = copy_ssa_name (def);
phi = create_phi_node (new_def, exit_bb);
- loop_vinfo->add_stmt (phi);
if (j == 0)
new_phis.quick_push (phi);
else
{
- def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]->stmt);
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]);
new_phis.quick_push (phi);
}
@@ -5446,10 +5441,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
if (nested_in_vect_loop)
{
- stmt_vec_info epilog_stmt_info = loop_vinfo->add_stmt (epilog_stmt);
- STMT_VINFO_RELATED_STMT (epilog_stmt_info)
- = STMT_VINFO_RELATED_STMT (loop_vinfo->lookup_stmt (new_phi));
-
if (!double_reduc)
scalar_results.quick_push (new_temp);
else
@@ -5666,7 +5657,7 @@ static bool
vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
gimple *reduc_def_stmt,
tree_code code, internal_fn reduc_fn,
tree ops[3], tree vectype_in,
@@ -5674,7 +5665,6 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
{
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
- stmt_vec_info new_stmt_info = NULL;
internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
int ncopies;
@@ -5785,23 +5775,22 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
if (i == vec_num - 1)
{
gimple_set_lhs (new_stmt, scalar_dest);
- new_stmt_info = vect_finish_replace_stmt (loop_vinfo,
- scalar_dest_def_info,
- new_stmt);
+ vect_finish_replace_stmt (loop_vinfo,
+ scalar_dest_def_info,
+ new_stmt);
}
else
- new_stmt_info = vect_finish_stmt_generation (loop_vinfo,
- scalar_dest_def_info,
- new_stmt, gsi);
+ vect_finish_stmt_generation (loop_vinfo,
+ scalar_dest_def_info,
+ new_stmt, gsi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
-
- if (!slp_node)
- {
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
- *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
+ else
+ {
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+ *vec_stmt = new_stmt;
+ }
}
return true;
@@ -6822,7 +6811,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
bool
vect_transform_reduction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node)
+ gimple **vec_stmt, slp_tree slp_node)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -6883,7 +6872,6 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
/* Transform. */
- stmt_vec_info new_stmt_info = NULL;
tree new_temp = NULL_TREE;
auto_vec<tree> vec_oprnds0;
auto_vec<tree> vec_oprnds1;
@@ -6938,6 +6926,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
{
+ gimple *new_stmt;
tree vop[3] = { def0, vec_oprnds1[i], NULL_TREE };
if (masked_loop_p && !mask_by_cond_expr)
{
@@ -6954,8 +6943,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
new_temp = make_ssa_name (vec_dest, call);
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
- new_stmt_info = vect_finish_stmt_generation (loop_vinfo,
- stmt_info, call, gsi);
+ vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi);
+ new_stmt = call;
}
else
{
@@ -6969,28 +6958,27 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
build_vect_cond_expr (code, vop, mask, gsi);
}
- gassign *new_stmt = gimple_build_assign (vec_dest, code,
- vop[0], vop[1], vop[2]);
+ new_stmt = gimple_build_assign (vec_dest, code,
+ vop[0], vop[1], vop[2]);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info = vect_finish_stmt_generation (loop_vinfo, stmt_info,
- new_stmt, gsi);
+ vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
}
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else if (single_defuse_cycle
&& i < ncopies - 1)
{
if (reduc_index == 0)
- vec_oprnds0.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ vec_oprnds0.safe_push (gimple_get_lhs (new_stmt));
else if (reduc_index == 1)
- vec_oprnds1.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ vec_oprnds1.safe_push (gimple_get_lhs (new_stmt));
else if (reduc_index == 2)
- vec_oprnds2.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ vec_oprnds2.safe_push (gimple_get_lhs (new_stmt));
}
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -7003,7 +6991,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
bool
vect_transform_cycle_phi (loop_vec_info loop_vinfo,
- stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
+ stmt_vec_info stmt_info, gimple **vec_stmt,
slp_tree slp_node, slp_instance slp_node_instance)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
@@ -7138,7 +7126,6 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
/* Create the reduction-phi that defines the reduction
operand. */
gphi *new_phi = create_phi_node (vec_dest, loop->header);
- stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
/* Set the loop-entry arg of the reduction-phi. */
if (j != 0 && nested_cycle)
@@ -7149,12 +7136,12 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
/* The loop-latch arg is set in epilogue processing. */
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi);
else
{
if (j == 0)
- *vec_stmt = new_phi_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info);
+ *vec_stmt = new_phi;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi);
}
}
}
@@ -7166,7 +7153,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
bool
vectorizable_lc_phi (loop_vec_info loop_vinfo,
- stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
+ stmt_vec_info stmt_info, gimple **vec_stmt,
slp_tree slp_node)
{
if (!loop_vinfo
@@ -7198,11 +7185,10 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo,
/* Create the vectorized LC PHI node. */
gphi *new_phi = create_phi_node (vec_dest, bb);
add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
- stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi);
}
if (!slp_node)
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
@@ -7262,7 +7248,7 @@ bool
vectorizable_induction (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -7495,8 +7481,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
/* Create the induction-phi that defines the induction-operand. */
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
induction_phi = create_phi_node (vec_dest, iv_loop->header);
- stmt_vec_info induction_phi_info
- = loop_vinfo->add_stmt (induction_phi);
induc_def = PHI_RESULT (induction_phi);
/* Create the iv update inside the loop */
@@ -7505,7 +7489,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
vec_def = gimple_build (&stmts,
PLUS_EXPR, step_vectype, vec_def, vec_step);
vec_def = gimple_convert (&stmts, vectype, vec_def);
- loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (vec_def));
gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
/* Set the arguments of the phi node: */
@@ -7513,7 +7496,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
UNKNOWN_LOCATION);
- SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi);
}
/* Fill up to the number of vectors we need for the whole group. */
nivs = least_common_multiple (group_size,
@@ -7545,7 +7528,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
step_vectype, NULL);
for (; ivn < nvects; ++ivn)
{
- gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt;
+ gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs];
tree def;
if (gimple_code (iv) == GIMPLE_PHI)
def = gimple_phi_result (iv);
@@ -7563,8 +7546,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING);
}
- SLP_TREE_VEC_STMTS (slp_node).quick_push
- (loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (def)));
+ SLP_TREE_VEC_STMTS (slp_node)
+ .quick_push (SSA_NAME_DEF_STMT (def));
}
}
@@ -7595,7 +7578,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
new_bb = gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop),
new_stmt);
gcc_assert (!new_bb);
- loop_vinfo->add_stmt (new_stmt);
}
}
else
@@ -7701,7 +7683,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
/* Create the induction-phi that defines the induction-operand. */
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
induction_phi = create_phi_node (vec_dest, iv_loop->header);
- stmt_vec_info induction_phi_info = loop_vinfo->add_stmt (induction_phi);
induc_def = PHI_RESULT (induction_phi);
/* Create the iv update inside the loop */
@@ -7711,15 +7692,14 @@ vectorizable_induction (loop_vec_info loop_vinfo,
vec_def = gimple_convert (&stmts, vectype, vec_def);
gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
new_stmt = SSA_NAME_DEF_STMT (vec_def);
- stmt_vec_info new_stmt_info = loop_vinfo->add_stmt (new_stmt);
/* Set the arguments of the phi node: */
add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
UNKNOWN_LOCATION);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi_info);
- *vec_stmt = induction_phi_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi);
+ *vec_stmt = induction_phi;
/* In case that vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
@@ -7768,8 +7748,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
new_stmt = SSA_NAME_DEF_STMT (vec_def);
- new_stmt_info = loop_vinfo->add_stmt (new_stmt);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
@@ -7944,7 +7923,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo,
gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
/* Get the correct slp vectorized stmt. */
- gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]->stmt;
+ gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry];
if (gphi *phi = dyn_cast <gphi *> (vec_stmt))
vec_lhs = gimple_phi_result (phi);
else
@@ -7957,7 +7936,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo,
else
{
/* For multiple copies, get the last copy. */
- vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()->stmt);
+ vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ());
/* Get the last lane in the vector. */
bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitsize);
@@ -8683,6 +8662,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
}
else
{
+ /* Ignore vector stmts created in the outer loop. */
stmt_info = loop_vinfo->lookup_stmt (stmt);
/* vector stmts created in the outer-loop during vectorization of
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index a1fd67341d7..930f47e0742 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -5503,4 +5503,7 @@ vect_pattern_recog (vec_info *vinfo)
&vect_vect_recog_func_ptrs[j], stmt_info);
}
}
+
+ /* After this no more add_stmt calls are allowed. */
+ vinfo->stmt_vec_info_ro = true;
}
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 44944bf8964..0217a524f05 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -3666,7 +3666,7 @@ tree
vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
{
if (SLP_TREE_VEC_STMTS (slp_node).exists ())
- return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt);
+ return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]);
else
return SLP_TREE_VEC_DEFS (slp_node)[i];
}
@@ -3680,9 +3680,9 @@ vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
{
unsigned j;
- stmt_vec_info vec_def_stmt_info;
- FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt_info)
- vec_defs->quick_push (gimple_get_lhs (vec_def_stmt_info->stmt));
+ gimple *vec_def_stmt;
+ FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt)
+ vec_defs->quick_push (gimple_get_lhs (vec_def_stmt));
}
else
vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
@@ -3880,7 +3880,7 @@ vect_transform_slp_perm_load (vec_info *vinfo,
/* Generate the permute statement if necessary. */
tree first_vec = dr_chain[first_vec_index + ri];
tree second_vec = dr_chain[second_vec_index + ri];
- stmt_vec_info perm_stmt_info;
+ gimple *perm_stmt;
if (! noop_p)
{
gassign *stmt = as_a <gassign *> (stmt_info->stmt);
@@ -3888,23 +3888,20 @@ vect_transform_slp_perm_load (vec_info *vinfo,
= vect_create_destination_var (gimple_assign_lhs (stmt),
vectype);
perm_dest = make_ssa_name (perm_dest);
- gassign *perm_stmt
+ perm_stmt
= gimple_build_assign (perm_dest, VEC_PERM_EXPR,
first_vec, second_vec,
mask_vec);
- perm_stmt_info
- = vect_finish_stmt_generation (vinfo,
- stmt_info, perm_stmt,
- gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
+ gsi);
}
else
/* If mask was NULL_TREE generate the requested
identity transform. */
- perm_stmt_info = vinfo->lookup_def (first_vec);
+ perm_stmt = SSA_NAME_DEF_STMT (first_vec);
/* Store the vector statement in NODE. */
- SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++]
- = perm_stmt_info;
+ SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt;
}
}
@@ -3995,8 +3992,8 @@ vect_schedule_slp_instance (vec_info *vinfo,
}
if (ocode != ERROR_MARK)
{
- vec<stmt_vec_info> v0;
- vec<stmt_vec_info> v1;
+ vec<gimple *> v0;
+ vec<gimple *> v1;
unsigned j;
tree tmask = NULL_TREE;
vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
@@ -4037,11 +4034,11 @@ vect_schedule_slp_instance (vec_info *vinfo,
gimple *vstmt;
vstmt = gimple_build_assign (make_ssa_name (vectype),
VEC_PERM_EXPR,
- gimple_assign_lhs (v0[j]->stmt),
- gimple_assign_lhs (v1[j]->stmt),
+ gimple_assign_lhs (v0[j]),
+ gimple_assign_lhs (v1[j]),
tmask);
- SLP_TREE_VEC_STMTS (node).quick_push
- (vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si));
+ vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si);
+ SLP_TREE_VEC_STMTS (node).quick_push (vstmt);
}
v0.release ();
v1.release ();
@@ -4109,12 +4106,12 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance)
if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) == 1)
{
- stmt_vec_info child_stmt_info;
+ gimple *child_stmt;
int j;
- FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info)
+ FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt)
{
- tree vect_lhs = gimple_get_lhs (child_stmt_info->stmt);
+ tree vect_lhs = gimple_get_lhs (child_stmt);
tree root_lhs = gimple_get_lhs (instance->root_stmt->stmt);
if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
TREE_TYPE (vect_lhs)))
@@ -4127,16 +4124,16 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance)
else if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) > 1)
{
int nelts = SLP_TREE_NUMBER_OF_VEC_STMTS (node);
- stmt_vec_info child_stmt_info;
+ gimple *child_stmt;
int j;
vec<constructor_elt, va_gc> *v;
vec_alloc (v, nelts);
- FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info)
+ FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt)
{
CONSTRUCTOR_APPEND_ELT (v,
NULL_TREE,
- gimple_get_lhs (child_stmt_info->stmt));
+ gimple_get_lhs (child_stmt));
}
tree lhs = gimple_get_lhs (instance->root_stmt->stmt);
tree rtype = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmt->stmt));
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index d9db2f079ad..686eb389f0c 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -1478,7 +1478,7 @@ vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo,
gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies);
for (unsigned i = 0; i < ncopies; ++i)
vec_oprnds->quick_push (gimple_get_lhs
- (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]->stmt));
+ (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]));
}
}
@@ -1540,13 +1540,10 @@ vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
vect_finish_stmt_generation. Set the location of the new
statement and create and return a stmt_vec_info for it. */
-static stmt_vec_info
-vect_finish_stmt_generation_1 (vec_info *vinfo,
+static void
+vect_finish_stmt_generation_1 (vec_info *,
stmt_vec_info stmt_info, gimple *vec_stmt)
{
- stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt);
- vec_stmt_info->vector_stmt = 1;
-
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
@@ -1563,15 +1560,13 @@ vect_finish_stmt_generation_1 (vec_info *vinfo,
}
else
gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
-
- return vec_stmt_info;
}
/* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
which sets the same scalar result as STMT_INFO did. Create and return a
stmt_vec_info for VEC_STMT. */
-stmt_vec_info
+void
vect_finish_replace_stmt (vec_info *vinfo,
stmt_vec_info stmt_info, gimple *vec_stmt)
{
@@ -1581,13 +1576,13 @@ vect_finish_replace_stmt (vec_info *vinfo,
gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
gsi_replace (&gsi, vec_stmt, true);
- return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
+ vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
}
/* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
-stmt_vec_info
+void
vect_finish_stmt_generation (vec_info *vinfo,
stmt_vec_info stmt_info, gimple *vec_stmt,
gimple_stmt_iterator *gsi)
@@ -1621,7 +1616,7 @@ vect_finish_stmt_generation (vec_info *vinfo,
}
}
gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
- return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
+ vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
}
/* We want to vectorize a call to combined function CFN with function
@@ -2564,7 +2559,7 @@ vect_build_zero_merge_argument (vec_info *vinfo,
static void
vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt,
+ gimple **vec_stmt,
gather_scatter_info *gs_info,
tree mask)
{
@@ -2661,7 +2656,6 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
tree src_op = NULL_TREE;
tree mask_op = NULL_TREE;
tree prev_res = NULL_TREE;
- stmt_vec_info prev_stmt_info = NULL;
if (!mask)
{
@@ -2765,29 +2759,26 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
}
src_op = build_zero_cst (srctype);
}
- gcall *new_call = gimple_build_call (gs_info->decl, 5, src_op, ptr, op,
- mask_arg, scale);
+ gimple *new_stmt = gimple_build_call (gs_info->decl, 5, src_op, ptr, op,
+ mask_arg, scale);
- stmt_vec_info new_stmt_info;
if (!useless_type_conversion_p (vectype, rettype))
{
gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
TYPE_VECTOR_SUBPARTS (rettype)));
op = vect_get_new_ssa_name (rettype, vect_simple_var);
- gimple_call_set_lhs (new_call, op);
- vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
+ gimple_call_set_lhs (new_stmt, op);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
var = make_ssa_name (vec_dest);
op = build1 (VIEW_CONVERT_EXPR, vectype, op);
- gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
- var = make_ssa_name (vec_dest, new_call);
- gimple_call_set_lhs (new_call, var);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
+ var = make_ssa_name (vec_dest, new_stmt);
+ gimple_call_set_lhs (new_stmt, var);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
if (modifier == NARROW)
@@ -2799,14 +2790,12 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
}
var = permute_vec_elements (vinfo, prev_res, var, perm_mask,
stmt_info, gsi);
- new_stmt_info = loop_vinfo->lookup_def (var);
+ new_stmt = SSA_NAME_DEF_STMT (var);
}
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
- if (prev_stmt_info == NULL)
- *vec_stmt = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
}
/* Prepare the base and offset in GS_INFO for vectorization.
@@ -2905,7 +2894,7 @@ vect_get_data_ptr_increment (vec_info *vinfo,
static bool
vectorizable_bswap (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
slp_tree *slp_op,
tree vectype_in, stmt_vector_for_cost *cost_vec)
{
@@ -2993,12 +2982,11 @@ vectorizable_bswap (vec_info *vinfo,
tem = make_ssa_name (vectype);
new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
vectype, tem2));
- stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -3043,7 +3031,7 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in,
static bool
vectorizable_call (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
gcall *stmt;
@@ -3333,7 +3321,6 @@ vectorizable_call (vec_info *vinfo,
bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- stmt_vec_info new_stmt_info = NULL;
if (modifier == NONE || ifn != IFN_LAST)
{
tree prev_res = NULL_TREE;
@@ -3359,6 +3346,7 @@ vectorizable_call (vec_info *vinfo,
vec<tree> vec_oprndsk = vec_defs[k];
vargs[k] = vec_oprndsk[i];
}
+ gimple *new_stmt;
if (modifier == NARROW)
{
/* We don't define any narrowing conditional functions
@@ -3376,12 +3364,10 @@ vectorizable_call (vec_info *vinfo,
continue;
}
new_temp = make_ssa_name (vec_dest);
- gimple *new_stmt
- = gimple_build_assign (new_temp, convert_code,
- prev_res, half_res);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, convert_code,
+ prev_res, half_res);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
else
{
@@ -3404,11 +3390,10 @@ vectorizable_call (vec_info *vinfo,
new_temp = make_ssa_name (vec_dest, call);
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
}
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
}
continue;
}
@@ -3434,6 +3419,7 @@ vectorizable_call (vec_info *vinfo,
vargs[mask_opno], gsi);
}
+ gimple *new_stmt;
if (cfn == CFN_GOMP_SIMD_LANE)
{
tree cst = build_index_vector (vectype_out, j * nunits_out, 1);
@@ -3442,9 +3428,8 @@ vectorizable_call (vec_info *vinfo,
gimple *init_stmt = gimple_build_assign (new_var, cst);
vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
new_temp = make_ssa_name (vec_dest);
- gimple *new_stmt = gimple_build_assign (new_temp, new_var);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, new_var);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else if (modifier == NARROW)
{
@@ -3462,10 +3447,9 @@ vectorizable_call (vec_info *vinfo,
continue;
}
new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt = gimple_build_assign (new_temp, convert_code,
- prev_res, half_res);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, convert_code,
+ prev_res, half_res);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
@@ -3477,13 +3461,13 @@ vectorizable_call (vec_info *vinfo,
new_temp = make_ssa_name (vec_dest, call);
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
}
if (j == (modifier == NARROW ? 1 : 0))
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
for (i = 0; i < nargs; i++)
{
@@ -3530,9 +3514,8 @@ vectorizable_call (vec_info *vinfo,
new_temp = make_ssa_name (vec_dest, call);
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (call);
}
continue;
}
@@ -3556,10 +3539,9 @@ vectorizable_call (vec_info *vinfo,
gcall *new_stmt = gimple_build_call_vec (fndecl, vargs);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_call_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -3695,7 +3677,7 @@ simd_clone_subparts (tree vectype)
static bool
vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *)
{
tree vec_dest;
@@ -4154,7 +4136,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
}
tree phi_res = copy_ssa_name (op);
gphi *new_phi = create_phi_node (phi_res, loop->header);
- loop_vinfo->add_stmt (new_phi);
add_phi_arg (new_phi, arginfo[i].op,
loop_preheader_edge (loop), UNKNOWN_LOCATION);
enum tree_code code
@@ -4171,7 +4152,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
= gimple_build_assign (phi_arg, code, phi_res, tcst);
gimple_stmt_iterator si = gsi_after_labels (loop->header);
gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
- loop_vinfo->add_stmt (new_stmt);
add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
UNKNOWN_LOCATION);
arginfo[i].op = phi_res;
@@ -4220,8 +4200,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
new_temp = make_ssa_name (rtype, new_call);
gimple_call_set_lhs (new_call, new_temp);
}
- stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
+ gimple *new_stmt = new_call;
if (vec_dest)
{
@@ -4244,15 +4224,12 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
else
t = build3 (BIT_FIELD_REF, vectype, new_temp,
bitsize_int (prec), bitsize_int (l * prec));
- gimple *new_stmt
- = gimple_build_assign (make_ssa_name (vectype), t);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (j == 0 && l == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (ratype)
@@ -4273,11 +4250,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
{
tree tem = build4 (ARRAY_REF, rtype, new_temp,
size_int (m), NULL_TREE, NULL_TREE);
- gimple *new_stmt
- = gimple_build_assign (make_ssa_name (rtype), tem);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ new_stmt = gimple_build_assign (make_ssa_name (rtype),
+ tem);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
gimple_assign_lhs (new_stmt));
}
@@ -4288,14 +4264,13 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
if ((j & (k - 1)) != k - 1)
continue;
vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
- gimple *new_stmt
+ new_stmt
= gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if ((unsigned) j == k - 1)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
continue;
}
else if (ratype)
@@ -4303,17 +4278,15 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
tree t = build_fold_addr_expr (new_temp);
t = build2 (MEM_REF, vectype, t,
build_int_cst (TREE_TYPE (t), 0));
- gimple *new_stmt
- = gimple_build_assign (make_ssa_name (vec_dest), t);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
}
}
if (j == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
for (i = 0; i < nargs; ++i)
@@ -4403,8 +4376,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
new_tmp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_tmp);
- stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (multi_step_cvt)
/* Store the resulting vector for next recursive call. */
@@ -4415,9 +4387,9 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
vectors in SLP_NODE or in vector info of the scalar statement
(or in STMT_VINFO_RELATED_STMT chain). */
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
@@ -4503,7 +4475,7 @@ vect_create_vectorized_promotion_stmts (vec_info *vinfo,
static bool
vectorizable_conversion (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree vec_dest;
@@ -4887,19 +4859,17 @@ vectorizable_conversion (vec_info *vinfo,
op0, &vec_oprnds0);
FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
{
- stmt_vec_info new_stmt_info;
/* Arguments are ready, create the new vector stmt. */
gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
break;
@@ -4936,23 +4906,21 @@ vectorizable_conversion (vec_info *vinfo,
FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
{
- stmt_vec_info new_stmt_info;
+ gimple *new_stmt;
if (cvt_type)
{
gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt
- = gimple_build_assign (new_temp, codecvt1, vop0);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, codecvt1, vop0);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
- new_stmt_info = vinfo->lookup_def (vop0);
+ new_stmt = SSA_NAME_DEF_STMT (vop0);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
break;
@@ -5025,7 +4993,7 @@ vect_nop_conversion_p (stmt_vec_info stmt_info)
static bool
vectorizable_assignment (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree vec_dest;
@@ -5155,7 +5123,6 @@ vectorizable_assignment (vec_info *vinfo,
vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds);
/* Arguments are ready. create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
{
if (CONVERT_EXPR_CODE_P (code)
@@ -5164,12 +5131,11 @@ vectorizable_assignment (vec_info *vinfo,
gassign *new_stmt = gimple_build_assign (vec_dest, vop);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
@@ -5225,7 +5191,7 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
static bool
vectorizable_shift (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree vec_dest;
@@ -5606,19 +5572,17 @@ vectorizable_shift (vec_info *vinfo,
vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1);
/* Arguments are ready. Create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
{
vop1 = vec_oprnds1[i];
gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -5642,7 +5606,7 @@ vectorizable_shift (vec_info *vinfo,
static bool
vectorizable_operation (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree vec_dest;
@@ -6015,9 +5979,9 @@ vectorizable_operation (vec_info *vinfo,
vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2);
/* Arguments are ready. Create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
{
+ gimple *new_stmt = NULL;
vop1 = ((op_type == binary_op || op_type == ternary_op)
? vec_oprnds1[i] : NULL_TREE);
vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
@@ -6034,33 +5998,30 @@ vectorizable_operation (vec_info *vinfo,
new_temp = make_ssa_name (vec_dest, call);
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
}
else
{
- gassign *new_stmt = gimple_build_assign (vec_dest, code,
- vop0, vop1, vop2);
+ new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (vec_cvt_dest)
{
new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
- gassign *new_stmt
- = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
- new_temp);
+ new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
+ new_temp);
new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -6723,7 +6684,7 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
static bool
vectorizable_scan_store (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, int ncopies)
+ gimple **vec_stmt, int ncopies)
{
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
@@ -6843,7 +6804,6 @@ vectorizable_scan_store (vec_info *vinfo,
rhs2, &vec_oprnds3);
for (int j = 0; j < ncopies; j++)
{
- stmt_vec_info new_stmt_info;
vec_oprnd1 = vec_oprnds1[j];
if (ldataref_ptr == NULL)
vec_oprnd2 = vec_oprnds2[j];
@@ -6861,8 +6821,8 @@ vectorizable_scan_store (vec_info *vinfo,
dataref_offset);
vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
}
@@ -6876,8 +6836,8 @@ vectorizable_scan_store (vec_info *vinfo,
!= scan_store_kind_perm))
? zero_vec : vec_oprnd1, v,
perms[i]);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
@@ -6893,9 +6853,9 @@ vectorizable_scan_store (vec_info *vinfo,
tree new_temp2 = make_ssa_name (vectype);
g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
new_temp, vec_oprnd1);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ vect_finish_stmt_generation (vinfo, stmt_info,
g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
new_temp = new_temp2;
}
@@ -6912,16 +6872,16 @@ vectorizable_scan_store (vec_info *vinfo,
tree new_temp2 = make_ssa_name (vectype);
g = gimple_build_assign (new_temp2, code, v, new_temp);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
v = new_temp2;
}
tree new_temp = make_ssa_name (vectype);
gimple *g = gimple_build_assign (new_temp, code, orig, v);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
tree last_perm_arg = new_temp;
/* For exclusive scan, new_temp computed above is the exclusive scan
@@ -6931,15 +6891,15 @@ vectorizable_scan_store (vec_info *vinfo,
{
last_perm_arg = make_ssa_name (vectype);
g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
}
orig = make_ssa_name (vectype);
g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
last_perm_arg, perms[units_log2]);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
if (!inscan_var_store)
{
@@ -6948,8 +6908,8 @@ vectorizable_scan_store (vec_info *vinfo,
dataref_offset);
vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
g = gimple_build_assign (data_ref, new_temp);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
}
}
@@ -6964,9 +6924,8 @@ vectorizable_scan_store (vec_info *vinfo,
dataref_offset);
vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
gimple *g = gimple_build_assign (data_ref, orig);
- stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
}
return true;
}
@@ -6983,7 +6942,7 @@ vectorizable_scan_store (vec_info *vinfo,
static bool
vectorizable_store (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree data_ref;
@@ -7399,10 +7358,9 @@ vectorizable_store (vec_info *vinfo,
gcall *new_stmt
= gimple_build_call (gs_info.decl, 5, ptr, mask_arg, op, src, scale);
- stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
@@ -7580,7 +7538,6 @@ vectorizable_store (vec_info *vinfo,
loop, &incr_gsi, insert_after,
&offvar, NULL);
incr = gsi_stmt (incr_gsi);
- loop_vinfo->add_stmt (incr);
stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
@@ -7643,9 +7600,7 @@ vectorizable_store (vec_info *vinfo,
/* And store it to *running_off. */
assign = gimple_build_assign (newref, elem);
- stmt_vec_info assign_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- assign, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
group_el += lnel;
if (! slp
@@ -7663,8 +7618,8 @@ vectorizable_store (vec_info *vinfo,
&& !slp)
{
if (j == 0 && i == 0)
- *vec_stmt = assign_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign_info);
+ *vec_stmt = assign;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign);
}
}
}
@@ -7777,7 +7732,7 @@ vectorizable_store (vec_info *vinfo,
gvec_oprnds.safe_grow_cleared (group_size);
for (j = 0; j < ncopies; j++)
{
- stmt_vec_info new_stmt_info;
+ gimple *new_stmt;
if (j == 0)
{
if (slp)
@@ -7924,15 +7879,15 @@ vectorizable_store (vec_info *vinfo,
gimple_call_set_lhs (call, data_ref);
}
gimple_call_set_nothrow (call, true);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
- call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
/* Record that VEC_ARRAY is now dead. */
vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
}
else
{
- new_stmt_info = NULL;
+ new_stmt = NULL;
if (grouped_store)
{
if (j == 0)
@@ -7970,8 +7925,8 @@ vectorizable_store (vec_info *vinfo,
(IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
scale, vec_oprnd);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
break;
}
@@ -8029,8 +7984,8 @@ vectorizable_store (vec_info *vinfo,
dataref_ptr, ptr,
final_mask, vec_oprnd);
gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
}
else
{
@@ -8050,10 +8005,8 @@ vectorizable_store (vec_info *vinfo,
= build_aligned_type (TREE_TYPE (data_ref),
TYPE_ALIGN (elem_type));
vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
- gassign *new_stmt
- = gimple_build_assign (data_ref, vec_oprnd);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (data_ref, vec_oprnd);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
if (slp)
@@ -8067,8 +8020,8 @@ vectorizable_store (vec_info *vinfo,
if (!slp)
{
if (j == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
@@ -8205,7 +8158,7 @@ hoist_defs_of_uses (stmt_vec_info stmt_info, class loop *loop)
static bool
vectorizable_load (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt, slp_tree slp_node,
+ gimple **vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree scalar_dest;
@@ -8535,27 +8488,20 @@ vectorizable_load (vec_info *vinfo,
gsi_next (&gsi2);
for (j = 0; j < ncopies; j++)
{
- stmt_vec_info new_stmt_info;
if (hoist_p)
- {
- new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
- vectype, NULL);
- gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp);
- new_stmt_info = vinfo->add_stmt (new_stmt);
- }
+ new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
+ vectype, NULL);
else
- {
- new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
- vectype, &gsi2);
- new_stmt_info = vinfo->lookup_def (new_temp);
- }
+ new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
+ vectype, &gsi2);
+ gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp);
if (slp)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
{
if (j == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
return true;
@@ -8566,7 +8512,6 @@ vectorizable_load (vec_info *vinfo,
{
gimple_stmt_iterator incr_gsi;
bool insert_after;
- gimple *incr;
tree offvar;
tree ivstep;
tree running_off;
@@ -8641,8 +8586,6 @@ vectorizable_load (vec_info *vinfo,
create_iv (stride_base, ivstep, NULL,
loop, &incr_gsi, insert_after,
&offvar, NULL);
- incr = gsi_stmt (incr_gsi);
- loop_vinfo->add_stmt (incr);
stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
@@ -8712,17 +8655,15 @@ vectorizable_load (vec_info *vinfo,
{
if (nloads > 1)
vec_alloc (v, nloads);
- stmt_vec_info new_stmt_info = NULL;
+ gimple *new_stmt = NULL;
for (i = 0; i < nloads; i++)
{
tree this_off = build_int_cst (TREE_TYPE (alias_off),
group_el * elsz + cst_offset);
tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
- gassign *new_stmt
- = gimple_build_assign (make_ssa_name (ltype), data_ref);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (make_ssa_name (ltype), data_ref);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (nloads > 1)
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
gimple_assign_lhs (new_stmt));
@@ -8745,32 +8686,29 @@ vectorizable_load (vec_info *vinfo,
tree vec_inv = build_constructor (lvectype, v);
new_temp = vect_init_vector (vinfo, stmt_info,
vec_inv, lvectype, gsi);
- new_stmt_info = vinfo->lookup_def (new_temp);
+ new_stmt = SSA_NAME_DEF_STMT (new_temp);
if (lvectype != vectype)
{
- gassign *new_stmt
- = gimple_build_assign (make_ssa_name (vectype),
- VIEW_CONVERT_EXPR,
- build1 (VIEW_CONVERT_EXPR,
- vectype, new_temp));
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ new_stmt = gimple_build_assign (make_ssa_name (vectype),
+ VIEW_CONVERT_EXPR,
+ build1 (VIEW_CONVERT_EXPR,
+ vectype, new_temp));
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
}
if (slp)
{
if (slp_perm)
- dr_chain.quick_push (gimple_assign_lhs (new_stmt_info->stmt));
+ dr_chain.quick_push (gimple_assign_lhs (new_stmt));
else
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
}
else
{
if (j == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
if (slp_perm)
@@ -9052,7 +8990,6 @@ vectorizable_load (vec_info *vinfo,
poly_uint64 group_elt = 0;
for (j = 0; j < ncopies; j++)
{
- stmt_vec_info new_stmt_info = NULL;
/* 1. Create the vector or array pointer update chain. */
if (j == 0)
{
@@ -9131,6 +9068,7 @@ vectorizable_load (vec_info *vinfo,
if (grouped_load || slp_perm)
dr_chain.create (vec_num);
+ gimple *new_stmt = NULL;
if (memory_access_type == VMAT_LOAD_STORE_LANES)
{
tree vec_array;
@@ -9166,8 +9104,8 @@ vectorizable_load (vec_info *vinfo,
}
gimple_call_set_lhs (call, vec_array);
gimple_call_set_nothrow (call, true);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
- call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ new_stmt = call;
/* Extract each vector into an SSA_NAME. */
for (i = 0; i < vec_num; i++)
@@ -9202,7 +9140,6 @@ vectorizable_load (vec_info *vinfo,
gsi, stmt_info, bump);
/* 2. Create the vector-load in the loop. */
- gimple *new_stmt = NULL;
switch (alignment_support_scheme)
{
case dr_aligned:
@@ -9448,9 +9385,7 @@ vectorizable_load (vec_info *vinfo,
}
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
/* 3. Handle explicit realignment if necessary/supported.
Create in loop:
@@ -9466,9 +9401,7 @@ vectorizable_load (vec_info *vinfo,
msq, lsq, realignment_token);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (alignment_support_scheme == dr_explicit_realign_optimized)
{
@@ -9486,7 +9419,7 @@ vectorizable_load (vec_info *vinfo,
tree perm_mask = perm_mask_for_reverse (vectype);
new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
perm_mask, stmt_info, gsi);
- new_stmt_info = vinfo->lookup_def (new_temp);
+ new_stmt = SSA_NAME_DEF_STMT (new_temp);
}
/* Collect vector loads and later create their permutation in
@@ -9496,7 +9429,7 @@ vectorizable_load (vec_info *vinfo,
/* Store vector loads in the corresponding SLP_NODE. */
if (slp && !slp_perm)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
/* With SLP permutation we load the gaps as well, without
we need to skip the gaps after we manage to fully load
@@ -9549,13 +9482,13 @@ vectorizable_load (vec_info *vinfo,
}
else
{
- if (j == 0)
- *vec_stmt = new_stmt_info;
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
}
dr_chain.release ();
}
+ if (!slp)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
}
@@ -9667,7 +9600,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
static bool
vectorizable_condition (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt,
+ gimple **vec_stmt,
slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{
tree scalar_dest = NULL_TREE;
@@ -10090,19 +10023,18 @@ vectorizable_condition (vec_info *vinfo,
}
}
- stmt_vec_info new_stmt_info;
+ gimple *new_stmt;
if (reduction_type == EXTRACT_LAST_REDUCTION)
{
gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
tree lhs = gimple_get_lhs (old_stmt);
- gcall *new_stmt = gimple_build_call_internal
+ new_stmt = gimple_build_call_internal
(IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
vec_then_clause);
gimple_call_set_lhs (new_stmt, lhs);
SSA_NAME_DEF_STMT (lhs) = new_stmt;
if (old_stmt == gsi_stmt (*gsi))
- new_stmt_info = vect_finish_replace_stmt (vinfo,
- stmt_info, new_stmt);
+ vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
else
{
/* In this case we're moving the definition to later in the
@@ -10110,23 +10042,20 @@ vectorizable_condition (vec_info *vinfo,
lhs are in phi statements. */
gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
gsi_remove (&old_gsi, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
}
else
{
new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt
- = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
- vec_then_clause, vec_else_clause);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
+ vec_then_clause, vec_else_clause);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -10151,7 +10080,7 @@ vectorizable_condition (vec_info *vinfo,
static bool
vectorizable_comparison (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- stmt_vec_info *vec_stmt,
+ gimple **vec_stmt,
slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{
tree lhs, rhs1, rhs2;
@@ -10344,27 +10273,24 @@ vectorizable_comparison (vec_info *vinfo,
/* Arguments are ready. Create the new vector stmt. */
FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
{
- stmt_vec_info new_stmt_info;
+ gimple *new_stmt;
vec_rhs2 = vec_oprnds1[i];
new_temp = make_ssa_name (mask);
if (bitop1 == NOP_EXPR)
{
- gassign *new_stmt = gimple_build_assign (new_temp, code,
- vec_rhs1, vec_rhs2);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ new_stmt = gimple_build_assign (new_temp, code,
+ vec_rhs1, vec_rhs2);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
- gassign *new_stmt;
if (bitop1 == BIT_NOT_EXPR)
new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
else
new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
vec_rhs2);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (bitop2 != NOP_EXPR)
{
tree res = make_ssa_name (mask);
@@ -10373,14 +10299,13 @@ vectorizable_comparison (vec_info *vinfo,
else
new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
}
if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
else
- STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
}
if (!slp_node)
@@ -10666,7 +10591,7 @@ vect_transform_stmt (vec_info *vinfo,
slp_tree slp_node, slp_instance slp_node_instance)
{
bool is_store = false;
- stmt_vec_info vec_stmt = NULL;
+ gimple *vec_stmt = NULL;
bool done;
gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
@@ -10808,14 +10733,14 @@ vect_transform_stmt (vec_info *vinfo,
&& (PHI_ARG_DEF_FROM_EDGE (phi, e)
== gimple_get_lhs (orig_stmt_info->stmt)))
{
- vec<stmt_vec_info> &phi_info
+ vec<gimple *> &phi_info
= STMT_VINFO_VEC_STMTS (STMT_VINFO_REDUC_DEF (orig_stmt_info));
- vec<stmt_vec_info> &vec_stmt
+ vec<gimple *> &vec_stmt
= STMT_VINFO_VEC_STMTS (stmt_info);
gcc_assert (phi_info.length () == vec_stmt.length ());
for (unsigned i = 0; i < phi_info.length (); ++i)
- add_phi_arg (as_a <gphi *> (phi_info[i]->stmt),
- gimple_get_lhs (vec_stmt[i]->stmt), e,
+ add_phi_arg (as_a <gphi *> (phi_info[i]),
+ gimple_get_lhs (vec_stmt[i]), e,
gimple_phi_arg_location (phi, e->dest_idx));
}
else if (slp_node
@@ -10827,7 +10752,7 @@ vect_transform_stmt (vec_info *vinfo,
gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length ()
== SLP_TREE_VEC_STMTS (slp_node).length ());
for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i)
- add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt),
+ add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]),
vect_get_slp_vect_def (slp_node, i),
e, gimple_phi_arg_location (phi, e->dest_idx));
}
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index e9fe3840e1e..76cfba5d497 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -461,6 +461,7 @@ vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in,
vec_info_shared *shared_)
: kind (kind_in),
shared (shared_),
+ stmt_vec_info_ro (false),
target_cost_data (target_cost_data_in)
{
stmt_vec_infos.create (50);
@@ -619,8 +620,7 @@ vec_info::replace_stmt (gimple_stmt_iterator *gsi, stmt_vec_info stmt_info,
{
gimple *old_stmt = stmt_info->stmt;
gcc_assert (!stmt_info->pattern_stmt_p && old_stmt == gsi_stmt (*gsi));
- set_vinfo_for_stmt (old_stmt, NULL);
- set_vinfo_for_stmt (new_stmt, stmt_info);
+ gimple_set_uid (new_stmt, gimple_uid (old_stmt));
stmt_info->stmt = new_stmt;
gsi_replace (gsi, new_stmt, true);
}
@@ -666,6 +666,7 @@ vec_info::set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info)
unsigned int uid = gimple_uid (stmt);
if (uid == 0)
{
+ gcc_assert (!stmt_vec_info_ro);
gcc_checking_assert (info);
uid = stmt_vec_infos.length () + 1;
gimple_set_uid (stmt, uid);
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index df056381531..14f68f7c4f6 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -138,7 +138,7 @@ struct _slp_tree {
tree vectype;
/* Vectorized stmt/s. */
- vec<stmt_vec_info> vec_stmts;
+ vec<gimple *> vec_stmts;
vec<tree> vec_defs;
/* Number of vector stmts that are created to replace the group of scalar
stmts. It is calculated during the transformation phase as the number of
@@ -338,6 +338,8 @@ public:
/* The mapping of GIMPLE UID to stmt_vec_info. */
vec<stmt_vec_info> stmt_vec_infos;
+ /* Whether the above mapping is complete. */
+ bool stmt_vec_info_ro;
/* The SLP graph. */
auto_vec<slp_instance> slp_instances;
@@ -965,7 +967,7 @@ public:
tree vectype;
/* The vectorized stmts. */
- vec<stmt_vec_info> vec_stmts;
+ vec<gimple *> vec_stmts;
/* The following is relevant only for stmts that contain a non-scalar
data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
@@ -1126,8 +1128,6 @@ public:
/* True if this is only suitable for SLP vectorization. */
bool slp_vect_only_p;
-
- bool vector_stmt;
};
/* Information about a gather/scatter call. */
@@ -1169,7 +1169,7 @@ struct gather_scatter_info {
#define STMT_VINFO_RELEVANT(S) (S)->relevant
#define STMT_VINFO_LIVE_P(S) (S)->live
#define STMT_VINFO_VECTYPE(S) (S)->vectype
-#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
+#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
@@ -1737,11 +1737,9 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
STMT_VINFO_VECTYPE (stmt_info), misalign, where);
}
-extern stmt_vec_info vect_finish_replace_stmt (vec_info *,
- stmt_vec_info, gimple *);
-extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
- stmt_vec_info, gimple *,
- gimple_stmt_iterator *);
+extern void vect_finish_replace_stmt (vec_info *, stmt_vec_info, gimple *);
+extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *,
+ gimple_stmt_iterator *);
extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
extern tree vect_get_store_rhs (stmt_vec_info);
void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned,
@@ -1872,16 +1870,16 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
stmt_vector_for_cost *);
extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
gimple_stmt_iterator *,
- stmt_vec_info *, slp_tree,
+ gimple **, slp_tree,
stmt_vector_for_cost *);
extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
gimple_stmt_iterator *,
- stmt_vec_info *, slp_tree);
+ gimple **, slp_tree);
extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
- stmt_vec_info *,
+ gimple **,
slp_tree, slp_instance);
extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info,
- stmt_vec_info *, slp_tree);
+ gimple **, slp_tree);
extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code);
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
stmt_vector_for_cost *,
--
2.26.2
More information about the Gcc-patches
mailing list