[PATCH] Introduce STMT_VINFO_VEC_STMTS
Richard Biener
rguenther@suse.de
Wed Jun 10 07:51:21 GMT 2020
This gets rid of the linked list of STMT_VINFO_VECT_STMT and
STMT_VINFO_RELATED_STMT in preparation for vectorized stmts no
longer needing a stmt_vec_info (just for this chaining). This
has ripple-down effects in all places we gather vectorized
defs. For this new interfaces are introduced and used
throughout vectorization, simplifying code in a lot of places
and merging it with the SLP way of gathering vectorized
operands. There is vect_get_vec_defs as the new recommended
unified interface and vect_get_vec_defs_for_operand as one
for non-SLP operation. I've resorted to keep the structure
of the code the same where using vect_get_vec_defs would have
been too disruptive for this already large patch.
Boostrap and regtest running on x86_64-unknown-linux-gnu.
Richard.
2020-06-10 Richard Biener <rguenther@suse.de>
* tree-vect-data-refs.c (vect_vfa_access_size): Adjust.
(vect_record_grouped_load_vectors): Likewise.
* tree-vect-loop.c (vect_create_epilog_for_reduction): Likewise.
(vectorize_fold_left_reduction): Likewise.
(vect_transform_reduction): Likewise.
(vect_transform_cycle_phi): Likewise.
(vectorizable_lc_phi): Likewise.
(vectorizable_induction): Likewise.
(vectorizable_live_operation): Likewise.
(vect_transform_loop): Likewise.
* tree-vect-slp.c (vect_get_slp_defs): New function, split out
from overload.
* tree-vect-stmts.c (vect_get_vec_def_for_operand_1): Remove.
(vect_get_vec_def_for_operand): Likewise.
(vect_get_vec_def_for_stmt_copy): Likewise.
(vect_get_vec_defs_for_stmt_copy): Likewise.
(vect_get_vec_defs_for_operand): New function.
(vect_get_vec_defs): Likewise.
(vect_build_gather_load_calls): Adjust.
(vect_get_gather_scatter_ops): Likewise.
(vectorizable_bswap): Likewise.
(vectorizable_call): Likewise.
(vectorizable_simd_clone_call): Likewise.
(vect_get_loop_based_defs): Remove.
(vect_create_vectorized_demotion_stmts): Adjust.
(vectorizable_conversion): Likewise.
(vectorizable_assignment): Likewise.
(vectorizable_shift): Likewise.
(vectorizable_operation): Likewise.
(vectorizable_scan_store): Likewise.
(vectorizable_store): Likewise.
(vectorizable_load): Likewise.
(vectorizable_condition): Likewise.
(vectorizable_comparison): Likewise.
(vect_transform_stmt): Adjust and remove no longer applicable
sanity checks.
* tree-vectorizer.c (vec_info::new_stmt_vec_info): Initialize
STMT_VINFO_VEC_STMTS.
(vec_info::free_stmt_vec_info): Relase it.
* tree-vectorizer.h (_stmt_vec_info::vectorized_stmt): Remove.
(_stmt_vec_info::vec_stmts): Add.
(STMT_VINFO_VEC_STMT): Remove.
(STMT_VINFO_VEC_STMTS): New.
(vect_get_vec_def_for_operand_1): Remove.
(vect_get_vec_def_for_operand): Likewise.
(vect_get_vec_defs_for_stmt_copy): Likewise.
(vect_get_vec_def_for_stmt_copy): Likewise.
(vect_get_vec_defs): New overloads.
(vect_get_vec_defs_for_operand): New.
(vect_get_slp_defs): Declare.
---
gcc/tree-vect-data-refs.c | 22 +-
gcc/tree-vect-loop.c | 319 +++---
gcc/tree-vect-slp.c | 33 +-
gcc/tree-vect-stmts.c | 1987 ++++++++++++++-----------------------
gcc/tree-vectorizer.c | 2 +
gcc/tree-vectorizer.h | 30 +-
6 files changed, 884 insertions(+), 1509 deletions(-)
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index fe543606a52..7edd9ebe3b6 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -3216,7 +3216,7 @@ vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == stmt_vinfo);
access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
}
- if (STMT_VINFO_VEC_STMT (stmt_vinfo)
+ if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists ()
&& (vect_supportable_dr_alignment (vinfo, dr_info, false)
== dr_explicit_realign_optimized))
{
@@ -6443,24 +6443,8 @@ vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
{
stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref);
/* We assume that if VEC_STMT is not NULL, this is a case of multiple
- copies, and we put the new vector statement in the first available
- RELATED_STMT. */
- if (!STMT_VINFO_VEC_STMT (next_stmt_info))
- STMT_VINFO_VEC_STMT (next_stmt_info) = new_stmt_info;
- else
- {
- stmt_vec_info prev_stmt_info
- = STMT_VINFO_VEC_STMT (next_stmt_info);
- stmt_vec_info rel_stmt_info
- = STMT_VINFO_RELATED_STMT (prev_stmt_info);
- while (rel_stmt_info)
- {
- prev_stmt_info = rel_stmt_info;
- rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info);
- }
-
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- }
+ copies, and we put the new vector statement last. */
+ STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt_info);
next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
gap_count = 1;
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 5329982e4c9..cc3d391d7c2 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -4485,7 +4485,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
= as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
- stmt_vec_info prev_phi_info;
tree vectype;
machine_mode mode;
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *outer_loop = NULL;
@@ -4493,7 +4492,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
tree scalar_dest;
tree scalar_type;
gimple *new_phi = NULL, *phi;
- stmt_vec_info phi_info;
gimple_stmt_iterator exit_gsi;
tree new_temp = NULL_TREE, new_name, new_scalar_dest;
gimple *epilog_stmt = NULL;
@@ -4563,15 +4561,9 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
}
else
{
+ stmt_vec_info reduc_info = loop_vinfo->lookup_stmt (reduc_def_stmt);
vec_num = 1;
- ncopies = 0;
- phi_info = STMT_VINFO_VEC_STMT (loop_vinfo->lookup_stmt (reduc_def_stmt));
- do
- {
- ncopies++;
- phi_info = STMT_VINFO_RELATED_STMT (phi_info);
- }
- while (phi_info);
+ ncopies = STMT_VINFO_VEC_STMTS (reduc_info).length ();
}
/* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
@@ -4593,7 +4585,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
{
if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
{
- gimple *vec_stmt = STMT_VINFO_VEC_STMT (cond_info)->stmt;
+ gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]->stmt;
gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
ccompares.safe_push
(std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt)),
@@ -4714,29 +4706,27 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
if (double_reduc)
loop = outer_loop;
exit_bb = single_exit (loop)->dest;
- prev_phi_info = NULL;
new_phis.create (slp_node ? vec_num : ncopies);
for (unsigned i = 0; i < vec_num; i++)
{
if (slp_node)
def = vect_get_slp_vect_def (slp_node, i);
else
- def = gimple_get_lhs (STMT_VINFO_VEC_STMT (rdef_info)->stmt);
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]->stmt);
for (j = 0; j < ncopies; j++)
{
tree new_def = copy_ssa_name (def);
phi = create_phi_node (new_def, exit_bb);
- stmt_vec_info phi_info = loop_vinfo->add_stmt (phi);
+ loop_vinfo->add_stmt (phi);
if (j == 0)
new_phis.quick_push (phi);
else
{
- def = vect_get_vec_def_for_stmt_copy (loop_vinfo, def);
- STMT_VINFO_RELATED_STMT (prev_phi_info) = phi_info;
+ def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]->stmt);
+ new_phis.quick_push (phi);
}
SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
- prev_phi_info = phi_info;
}
}
@@ -4807,15 +4797,12 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
/* Likewise if we couldn't use a single defuse cycle. */
else if (ncopies > 1)
{
- gcc_assert (new_phis.length () == 1);
gimple_seq stmts = NULL;
tree first_vect = PHI_RESULT (new_phis[0]);
first_vect = gimple_convert (&stmts, vectype, first_vect);
- stmt_vec_info next_phi_info = loop_vinfo->lookup_stmt (new_phis[0]);
for (int k = 1; k < ncopies; ++k)
{
- next_phi_info = STMT_VINFO_RELATED_STMT (next_phi_info);
- tree second_vect = PHI_RESULT (next_phi_info->stmt);
+ tree second_vect = PHI_RESULT (new_phis[k]);
second_vect = gimple_convert (&stmts, vectype, second_vect);
first_vect = gimple_build (&stmts, code, vectype,
first_vect, second_vect);
@@ -5721,10 +5708,8 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
}
else
{
- tree loop_vec_def0 = vect_get_vec_def_for_operand (loop_vinfo,
- op0, stmt_info);
- vec_oprnds0.create (1);
- vec_oprnds0.quick_push (loop_vec_def0);
+ vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, 1,
+ op0, &vec_oprnds0);
scalar_dest_def_info = stmt_info;
}
@@ -5814,7 +5799,10 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
}
if (!slp_node)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ {
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = new_stmt_info;
+ }
return true;
}
@@ -6840,7 +6828,6 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
- int j;
int vec_num;
stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
@@ -6897,7 +6884,6 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
/* Transform. */
stmt_vec_info new_stmt_info = NULL;
- stmt_vec_info prev_stmt_info;
tree new_temp = NULL_TREE;
auto_vec<tree> vec_oprnds0;
auto_vec<tree> vec_oprnds1;
@@ -6932,139 +6918,83 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
tree scalar_dest = gimple_assign_lhs (stmt);
tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
- prev_stmt_info = NULL;
- if (!slp_node)
+ vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, ncopies,
+ single_defuse_cycle && reduc_index == 0
+ ? NULL_TREE : ops[0], &vec_oprnds0,
+ single_defuse_cycle && reduc_index == 1
+ ? NULL_TREE : ops[1], &vec_oprnds1,
+ op_type == ternary_op
+ && !(single_defuse_cycle && reduc_index == 2)
+ ? ops[2] : NULL_TREE, &vec_oprnds2);
+ if (single_defuse_cycle)
{
- vec_oprnds0.create (1);
- vec_oprnds1.create (1);
- if (op_type == ternary_op)
- vec_oprnds2.create (1);
+ gcc_assert (!slp_node);
+ vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, 1,
+ ops[reduc_index],
+ reduc_index == 0 ? &vec_oprnds0
+ : (reduc_index == 1 ? &vec_oprnds1
+ : &vec_oprnds2));
}
- for (j = 0; j < ncopies; j++)
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
{
- /* Handle uses. */
- if (j == 0)
- {
- if (slp_node)
- {
- /* Get vec defs for all the operands except the reduction index,
- ensuring the ordering of the ops in the vector is kept. */
- auto_vec<vec<tree>, 3> vec_defs;
- vect_get_slp_defs (loop_vinfo, slp_node, &vec_defs);
- vec_oprnds0.safe_splice (vec_defs[0]);
- vec_defs[0].release ();
- vec_oprnds1.safe_splice (vec_defs[1]);
- vec_defs[1].release ();
- if (op_type == ternary_op)
- {
- vec_oprnds2.safe_splice (vec_defs[2]);
- vec_defs[2].release ();
- }
- }
- else
+ tree vop[3] = { def0, vec_oprnds1[i], NULL_TREE };
+ if (masked_loop_p && !mask_by_cond_expr)
+ {
+ /* Make sure that the reduction accumulator is vop[0]. */
+ if (reduc_index == 1)
{
- vec_oprnds0.quick_push
- (vect_get_vec_def_for_operand (loop_vinfo, ops[0], stmt_info));
- vec_oprnds1.quick_push
- (vect_get_vec_def_for_operand (loop_vinfo, ops[1], stmt_info));
- if (op_type == ternary_op)
- vec_oprnds2.quick_push
- (vect_get_vec_def_for_operand (loop_vinfo, ops[2], stmt_info));
+ gcc_assert (commutative_tree_code (code));
+ std::swap (vop[0], vop[1]);
}
- }
+ tree mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+ vectype_in, i);
+ gcall *call = gimple_build_call_internal (cond_fn, 4, mask,
+ vop[0], vop[1], vop[0]);
+ new_temp = make_ssa_name (vec_dest, call);
+ gimple_call_set_lhs (call, new_temp);
+ gimple_call_set_nothrow (call, true);
+ new_stmt_info = vect_finish_stmt_generation (loop_vinfo,
+ stmt_info, call, gsi);
+ }
else
- {
- if (!slp_node)
- {
- gcc_assert (reduc_index != -1 || ! single_defuse_cycle);
-
- if (single_defuse_cycle && reduc_index == 0)
- vec_oprnds0[0] = gimple_get_lhs (new_stmt_info->stmt);
- else
- vec_oprnds0[0]
- = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_oprnds0[0]);
- if (single_defuse_cycle && reduc_index == 1)
- vec_oprnds1[0] = gimple_get_lhs (new_stmt_info->stmt);
- else
- vec_oprnds1[0]
- = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_oprnds1[0]);
- if (op_type == ternary_op)
- {
- if (single_defuse_cycle && reduc_index == 2)
- vec_oprnds2[0] = gimple_get_lhs (new_stmt_info->stmt);
- else
- vec_oprnds2[0]
- = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_oprnds2[0]);
- }
- }
- }
+ {
+ if (op_type == ternary_op)
+ vop[2] = vec_oprnds2[i];
- FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
- {
- tree vop[3] = { def0, vec_oprnds1[i], NULL_TREE };
- if (masked_loop_p && !mask_by_cond_expr)
+ if (masked_loop_p && mask_by_cond_expr)
{
- /* Make sure that the reduction accumulator is vop[0]. */
- if (reduc_index == 1)
- {
- gcc_assert (commutative_tree_code (code));
- std::swap (vop[0], vop[1]);
- }
tree mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
- vectype_in, i * ncopies + j);
- gcall *call = gimple_build_call_internal (cond_fn, 4, mask,
- vop[0], vop[1],
- vop[0]);
- new_temp = make_ssa_name (vec_dest, call);
- gimple_call_set_lhs (call, new_temp);
- gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (loop_vinfo,
- stmt_info, call, gsi);
- }
- else
- {
- if (op_type == ternary_op)
- vop[2] = vec_oprnds2[i];
-
- if (masked_loop_p && mask_by_cond_expr)
- {
- tree mask = vect_get_loop_mask (gsi, masks,
- vec_num * ncopies,
- vectype_in, i * ncopies + j);
- build_vect_cond_expr (code, vop, mask, gsi);
- }
-
- gassign *new_stmt = gimple_build_assign (vec_dest, code,
- vop[0], vop[1], vop[2]);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (loop_vinfo,
- stmt_info, new_stmt, gsi);
+ vectype_in, i);
+ build_vect_cond_expr (code, vop, mask, gsi);
}
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
-
- if (slp_node || single_defuse_cycle)
- continue;
+ gassign *new_stmt = gimple_build_assign (vec_dest, code,
+ vop[0], vop[1], vop[2]);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ new_stmt_info = vect_finish_stmt_generation (loop_vinfo, stmt_info,
+ new_stmt, gsi);
+ }
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ else if (single_defuse_cycle
+ && i < ncopies - 1)
+ {
+ if (reduc_index == 0)
+ vec_oprnds0.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ else if (reduc_index == 1)
+ vec_oprnds1.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ else if (reduc_index == 2)
+ vec_oprnds2.safe_push (gimple_get_lhs (new_stmt_info->stmt));
+ }
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
- if (single_defuse_cycle && !slp_node)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
}
@@ -7080,7 +7010,6 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
- stmt_vec_info prev_phi_info;
int j;
bool nested_cycle = false;
int vec_num;
@@ -7171,14 +7100,17 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
}
vec_initial_def = build_vector_from_val (vectype_out, induc_val);
+ vec_initial_defs.create (ncopies);
+ for (i = 0; i < ncopies; ++i)
+ vec_initial_defs.quick_push (vec_initial_def);
}
else if (nested_cycle)
{
/* Do not use an adjustment def as that case is not supported
correctly if ncopies is not one. */
- vec_initial_def = vect_get_vec_def_for_operand (loop_vinfo,
- initial_def,
- reduc_stmt_info);
+ vect_get_vec_defs_for_operand (loop_vinfo, reduc_stmt_info,
+ ncopies, initial_def,
+ &vec_initial_defs);
}
else
{
@@ -7191,13 +7123,13 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
= get_initial_def_for_reduction (loop_vinfo, reduc_stmt_info, code,
initial_def, adjustment_defp);
STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info) = adjustment_def;
+ vec_initial_defs.create (ncopies);
+ for (i = 0; i < ncopies; ++i)
+ vec_initial_defs.quick_push (vec_initial_def);
}
- vec_initial_defs.create (1);
- vec_initial_defs.quick_push (vec_initial_def);
}
/* Generate the reduction PHIs upfront. */
- prev_phi_info = NULL;
for (i = 0; i < vec_num; i++)
{
tree vec_init_def = vec_initial_defs[i];
@@ -7210,8 +7142,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
/* Set the loop-entry arg of the reduction-phi. */
if (j != 0 && nested_cycle)
- vec_init_def = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_init_def);
+ vec_init_def = vec_initial_defs[j];
add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
UNKNOWN_LOCATION);
@@ -7222,10 +7153,8 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
else
{
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi_info;
- else
- STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info;
- prev_phi_info = new_phi_info;
+ *vec_stmt = new_phi_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info);
}
}
}
@@ -7260,43 +7189,23 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo,
basic_block bb = gimple_bb (stmt_info->stmt);
edge e = single_pred_edge (bb);
tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
- vec<tree> vec_oprnds = vNULL;
- vect_get_vec_defs (loop_vinfo,
- gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE,
- stmt_info, &vec_oprnds, NULL, slp_node);
- if (slp_node)
- {
- unsigned vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
- gcc_assert (vec_oprnds.length () == vec_num);
- for (unsigned i = 0; i < vec_num; i++)
- {
- /* Create the vectorized LC PHI node. */
- gphi *new_phi = create_phi_node (vec_dest, bb);
- add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
- stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
- }
- }
- else
- {
- unsigned ncopies = vect_get_num_copies (loop_vinfo, vectype);
- stmt_vec_info prev_phi_info = NULL;
- for (unsigned i = 0; i < ncopies; i++)
- {
- if (i != 0)
- vect_get_vec_defs_for_stmt_copy (loop_vinfo, &vec_oprnds, NULL);
- /* Create the vectorized LC PHI node. */
- gphi *new_phi = create_phi_node (vec_dest, bb);
- add_phi_arg (new_phi, vec_oprnds[0], e, UNKNOWN_LOCATION);
- stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
- if (i == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi_info;
- else
- STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info;
- prev_phi_info = new_phi_info;
- }
+ auto_vec<tree> vec_oprnds;
+ vect_get_vec_defs (loop_vinfo, stmt_info, slp_node,
+ !slp_node ? vect_get_num_copies (loop_vinfo, vectype) : 1,
+ gimple_phi_arg_def (stmt_info->stmt, 0), &vec_oprnds);
+ for (unsigned i = 0; i < vec_oprnds.length (); i++)
+ {
+ /* Create the vectorized LC PHI node. */
+ gphi *new_phi = create_phi_node (vec_dest, bb);
+ add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
+ stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi);
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info);
}
- vec_oprnds.release ();
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
}
@@ -7668,8 +7577,10 @@ vectorizable_induction (loop_vec_info loop_vinfo,
/* iv_loop is nested in the loop to be vectorized. init_expr had already
been created during vectorization of previous stmts. We obtain it
from the STMT_VINFO_VEC_STMT of the defining stmt. */
- vec_init = vect_get_vec_def_for_operand (loop_vinfo,
- init_expr, stmt_info);
+ auto_vec<tree> vec_inits;
+ vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, 1,
+ init_expr, &vec_inits);
+ vec_init = vec_inits[0];
/* If the initial value is not of proper type, convert it. */
if (!useless_type_conversion_p (vectype, TREE_TYPE (vec_init)))
{
@@ -7807,7 +7718,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
UNKNOWN_LOCATION);
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = induction_phi_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi_info);
+ *vec_stmt = induction_phi_info;
/* In case that vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
@@ -7818,7 +7730,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
if (ncopies > 1)
{
gimple_seq seq = NULL;
- stmt_vec_info prev_stmt_vinfo;
/* FORNOW. This restriction should be relaxed. */
gcc_assert (!nested_in_vect_loop);
@@ -7846,7 +7757,6 @@ vectorizable_induction (loop_vec_info loop_vinfo,
new_vec, step_vectype, NULL);
vec_def = induc_def;
- prev_stmt_vinfo = induction_phi_info;
for (i = 1; i < ncopies; i++)
{
/* vec_i = vec_prev + vec_step */
@@ -7859,8 +7769,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
new_stmt = SSA_NAME_DEF_STMT (vec_def);
new_stmt_info = loop_vinfo->add_stmt (new_stmt);
- STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt_info;
- prev_stmt_vinfo = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
@@ -8047,14 +7956,8 @@ vectorizable_live_operation (loop_vec_info loop_vinfo,
}
else
{
- enum vect_def_type dt = STMT_VINFO_DEF_TYPE (stmt_info);
- vec_lhs = vect_get_vec_def_for_operand_1 (stmt_info, dt);
- gcc_checking_assert (ncopies == 1
- || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
-
/* For multiple copies, get the last copy. */
- for (int i = 1; i < ncopies; ++i)
- vec_lhs = vect_get_vec_def_for_stmt_copy (loop_vinfo, vec_lhs);
+ vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()->stmt);
/* Get the last lane in the vector. */
bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitsize);
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 1d26c312aa8..44944bf8964 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -3671,6 +3671,23 @@ vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
return SLP_TREE_VEC_DEFS (slp_node)[i];
}
+/* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
+
+void
+vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
+{
+ vec_defs->create (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node));
+ if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
+ {
+ unsigned j;
+ stmt_vec_info vec_def_stmt_info;
+ FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt_info)
+ vec_defs->quick_push (gimple_get_lhs (vec_def_stmt_info->stmt));
+ }
+ else
+ vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
+}
+
/* Get N vectorized definitions for SLP_NODE. */
void
@@ -3683,22 +3700,8 @@ vect_get_slp_defs (vec_info *,
for (unsigned i = 0; i < n; ++i)
{
slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
-
vec<tree> vec_defs = vNULL;
-
- /* For each operand we check if it has vectorized definitions in a child
- node or we need to create them (for invariants and constants). */
- vec_defs.create (SLP_TREE_NUMBER_OF_VEC_STMTS (child));
- if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
- {
- unsigned j;
- stmt_vec_info vec_def_stmt_info;
- FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (child), j, vec_def_stmt_info)
- vec_defs.quick_push (gimple_get_lhs (vec_def_stmt_info->stmt));
- }
- else
- vec_defs.splice (SLP_TREE_VEC_DEFS (child));
-
+ vect_get_slp_defs (child, &vec_defs);
vec_oprnds->quick_push (vec_defs);
}
}
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 61adf7d7fa4..b40f19e301c 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -1419,76 +1419,23 @@ vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
return new_temp;
}
-/* Function vect_get_vec_def_for_operand_1.
- For a defining stmt DEF_STMT_INFO of a scalar stmt, return a vector def
- with type DT that will be used in the vectorized stmt. */
+/* Function vect_get_vec_defs_for_operand.
-tree
-vect_get_vec_def_for_operand_1 (stmt_vec_info def_stmt_info,
- enum vect_def_type dt)
-{
- tree vec_oprnd;
- stmt_vec_info vec_stmt_info;
-
- switch (dt)
- {
- /* operand is a constant or a loop invariant. */
- case vect_constant_def:
- case vect_external_def:
- /* Code should use vect_get_vec_def_for_operand. */
- gcc_unreachable ();
-
- /* Operand is defined by a loop header phi. In case of nested
- cycles we also may have uses of the backedge def. */
- case vect_reduction_def:
- case vect_double_reduction_def:
- case vect_nested_cycle:
- case vect_induction_def:
- gcc_assert (gimple_code (def_stmt_info->stmt) == GIMPLE_PHI
- || dt == vect_nested_cycle);
- /* Fallthru. */
-
- /* operand is defined inside the loop. */
- case vect_internal_def:
- {
- /* Get the def from the vectorized stmt. */
- vec_stmt_info = STMT_VINFO_VEC_STMT (def_stmt_info);
- /* Get vectorized pattern statement. */
- if (!vec_stmt_info
- && STMT_VINFO_IN_PATTERN_P (def_stmt_info)
- && !STMT_VINFO_RELEVANT (def_stmt_info))
- vec_stmt_info = (STMT_VINFO_VEC_STMT
- (STMT_VINFO_RELATED_STMT (def_stmt_info)));
- gcc_assert (vec_stmt_info);
- if (gphi *phi = dyn_cast <gphi *> (vec_stmt_info->stmt))
- vec_oprnd = PHI_RESULT (phi);
- else
- vec_oprnd = gimple_get_lhs (vec_stmt_info->stmt);
- return vec_oprnd;
- }
-
- default:
- gcc_unreachable ();
- }
-}
-
-
-/* Function vect_get_vec_def_for_operand.
-
- OP is an operand in STMT_VINFO. This function returns a (vector) def
- that will be used in the vectorized stmt for STMT_VINFO.
+ OP is an operand in STMT_VINFO. This function returns a vector of
+ NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
In the case that OP is an SSA_NAME which is defined in the loop, then
- STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
+ STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
In case OP is an invariant or constant, a new stmt that creates a vector def
needs to be introduced. VECTYPE may be used to specify a required type for
vector invariant. */
-tree
-vect_get_vec_def_for_operand (vec_info *vinfo,
- tree op, stmt_vec_info stmt_vinfo, tree vectype)
+void
+vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo,
+ unsigned ncopies,
+ tree op, vec<tree> *vec_oprnds, tree vectype)
{
gimple *def_stmt;
enum vect_def_type dt;
@@ -1497,7 +1444,7 @@ vect_get_vec_def_for_operand (vec_info *vinfo,
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
- "vect_get_vec_def_for_operand: %T\n", op);
+ "vect_get_vec_defs_for_operand: %T\n", op);
stmt_vec_info def_stmt_info;
is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt,
@@ -1506,6 +1453,7 @@ vect_get_vec_def_for_operand (vec_info *vinfo,
if (def_stmt && dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = %G", def_stmt);
+ vec_oprnds->create (ncopies);
if (dt == vect_constant_def || dt == vect_external_def)
{
tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
@@ -1520,104 +1468,17 @@ vect_get_vec_def_for_operand (vec_info *vinfo,
vector_type = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op));
gcc_assert (vector_type);
- return vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL);
+ tree vop = vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL);
+ while (ncopies--)
+ vec_oprnds->quick_push (vop);
}
else
- return vect_get_vec_def_for_operand_1 (def_stmt_info, dt);
-}
-
-
-/* Function vect_get_vec_def_for_stmt_copy
-
- Return a vector-def for an operand. This function is used when the
- vectorized stmt to be created (by the caller to this function) is a "copy"
- created in case the vectorized result cannot fit in one vector, and several
- copies of the vector-stmt are required. In this case the vector-def is
- retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
- of the stmt that defines VEC_OPRND. VINFO describes the vectorization.
-
- Context:
- In case the vectorization factor (VF) is bigger than the number
- of elements that can fit in a vectype (nunits), we have to generate
- more than one vector stmt to vectorize the scalar stmt. This situation
- arises when there are multiple data-types operated upon in the loop; the
- smallest data-type determines the VF, and as a result, when vectorizing
- stmts operating on wider types we need to create 'VF/nunits' "copies" of the
- vector stmt (each computing a vector of 'nunits' results, and together
- computing 'VF' results in each iteration). This function is called when
- vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
- which VF=16 and nunits=4, so the number of copies required is 4):
-
- scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
-
- S1: x = load VS1.0: vx.0 = memref0 VS1.1
- VS1.1: vx.1 = memref1 VS1.2
- VS1.2: vx.2 = memref2 VS1.3
- VS1.3: vx.3 = memref3
-
- S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
- VSnew.1: vz1 = vx.1 + ... VSnew.2
- VSnew.2: vz2 = vx.2 + ... VSnew.3
- VSnew.3: vz3 = vx.3 + ...
-
- The vectorization of S1 is explained in vectorizable_load.
- The vectorization of S2:
- To create the first vector-stmt out of the 4 copies - VSnew.0 -
- the function 'vect_get_vec_def_for_operand' is called to
- get the relevant vector-def for each operand of S2. For operand x it
- returns the vector-def 'vx.0'.
-
- To create the remaining copies of the vector-stmt (VSnew.j), this
- function is called to get the relevant vector-def for each operand. It is
- obtained from the respective VS1.j stmt, which is recorded in the
- STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
-
- For example, to obtain the vector-def 'vx.1' in order to create the
- vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
- Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
- STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
- and return its def ('vx.1').
- Overall, to create the above sequence this function will be called 3 times:
- vx.1 = vect_get_vec_def_for_stmt_copy (vinfo, vx.0);
- vx.2 = vect_get_vec_def_for_stmt_copy (vinfo, vx.1);
- vx.3 = vect_get_vec_def_for_stmt_copy (vinfo, vx.2); */
-
-tree
-vect_get_vec_def_for_stmt_copy (vec_info *vinfo, tree vec_oprnd)
-{
- stmt_vec_info def_stmt_info = vinfo->lookup_def (vec_oprnd);
- if (!def_stmt_info)
- /* Do nothing; can reuse same def. */
- return vec_oprnd;
-
- def_stmt_info = STMT_VINFO_RELATED_STMT (def_stmt_info);
- gcc_assert (def_stmt_info);
- if (gphi *phi = dyn_cast <gphi *> (def_stmt_info->stmt))
- vec_oprnd = PHI_RESULT (phi);
- else
- vec_oprnd = gimple_get_lhs (def_stmt_info->stmt);
- return vec_oprnd;
-}
-
-
-/* Get vectorized definitions for the operands to create a copy of an original
- stmt. See vect_get_vec_def_for_stmt_copy () for details. */
-
-void
-vect_get_vec_defs_for_stmt_copy (vec_info *vinfo,
- vec<tree> *vec_oprnds0,
- vec<tree> *vec_oprnds1)
-{
- tree vec_oprnd = vec_oprnds0->pop ();
-
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd);
- vec_oprnds0->quick_push (vec_oprnd);
-
- if (vec_oprnds1 && vec_oprnds1->length ())
{
- vec_oprnd = vec_oprnds1->pop ();
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd);
- vec_oprnds1->quick_push (vec_oprnd);
+ def_stmt_info = vect_stmt_to_vectorize (def_stmt_info);
+ gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies);
+ for (unsigned i = 0; i < ncopies; ++i)
+ vec_oprnds->quick_push (gimple_get_lhs
+ (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]->stmt));
}
}
@@ -1625,36 +1486,56 @@ vect_get_vec_defs_for_stmt_copy (vec_info *vinfo,
/* Get vectorized definitions for OP0 and OP1. */
void
-vect_get_vec_defs (vec_info *vinfo, tree op0, tree op1, stmt_vec_info stmt_info,
- vec<tree> *vec_oprnds0,
- vec<tree> *vec_oprnds1,
- slp_tree slp_node)
+vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
+ unsigned ncopies,
+ tree op0, vec<tree> *vec_oprnds0, tree vectype0,
+ tree op1, vec<tree> *vec_oprnds1, tree vectype1,
+ tree op2, vec<tree> *vec_oprnds2, tree vectype2,
+ tree op3, vec<tree> *vec_oprnds3, tree vectype3)
{
if (slp_node)
{
- auto_vec<vec<tree> > vec_defs (SLP_TREE_CHILDREN (slp_node).length ());
- vect_get_slp_defs (vinfo, slp_node, &vec_defs, op1 ? 2 : 1);
- *vec_oprnds0 = vec_defs[0];
+ if (op0)
+ vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
if (op1)
- *vec_oprnds1 = vec_defs[1];
+ vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
+ if (op2)
+ vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
+ if (op3)
+ vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
}
else
{
- tree vec_oprnd;
-
- vec_oprnds0->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (vinfo, op0, stmt_info);
- vec_oprnds0->quick_push (vec_oprnd);
-
+ if (op0)
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ op0, vec_oprnds0, vectype0);
if (op1)
- {
- vec_oprnds1->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (vinfo, op1, stmt_info);
- vec_oprnds1->quick_push (vec_oprnd);
- }
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ op1, vec_oprnds1, vectype1);
+ if (op2)
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ op2, vec_oprnds2, vectype2);
+ if (op3)
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ op3, vec_oprnds3, vectype3);
}
}
+void
+vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
+ unsigned ncopies,
+ tree op0, vec<tree> *vec_oprnds0,
+ tree op1, vec<tree> *vec_oprnds1,
+ tree op2, vec<tree> *vec_oprnds2,
+ tree op3, vec<tree> *vec_oprnds3)
+{
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ op0, vec_oprnds0, NULL_TREE,
+ op1, vec_oprnds1, NULL_TREE,
+ op2, vec_oprnds2, NULL_TREE,
+ op3, vec_oprnds3, NULL_TREE);
+}
+
/* Helper function called by vect_finish_replace_stmt and
vect_finish_stmt_generation. Set the location of the new
statement and create and return a stmt_vec_info for it. */
@@ -1664,6 +1545,7 @@ vect_finish_stmt_generation_1 (vec_info *vinfo,
stmt_vec_info stmt_info, gimple *vec_stmt)
{
stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt);
+ vec_stmt_info->vector_stmt = 1;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
@@ -2787,18 +2669,23 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
}
+ auto_vec<tree> vec_oprnds0;
+ auto_vec<tree> vec_masks;
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ modifier == WIDEN ? ncopies / 2 : ncopies,
+ gs_info->offset, &vec_oprnds0);
+ if (mask)
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ modifier == NARROW ? ncopies / 2 : ncopies,
+ mask, &vec_masks);
for (int j = 0; j < ncopies; ++j)
{
tree op, var;
if (modifier == WIDEN && (j & 1))
op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
perm_mask, stmt_info, gsi);
- else if (j == 0)
- op = vec_oprnd0
- = vect_get_vec_def_for_operand (vinfo, gs_info->offset, stmt_info);
else
- op = vec_oprnd0 = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_oprnd0);
+ op = vec_oprnd0 = vec_oprnds0[modifier == WIDEN ? j / 2 : j];
if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
{
@@ -2818,11 +2705,13 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
mask_perm_mask, stmt_info, gsi);
else
{
- if (j == 0)
- vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info);
- else if (modifier != NARROW || (j & 1) == 0)
- vec_mask = vect_get_vec_def_for_stmt_copy (loop_vinfo,
- vec_mask);
+ if (modifier == NARROW)
+ {
+ if ((j & 1) == 0)
+ vec_mask = vec_masks[j / 2];
+ }
+ else
+ vec_mask = vec_masks[j];
mask_op = vec_mask;
if (!useless_type_conversion_p (masktype, TREE_TYPE (vec_mask)))
@@ -2913,10 +2802,9 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
new_stmt_info = loop_vinfo->lookup_def (var);
}
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
if (prev_stmt_info == NULL)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
+ *vec_stmt = new_stmt_info;
prev_stmt_info = new_stmt_info;
}
}
@@ -2931,7 +2819,8 @@ static void
vect_get_gather_scatter_ops (vec_info *vinfo,
class loop *loop, stmt_vec_info stmt_info,
gather_scatter_info *gs_info,
- tree *dataref_ptr, tree *vec_offset)
+ tree *dataref_ptr, vec<tree> *vec_offset,
+ unsigned ncopies)
{
gimple_seq stmts = NULL;
*dataref_ptr = force_gimple_operand (gs_info->base, &stmts, true, NULL_TREE);
@@ -2942,8 +2831,8 @@ vect_get_gather_scatter_ops (vec_info *vinfo,
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
gcc_assert (!new_bb);
}
- *vec_offset = vect_get_vec_def_for_operand (vinfo, gs_info->offset, stmt_info,
- gs_info->offset_vectype);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies, gs_info->offset,
+ vec_offset, gs_info->offset_vectype);
}
/* Prepare to implement a grouped or strided load or store using
@@ -3085,51 +2974,36 @@ vectorizable_bswap (vec_info *vinfo,
/* Transform. */
vec<tree> vec_oprnds = vNULL;
- stmt_vec_info new_stmt_info = NULL;
- stmt_vec_info prev_stmt_info = NULL;
- for (unsigned j = 0; j < ncopies; j++)
- {
- /* Handle uses. */
- if (j == 0)
- vect_get_vec_defs (vinfo, op, NULL, stmt_info, &vec_oprnds, NULL,
- slp_node);
- else
- vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds, NULL);
-
- /* Arguments are ready. create the new vector stmt. */
- unsigned i;
- tree vop;
- FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
- {
- gimple *new_stmt;
- tree tem = make_ssa_name (char_vectype);
- new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
- char_vectype, vop));
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- tree tem2 = make_ssa_name (char_vectype);
- new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
- tem, tem, bswap_vconst);
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- tem = make_ssa_name (vectype);
- new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
- vectype, tem2));
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
-
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ op, &vec_oprnds);
+ /* Arguments are ready. create the new vector stmt. */
+ unsigned i;
+ tree vop;
+ FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
+ {
+ gimple *new_stmt;
+ tree tem = make_ssa_name (char_vectype);
+ new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
+ char_vectype, vop));
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ tree tem2 = make_ssa_name (char_vectype);
+ new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
+ tem, tem, bswap_vconst);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ tem = make_ssa_name (vectype);
+ new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
+ vectype, tem2));
+ stmt_vec_info new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
vec_oprnds.release ();
return true;
}
@@ -3177,7 +3051,6 @@ vectorizable_call (vec_info *vinfo,
tree scalar_dest;
tree op;
tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
- stmt_vec_info prev_stmt_info;
tree vectype_out, vectype_in;
poly_uint64 nunits_in;
poly_uint64 nunits_out;
@@ -3461,18 +3334,17 @@ vectorizable_call (vec_info *vinfo,
bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
stmt_vec_info new_stmt_info = NULL;
- prev_stmt_info = NULL;
if (modifier == NONE || ifn != IFN_LAST)
{
tree prev_res = NULL_TREE;
vargs.safe_grow (nargs);
orig_vargs.safe_grow (nargs);
+ auto_vec<vec<tree> > vec_defs (nargs);
for (j = 0; j < ncopies; ++j)
{
/* Build argument list for the vectorized call. */
if (slp_node)
{
- auto_vec<vec<tree> > vec_defs (nargs);
vec<tree> vec_oprnds0;
vect_get_slp_defs (vinfo, slp_node, &vec_defs);
@@ -3538,12 +3410,6 @@ vectorizable_call (vec_info *vinfo,
}
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
-
- for (i = 0; i < nargs; i++)
- {
- vec<tree> vec_oprndsi = vec_defs[i];
- vec_oprndsi.release ();
- }
continue;
}
@@ -3551,14 +3417,12 @@ vectorizable_call (vec_info *vinfo,
{
op = gimple_call_arg (stmt, i);
if (j == 0)
- vec_oprnd0
- = vect_get_vec_def_for_operand (vinfo,
- op, stmt_info, vectypes[i]);
- else
- vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (vinfo, orig_vargs[i]);
-
- orig_vargs[i] = vargs[i] = vec_oprnd0;
+ {
+ vec_defs.quick_push (vNULL);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ op, &vec_defs[i]);
+ }
+ orig_vargs[i] = vargs[i] = vec_defs[i][j];
}
if (mask_opno >= 0 && masked_loop_p)
@@ -3618,15 +3482,18 @@ vectorizable_call (vec_info *vinfo,
}
if (j == (modifier == NARROW ? 1 : 0))
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ }
+ for (i = 0; i < nargs; i++)
+ {
+ vec<tree> vec_oprndsi = vec_defs[i];
+ vec_oprndsi.release ();
}
}
else if (modifier == NARROW)
{
+ auto_vec<vec<tree> > vec_defs (nargs);
/* We don't define any narrowing conditional functions at present. */
gcc_assert (mask_opno < 0);
for (j = 0; j < ncopies; ++j)
@@ -3639,7 +3506,6 @@ vectorizable_call (vec_info *vinfo,
if (slp_node)
{
- auto_vec<vec<tree> > vec_defs (nargs);
vec<tree> vec_oprnds0;
vect_get_slp_defs (vinfo, slp_node, &vec_defs);
@@ -3668,12 +3534,6 @@ vectorizable_call (vec_info *vinfo,
= vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
-
- for (i = 0; i < nargs; i++)
- {
- vec<tree> vec_oprndsi = vec_defs[i];
- vec_oprndsi.release ();
- }
continue;
}
@@ -3682,21 +3542,12 @@ vectorizable_call (vec_info *vinfo,
op = gimple_call_arg (stmt, i);
if (j == 0)
{
- vec_oprnd0
- = vect_get_vec_def_for_operand (vinfo, op, stmt_info,
- vectypes[i]);
- vec_oprnd1
- = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd0);
- }
- else
- {
- vec_oprnd1 = gimple_call_arg (new_stmt_info->stmt,
- 2 * i + 1);
- vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd1);
- vec_oprnd1
- = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd0);
+ vec_defs.quick_push (vNULL);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, 2 * ncopies,
+ op, &vec_defs[i], vectypes[i]);
}
+ vec_oprnd0 = vec_defs[i][2*j];
+ vec_oprnd1 = vec_defs[i][2*j+1];
vargs.quick_push (vec_oprnd0);
vargs.quick_push (vec_oprnd1);
@@ -3708,15 +3559,17 @@ vectorizable_call (vec_info *vinfo,
new_stmt_info
= vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
- *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
+ for (i = 0; i < nargs; i++)
+ {
+ vec<tree> vec_oprndsi = vec_defs[i];
+ vec_oprndsi.release ();
+ }
}
else
/* No current target implements this case. */
@@ -3849,7 +3702,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
tree scalar_dest;
tree op, type;
tree vec_oprnd0 = NULL_TREE;
- stmt_vec_info prev_stmt_info;
tree vectype;
unsigned int nunits;
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -4176,7 +4028,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
}
}
- prev_stmt_info = NULL;
+ auto_vec<vec<tree> > vec_oprnds;
+ auto_vec<unsigned> vec_oprnds_i;
+ vec_oprnds.safe_grow_cleared (nargs);
+ vec_oprnds_i.safe_grow_cleared (nargs);
for (j = 0; j < ncopies; ++j)
{
/* Build argument list for the vectorized call. */
@@ -4205,15 +4060,18 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
/ simd_clone_subparts (atype));
gcc_assert ((k & (k - 1)) == 0);
if (m == 0)
- vec_oprnd0
- = vect_get_vec_def_for_operand (vinfo, op, stmt_info);
+ {
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ ncopies * o / k, op,
+ &vec_oprnds[i]);
+ vec_oprnds_i[i] = 0;
+ vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
+ }
else
{
vec_oprnd0 = arginfo[i].op;
if ((m & (k - 1)) == 0)
- vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd0);
+ vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
}
arginfo[i].op = vec_oprnd0;
vec_oprnd0
@@ -4240,13 +4098,16 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
for (l = 0; l < k; l++)
{
if (m == 0 && l == 0)
- vec_oprnd0
- = vect_get_vec_def_for_operand (vinfo,
- op, stmt_info);
+ {
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ k * o * ncopies,
+ op,
+ &vec_oprnds[i]);
+ vec_oprnds_i[i] = 0;
+ vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
+ }
else
- vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (vinfo,
- arginfo[i].op);
+ vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
arginfo[i].op = vec_oprnd0;
if (k == 1)
break;
@@ -4390,12 +4251,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
new_stmt, gsi);
if (j == 0 && l == 0)
- STMT_VINFO_VEC_STMT (stmt_info)
- = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
if (ratype)
@@ -4437,11 +4294,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
= vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if ((unsigned) j == k - 1)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
continue;
}
else if (ratype)
@@ -4458,13 +4312,15 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
}
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ for (i = 0; i < nargs; ++i)
+ {
+ vec<tree> oprndsi = vec_oprnds[i];
+ oprndsi.release ();
+ }
vargs.release ();
/* The call in STMT might prevent it from being removed in dce.
@@ -4522,43 +4378,6 @@ vect_gen_widened_results_half (vec_info *vinfo, enum tree_code code,
}
-/* Get vectorized definitions for loop-based vectorization of STMT_INFO.
- For the first operand we call vect_get_vec_def_for_operand (with OPRND
- containing scalar operand), and for the rest we get a copy with
- vect_get_vec_def_for_stmt_copy() using the previous vector definition
- (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
- The vectors are collected into VEC_OPRNDS. */
-
-static void
-vect_get_loop_based_defs (vec_info *vinfo, tree *oprnd, stmt_vec_info stmt_info,
- vec<tree> *vec_oprnds, int multi_step_cvt)
-{
- tree vec_oprnd;
-
- /* Get first vector operand. */
- /* All the vector operands except the very first one (that is scalar oprnd)
- are stmt copies. */
- if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
- vec_oprnd = vect_get_vec_def_for_operand (vinfo, *oprnd, stmt_info);
- else
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, *oprnd);
-
- vec_oprnds->quick_push (vec_oprnd);
-
- /* Get second vector operand. */
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd);
- vec_oprnds->quick_push (vec_oprnd);
-
- *oprnd = vec_oprnd;
-
- /* For conversion in multiple steps, continue to get operands
- recursively. */
- if (multi_step_cvt)
- vect_get_loop_based_defs (vinfo, oprnd, stmt_info, vec_oprnds,
- multi_step_cvt - 1);
-}
-
-
/* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
For multi-step conversions store the resulting vectors and call the function
recursively. */
@@ -4569,8 +4388,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
stmt_vec_info stmt_info,
vec<tree> vec_dsts,
gimple_stmt_iterator *gsi,
- slp_tree slp_node, enum tree_code code,
- stmt_vec_info *prev_stmt_info)
+ slp_tree slp_node, enum tree_code code)
{
unsigned int i;
tree vop0, vop1, new_tmp, vec_dest;
@@ -4599,14 +4417,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- {
- if (!*prev_stmt_info)
- STMT_VINFO_VEC_STMT (stmt_info) = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt_info;
-
- *prev_stmt_info = new_stmt_info;
- }
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
@@ -4622,8 +4433,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
multi_step_cvt - 1,
stmt_info, vec_dsts, gsi,
- slp_node, VEC_PACK_TRUNC_EXPR,
- prev_stmt_info);
+ slp_node, VEC_PACK_TRUNC_EXPR);
}
vec_dsts.quick_push (vec_dest);
@@ -4699,18 +4509,16 @@ vectorizable_conversion (vec_info *vinfo,
tree vec_dest;
tree scalar_dest;
tree op0, op1 = NULL_TREE;
- tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
tree new_temp;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
int ndts = 2;
- stmt_vec_info prev_stmt_info;
poly_uint64 nunits_in;
poly_uint64 nunits_out;
tree vectype_out, vectype_in;
- int ncopies, i, j;
+ int ncopies, i;
tree lhs_type, rhs_type;
enum { NARROW, NONE, WIDEN } modifier;
vec<tree> vec_oprnds0 = vNULL;
@@ -4719,7 +4527,7 @@ vectorizable_conversion (vec_info *vinfo,
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
int multi_step_cvt = 0;
vec<tree> interm_types = vNULL;
- tree last_oprnd, intermediate_type, cvt_type = NULL_TREE;
+ tree intermediate_type, cvt_type = NULL_TREE;
int op_type;
unsigned short fltsz;
@@ -5059,57 +4867,39 @@ vectorizable_conversion (vec_info *vinfo,
modifier == WIDEN
? vectype_out : cvt_type);
+ int ninputs = 1;
if (!slp_node)
{
if (modifier == WIDEN)
+ ;
+ else if (modifier == NARROW)
{
- vec_oprnds0.create (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1);
- if (op_type == binary_op)
- vec_oprnds1.create (1);
+ if (multi_step_cvt)
+ ninputs = vect_pow2 (multi_step_cvt);
+ ninputs *= 2;
}
- else if (modifier == NARROW)
- vec_oprnds0.create (
- 2 * (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
}
- else if (code == WIDEN_LSHIFT_EXPR)
- vec_oprnds1.create (slp_node->vec_stmts_size);
- last_oprnd = op0;
- prev_stmt_info = NULL;
switch (modifier)
{
case NONE:
- for (j = 0; j < ncopies; j++)
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ op0, &vec_oprnds0);
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
{
- if (j == 0)
- vect_get_vec_defs (vinfo, op0, NULL, stmt_info, &vec_oprnds0,
- NULL, slp_node);
- else
- vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds0, NULL);
-
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
- {
- stmt_vec_info new_stmt_info;
- /* Arguments are ready, create the new vector stmt. */
- gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
- gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ stmt_vec_info new_stmt_info;
+ /* Arguments are ready, create the new vector stmt. */
+ gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
+ gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- else
- {
- if (!prev_stmt_info)
- STMT_VINFO_VEC_STMT (stmt_info)
- = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
- }
- }
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
break;
@@ -5118,111 +4908,54 @@ vectorizable_conversion (vec_info *vinfo,
of elements that we can fit in a vectype (nunits), we have to
generate more than one vector stmt - i.e - we need to "unroll"
the vector stmt by a factor VF/nunits. */
- for (j = 0; j < ncopies; j++)
- {
- /* Handle uses. */
- if (j == 0)
- {
- if (slp_node)
- {
- if (code == WIDEN_LSHIFT_EXPR)
- {
- unsigned int k;
-
- vec_oprnd1 = op1;
- /* Store vec_oprnd1 for every vector stmt to be created
- for SLP_NODE. We check during the analysis that all
- the shift arguments are the same. */
- for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
- vec_oprnds1.quick_push (vec_oprnd1);
-
- vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info,
- &vec_oprnds0, NULL, slp_node);
- }
- else
- vect_get_vec_defs (vinfo, op0, op1, stmt_info, &vec_oprnds0,
- &vec_oprnds1, slp_node);
- }
- else
- {
- vec_oprnd0 = vect_get_vec_def_for_operand (vinfo,
- op0, stmt_info);
- vec_oprnds0.quick_push (vec_oprnd0);
- if (op_type == binary_op)
- {
- if (code == WIDEN_LSHIFT_EXPR)
- vec_oprnd1 = op1;
- else
- vec_oprnd1
- = vect_get_vec_def_for_operand (vinfo,
- op1, stmt_info);
- vec_oprnds1.quick_push (vec_oprnd1);
- }
- }
- }
- else
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies * ninputs,
+ op0, &vec_oprnds0,
+ code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
+ &vec_oprnds1);
+ if (code == WIDEN_LSHIFT_EXPR)
+ {
+ vec_oprnds1.create (ncopies * ninputs);
+ for (i = 0; i < ncopies * ninputs; ++i)
+ vec_oprnds1.quick_push (op1);
+ }
+ /* Arguments are ready. Create the new vector stmts. */
+ for (i = multi_step_cvt; i >= 0; i--)
+ {
+ tree this_dest = vec_dsts[i];
+ enum tree_code c1 = code1, c2 = code2;
+ if (i == 0 && codecvt2 != ERROR_MARK)
{
- vec_oprnd0 = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd0);
- vec_oprnds0.truncate (0);
- vec_oprnds0.quick_push (vec_oprnd0);
- if (op_type == binary_op)
- {
- if (code == WIDEN_LSHIFT_EXPR)
- vec_oprnd1 = op1;
- else
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd1);
- vec_oprnds1.truncate (0);
- vec_oprnds1.quick_push (vec_oprnd1);
- }
+ c1 = codecvt1;
+ c2 = codecvt2;
}
+ vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
+ &vec_oprnds1, stmt_info,
+ this_dest, gsi,
+ c1, c2, op_type);
+ }
- /* Arguments are ready. Create the new vector stmts. */
- for (i = multi_step_cvt; i >= 0; i--)
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
+ {
+ stmt_vec_info new_stmt_info;
+ if (cvt_type)
{
- tree this_dest = vec_dsts[i];
- enum tree_code c1 = code1, c2 = code2;
- if (i == 0 && codecvt2 != ERROR_MARK)
- {
- c1 = codecvt1;
- c2 = codecvt2;
- }
- vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
- &vec_oprnds1, stmt_info,
- this_dest, gsi,
- c1, c2, op_type);
+ gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
+ new_temp = make_ssa_name (vec_dest);
+ gassign *new_stmt
+ = gimple_build_assign (new_temp, codecvt1, vop0);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
+ else
+ /* This use goes away. */
+ new_stmt_info = vinfo->lookup_stmt (SSA_NAME_DEF_STMT (vop0),
+ false, true);
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
- {
- stmt_vec_info new_stmt_info;
- if (cvt_type)
- {
- gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
- new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt
- = gimple_build_assign (new_temp, codecvt1, vop0);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
- }
- else
- new_stmt_info = vinfo->lookup_def (vop0);
-
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- else
- {
- if (!prev_stmt_info)
- STMT_VINFO_VEC_STMT (stmt_info) = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
- }
- }
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
-
- *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
break;
case NARROW:
@@ -5230,42 +4963,28 @@ vectorizable_conversion (vec_info *vinfo,
of elements that we can fit in a vectype (nunits), we have to
generate more than one vector stmt - i.e - we need to "unroll"
the vector stmt by a factor VF/nunits. */
- for (j = 0; j < ncopies; j++)
- {
- /* Handle uses. */
- if (slp_node)
- vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info, &vec_oprnds0,
- NULL, slp_node);
- else
- {
- vec_oprnds0.truncate (0);
- vect_get_loop_based_defs (vinfo,
- &last_oprnd, stmt_info, &vec_oprnds0,
- vect_pow2 (multi_step_cvt) - 1);
- }
-
- /* Arguments are ready. Create the new vector stmts. */
- if (cvt_type)
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
- {
- gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
- new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt
- = gimple_build_assign (new_temp, codecvt1, vop0);
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- vec_oprnds0[i] = new_temp;
- }
-
- vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
- multi_step_cvt,
- stmt_info, vec_dsts, gsi,
- slp_node, code1,
- &prev_stmt_info);
- }
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies * ninputs,
+ op0, &vec_oprnds0);
+ /* Arguments are ready. Create the new vector stmts. */
+ if (cvt_type)
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
+ {
+ gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
+ new_temp = make_ssa_name (vec_dest);
+ gassign *new_stmt
+ = gimple_build_assign (new_temp, codecvt1, vop0);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vec_oprnds0[i] = new_temp;
+ }
- *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
+ multi_step_cvt,
+ stmt_info, vec_dsts, gsi,
+ slp_node, code1);
break;
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
vec_oprnds0.release ();
vec_oprnds1.release ();
@@ -5319,11 +5038,10 @@ vectorizable_assignment (vec_info *vinfo,
enum vect_def_type dt[1] = {vect_unknown_def_type};
int ndts = 1;
int ncopies;
- int i, j;
+ int i;
vec<tree> vec_oprnds = vNULL;
tree vop;
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
- stmt_vec_info prev_stmt_info = NULL;
enum tree_code code;
tree vectype_in;
@@ -5436,41 +5154,27 @@ vectorizable_assignment (vec_info *vinfo,
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle use. */
- for (j = 0; j < ncopies; j++)
- {
- /* Handle uses. */
- if (j == 0)
- vect_get_vec_defs (vinfo, op, NULL, stmt_info, &vec_oprnds, NULL,
- slp_node);
- else
- vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds, NULL);
-
- /* Arguments are ready. create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
- FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
- {
- if (CONVERT_EXPR_CODE_P (code)
- || code == VIEW_CONVERT_EXPR)
- vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
- gassign *new_stmt = gimple_build_assign (vec_dest, vop);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds);
+ /* Arguments are ready. create the new vector stmt. */
+ stmt_vec_info new_stmt_info = NULL;
+ FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
+ {
+ if (CONVERT_EXPR_CODE_P (code)
+ || code == VIEW_CONVERT_EXPR)
+ vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
+ gassign *new_stmt = gimple_build_assign (vec_dest, vop);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
vec_oprnds.release ();
return true;
@@ -5540,13 +5244,12 @@ vectorizable_shift (vec_info *vinfo,
machine_mode optab_op2_mode;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
int ndts = 2;
- stmt_vec_info prev_stmt_info;
poly_uint64 nunits_in;
poly_uint64 nunits_out;
tree vectype_out;
tree op1_vectype;
int ncopies;
- int j, i;
+ int i;
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
tree vop0, vop1;
@@ -5850,99 +5553,79 @@ vectorizable_shift (vec_info *vinfo,
/* Handle def. */
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- prev_stmt_info = NULL;
- for (j = 0; j < ncopies; j++)
+ if (scalar_shift_arg)
{
- /* Handle uses. */
- if (j == 0)
- {
- if (scalar_shift_arg)
- {
- /* Vector shl and shr insn patterns can be defined with scalar
- operand 2 (shift operand). In this case, use constant or loop
- invariant op1 directly, without extending it to vector mode
- first. */
- optab_op2_mode = insn_data[icode].operand[2].mode;
- if (!VECTOR_MODE_P (optab_op2_mode))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "operand 1 using scalar mode.\n");
- vec_oprnd1 = op1;
- vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : 1);
- vec_oprnds1.quick_push (vec_oprnd1);
- if (slp_node)
- {
- /* Store vec_oprnd1 for every vector stmt to be created
- for SLP_NODE. We check during the analysis that all
- the shift arguments are the same.
- TODO: Allow different constants for different vector
- stmts generated for an SLP instance. */
- for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
- vec_oprnds1.quick_push (vec_oprnd1);
- }
- }
- }
- else if (slp_node && incompatible_op1_vectype_p)
- {
- if (was_scalar_shift_arg)
- {
- /* If the argument was the same in all lanes create
- the correctly typed vector shift amount directly. */
- op1 = fold_convert (TREE_TYPE (vectype), op1);
- op1 = vect_init_vector (vinfo, stmt_info,
- op1, TREE_TYPE (vectype),
- !loop_vinfo ? gsi : NULL);
- vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
- !loop_vinfo ? gsi : NULL);
- vec_oprnds1.create (slp_node->vec_stmts_size);
- for (k = 0; k < slp_node->vec_stmts_size; k++)
- vec_oprnds1.quick_push (vec_oprnd1);
- }
- else if (dt[1] == vect_constant_def)
- /* The constant shift amount has been adjusted in place. */
- ;
- else
- gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
- }
-
- /* vec_oprnd1 is available if operand 1 should be of a scalar-type
- (a special case for certain kind of vector shifts); otherwise,
- operand 1 should be of a vector type (the usual case). */
- if (vec_oprnd1)
- vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info,
- &vec_oprnds0, NULL, slp_node);
- else
- vect_get_vec_defs (vinfo, op0, op1, stmt_info,
- &vec_oprnds0, &vec_oprnds1, slp_node);
- }
+ /* Vector shl and shr insn patterns can be defined with scalar
+ operand 2 (shift operand). In this case, use constant or loop
+ invariant op1 directly, without extending it to vector mode
+ first. */
+ optab_op2_mode = insn_data[icode].operand[2].mode;
+ if (!VECTOR_MODE_P (optab_op2_mode))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "operand 1 using scalar mode.\n");
+ vec_oprnd1 = op1;
+ vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : ncopies);
+ vec_oprnds1.quick_push (vec_oprnd1);
+ /* Store vec_oprnd1 for every vector stmt to be created.
+ We check during the analysis that all the shift arguments
+ are the same.
+ TODO: Allow different constants for different vector
+ stmts generated for an SLP instance. */
+ for (k = 0;
+ k < (slp_node ? slp_node->vec_stmts_size - 1 : ncopies - 1); k++)
+ vec_oprnds1.quick_push (vec_oprnd1);
+ }
+ }
+ else if (slp_node && incompatible_op1_vectype_p)
+ {
+ if (was_scalar_shift_arg)
+ {
+ /* If the argument was the same in all lanes create
+ the correctly typed vector shift amount directly. */
+ op1 = fold_convert (TREE_TYPE (vectype), op1);
+ op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
+ !loop_vinfo ? gsi : NULL);
+ vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
+ !loop_vinfo ? gsi : NULL);
+ vec_oprnds1.create (slp_node->vec_stmts_size);
+ for (k = 0; k < slp_node->vec_stmts_size; k++)
+ vec_oprnds1.quick_push (vec_oprnd1);
+ }
+ else if (dt[1] == vect_constant_def)
+ /* The constant shift amount has been adjusted in place. */
+ ;
else
- vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds0, &vec_oprnds1);
+ gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
+ }
- /* Arguments are ready. Create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
- {
- vop1 = vec_oprnds1[i];
- gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
+ /* vec_oprnd1 is available if operand 1 should be of a scalar-type
+ (a special case for certain kind of vector shifts); otherwise,
+ operand 1 should be of a vector type (the usual case). */
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ op0, &vec_oprnds0,
+ vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1);
+ /* Arguments are ready. Create the new vector stmt. */
+ stmt_vec_info new_stmt_info = NULL;
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
+ {
+ vop1 = vec_oprnds1[i];
+ gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
vec_oprnds0.release ();
vec_oprnds1.release ();
@@ -5978,12 +5661,11 @@ vectorizable_operation (vec_info *vinfo,
enum vect_def_type dt[3]
= {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
int ndts = 3;
- stmt_vec_info prev_stmt_info;
poly_uint64 nunits_in;
poly_uint64 nunits_out;
tree vectype_out;
int ncopies, vec_num;
- int j, i;
+ int i;
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
vec<tree> vec_oprnds2 = vNULL;
@@ -6326,112 +6008,66 @@ vectorizable_operation (vec_info *vinfo,
VS1_2: vx2 = memref2 VS1_3 -
VS1_3: vx3 = memref3 - -
S1: x = load - VS1_0
- VS2_0: vz0 = vx0 + v1 VS2_1 -
- VS2_1: vz1 = vx1 + v1 VS2_2 -
- VS2_2: vz2 = vx2 + v1 VS2_3 -
- VS2_3: vz3 = vx3 + v1 - -
- S2: z = x + 1 - VS2_0 */
-
- prev_stmt_info = NULL;
- for (j = 0; j < ncopies; j++)
- {
- /* Handle uses. */
- if (j == 0)
- {
- if (op_type == binary_op)
- vect_get_vec_defs (vinfo, op0, op1, stmt_info,
- &vec_oprnds0, &vec_oprnds1, slp_node);
- else if (op_type == ternary_op)
- {
- if (slp_node)
- {
- auto_vec<vec<tree> > vec_defs(3);
- vect_get_slp_defs (vinfo, slp_node, &vec_defs);
- vec_oprnds0 = vec_defs[0];
- vec_oprnds1 = vec_defs[1];
- vec_oprnds2 = vec_defs[2];
- }
- else
- {
- vect_get_vec_defs (vinfo, op0, op1, stmt_info, &vec_oprnds0,
- &vec_oprnds1, NULL);
- vect_get_vec_defs (vinfo, op2, NULL_TREE, stmt_info,
- &vec_oprnds2, NULL, NULL);
- }
- }
- else
- vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info, &vec_oprnds0,
- NULL, slp_node);
- }
- else
- {
- vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds0, &vec_oprnds1);
- if (op_type == ternary_op)
- {
- tree vec_oprnd = vec_oprnds2.pop ();
- vec_oprnds2.quick_push (vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd));
- }
- }
-
- /* Arguments are ready. Create the new vector stmt. */
- stmt_vec_info new_stmt_info = NULL;
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
- {
- vop1 = ((op_type == binary_op || op_type == ternary_op)
- ? vec_oprnds1[i] : NULL_TREE);
- vop2 = ((op_type == ternary_op)
- ? vec_oprnds2[i] : NULL_TREE);
- if (masked_loop_p && reduc_idx >= 0)
- {
- /* Perform the operation on active elements only and take
- inactive elements from the reduction chain input. */
- gcc_assert (!vop2);
- vop2 = reduc_idx == 1 ? vop1 : vop0;
- tree mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
- vectype, i * ncopies + j);
- gcall *call = gimple_build_call_internal (cond_fn, 4, mask,
- vop0, vop1, vop2);
- new_temp = make_ssa_name (vec_dest, call);
- gimple_call_set_lhs (call, new_temp);
- gimple_call_set_nothrow (call, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
- }
- else
+ VS2_0: vz0 = vx0 + v1 VS2_1 -
+ VS2_1: vz1 = vx1 + v1 VS2_2 -
+ VS2_2: vz2 = vx2 + v1 VS2_3 -
+ VS2_3: vz3 = vx3 + v1 - -
+ S2: z = x + 1 - VS2_0 */
+
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2);
+ /* Arguments are ready. Create the new vector stmt. */
+ stmt_vec_info new_stmt_info = NULL;
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
+ {
+ vop1 = ((op_type == binary_op || op_type == ternary_op)
+ ? vec_oprnds1[i] : NULL_TREE);
+ vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
+ if (masked_loop_p && reduc_idx >= 0)
+ {
+ /* Perform the operation on active elements only and take
+ inactive elements from the reduction chain input. */
+ gcc_assert (!vop2);
+ vop2 = reduc_idx == 1 ? vop1 : vop0;
+ tree mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+ vectype, i);
+ gcall *call = gimple_build_call_internal (cond_fn, 4, mask,
+ vop0, vop1, vop2);
+ new_temp = make_ssa_name (vec_dest, call);
+ gimple_call_set_lhs (call, new_temp);
+ gimple_call_set_nothrow (call, true);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ }
+ else
+ {
+ gassign *new_stmt = gimple_build_assign (vec_dest, code,
+ vop0, vop1, vop2);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ if (vec_cvt_dest)
{
- gassign *new_stmt = gimple_build_assign (vec_dest, code,
- vop0, vop1, vop2);
- new_temp = make_ssa_name (vec_dest, new_stmt);
+ new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
+ gassign *new_stmt
+ = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
+ new_temp);
+ new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (vec_cvt_dest)
- {
- new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
- gassign *new_stmt
- = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
- new_temp);
- new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
- gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
- }
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
-
+ }
if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
vec_oprnds0.release ();
vec_oprnds1.release ();
vec_oprnds2.release ();
@@ -7189,7 +6825,6 @@ vectorizable_scan_store (vec_info *vinfo,
perms[i] = vect_gen_perm_mask_checked (vectype, indices);
}
- stmt_vec_info prev_stmt_info = NULL;
tree vec_oprnd1 = NULL_TREE;
tree vec_oprnd2 = NULL_TREE;
tree vec_oprnd3 = NULL_TREE;
@@ -7201,26 +6836,24 @@ vectorizable_scan_store (vec_info *vinfo,
tree orig = NULL_TREE;
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
+ auto_vec<tree> vec_oprnds1;
+ auto_vec<tree> vec_oprnds2;
+ auto_vec<tree> vec_oprnds3;
+ vect_get_vec_defs (vinfo, stmt_info, NULL, ncopies,
+ *init, &vec_oprnds1,
+ ldataref_ptr == NULL ? rhs1 : NULL, &vec_oprnds2,
+ rhs2, &vec_oprnds3);
for (int j = 0; j < ncopies; j++)
{
stmt_vec_info new_stmt_info;
+ vec_oprnd1 = vec_oprnds1[j];
+ if (ldataref_ptr == NULL)
+ vec_oprnd2 = vec_oprnds2[j];
+ vec_oprnd3 = vec_oprnds3[j];
if (j == 0)
- {
- vec_oprnd1 = vect_get_vec_def_for_operand (vinfo, *init, stmt_info);
- if (ldataref_ptr == NULL)
- vec_oprnd2 = vect_get_vec_def_for_operand (vinfo, rhs1, stmt_info);
- vec_oprnd3 = vect_get_vec_def_for_operand (vinfo, rhs2, stmt_info);
- orig = vec_oprnd3;
- }
- else
- {
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd1);
- if (ldataref_ptr == NULL)
- vec_oprnd2 = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd2);
- vec_oprnd3 = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd3);
- if (!inscan_var_store)
- dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
- }
+ orig = vec_oprnd3;
+ else if (!inscan_var_store)
+ dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
if (ldataref_ptr)
{
@@ -7231,11 +6864,8 @@ vectorizable_scan_store (vec_info *vinfo,
vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- if (prev_stmt_info == NULL)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
}
tree v = vec_oprnd2;
@@ -7249,11 +6879,8 @@ vectorizable_scan_store (vec_info *vinfo,
? zero_vec : vec_oprnd1, v,
perms[i]);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- if (prev_stmt_info == NULL)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
{
@@ -7270,8 +6897,7 @@ vectorizable_scan_store (vec_info *vinfo,
new_temp, vec_oprnd1);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
new_temp = new_temp2;
}
@@ -7289,8 +6915,7 @@ vectorizable_scan_store (vec_info *vinfo,
tree new_temp2 = make_ssa_name (vectype);
g = gimple_build_assign (new_temp2, code, v, new_temp);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
v = new_temp2;
}
@@ -7298,8 +6923,7 @@ vectorizable_scan_store (vec_info *vinfo,
tree new_temp = make_ssa_name (vectype);
gimple *g = gimple_build_assign (new_temp, code, orig, v);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
tree last_perm_arg = new_temp;
/* For exclusive scan, new_temp computed above is the exclusive scan
@@ -7310,16 +6934,14 @@ vectorizable_scan_store (vec_info *vinfo,
last_perm_arg = make_ssa_name (vectype);
g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
orig = make_ssa_name (vectype);
g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
last_perm_arg, perms[units_log2]);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
if (!inscan_var_store)
{
@@ -7329,8 +6951,7 @@ vectorizable_scan_store (vec_info *vinfo,
vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
g = gimple_build_assign (data_ref, new_temp);
new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
@@ -7347,8 +6968,7 @@ vectorizable_scan_store (vec_info *vinfo,
gimple *g = gimple_build_assign (data_ref, orig);
stmt_vec_info new_stmt_info
= vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
return true;
}
@@ -7379,7 +6999,6 @@ vectorizable_store (vec_info *vinfo,
enum dr_alignment_support alignment_support_scheme;
enum vect_def_type rhs_dt = vect_unknown_def_type;
enum vect_def_type mask_dt = vect_unknown_def_type;
- stmt_vec_info prev_stmt_info = NULL;
tree dataref_ptr = NULL_TREE;
tree dataref_offset = NULL_TREE;
gimple *ptr_incr = NULL;
@@ -7668,57 +7287,55 @@ vectorizable_store (vec_info *vinfo,
scale = build_int_cst (scaletype, gs_info.scale);
- prev_stmt_info = NULL;
+ auto_vec<tree> vec_oprnds0;
+ auto_vec<tree> vec_oprnds1;
+ auto_vec<tree> vec_masks;
+ if (mask)
+ {
+ tree mask_vectype = truth_type_for (vectype);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ modifier == NARROW
+ ? ncopies / 2 : ncopies,
+ mask, &vec_masks, mask_vectype);
+ }
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ modifier == WIDEN
+ ? ncopies / 2 : ncopies,
+ gs_info.offset, &vec_oprnds0);
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ modifier == NARROW
+ ? ncopies / 2 : ncopies,
+ op, &vec_oprnds1);
for (j = 0; j < ncopies; ++j)
{
- if (j == 0)
+ if (modifier == WIDEN)
{
- src = vec_oprnd1 = vect_get_vec_def_for_operand (vinfo,
- op, stmt_info);
- op = vec_oprnd0 = vect_get_vec_def_for_operand (vinfo,
- gs_info.offset,
- stmt_info);
+ if (j & 1)
+ op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
+ perm_mask, stmt_info, gsi);
+ else
+ op = vec_oprnd0 = vec_oprnds0[j / 2];
+ src = vec_oprnd1 = vec_oprnds1[j];
if (mask)
- {
- tree mask_vectype = truth_type_for (vectype);
- mask_op = vec_mask
- = vect_get_vec_def_for_operand (vinfo, mask,
- stmt_info, mask_vectype);
- }
+ mask_op = vec_mask = vec_masks[j];
}
- else if (modifier != NONE && (j & 1))
+ else if (modifier == NARROW)
{
- if (modifier == WIDEN)
- {
- src
- = vec_oprnd1 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd1);
- op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
- perm_mask, stmt_info, gsi);
- if (mask)
- mask_op
- = vec_mask = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_mask);
- }
- else if (modifier == NARROW)
- {
- src = permute_vec_elements (vinfo, vec_oprnd1, vec_oprnd1,
- perm_mask, stmt_info, gsi);
- op = vec_oprnd0 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd0);
- }
+ if (j & 1)
+ src = permute_vec_elements (vinfo, vec_oprnd1, vec_oprnd1,
+ perm_mask, stmt_info, gsi);
else
- gcc_unreachable ();
+ src = vec_oprnd1 = vec_oprnds1[j / 2];
+ op = vec_oprnd0 = vec_oprnds0[j];
+ if (mask)
+ mask_op = vec_mask = vec_masks[j / 2];
}
else
{
- src = vec_oprnd1 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd1);
- op = vec_oprnd0 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd0);
+ op = vec_oprnd0 = vec_oprnds0[j];
+ src = vec_oprnd1 = vec_oprnds1[j];
if (mask)
- mask_op = vec_mask = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_mask);
+ mask_op = vec_mask = vec_masks[j];
}
if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
@@ -7787,12 +7404,9 @@ vectorizable_store (vec_info *vinfo,
stmt_vec_info new_stmt_info
= vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (prev_stmt_info == NULL)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
}
else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
@@ -7972,7 +7586,6 @@ vectorizable_store (vec_info *vinfo,
stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
- prev_stmt_info = NULL;
alias_off = build_int_cst (ref_type, 0);
stmt_vec_info next_stmt_info = first_stmt_info;
for (g = 0; g < group_size; g++)
@@ -7989,36 +7602,16 @@ vectorizable_store (vec_info *vinfo,
vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
running_off = newoff;
}
+ if (!slp)
+ op = vect_get_store_rhs (next_stmt_info);
+ vect_get_vec_defs (vinfo, next_stmt_info, slp_node, ncopies,
+ op, &vec_oprnds);
unsigned int group_el = 0;
unsigned HOST_WIDE_INT
elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
for (j = 0; j < ncopies; j++)
{
- /* We've set op and dt above, from vect_get_store_rhs,
- and first_stmt_info == stmt_info. */
- if (j == 0)
- {
- if (slp)
- {
- vect_get_vec_defs (vinfo, op, NULL_TREE, stmt_info,
- &vec_oprnds, NULL, slp_node);
- vec_oprnd = vec_oprnds[0];
- }
- else
- {
- op = vect_get_store_rhs (next_stmt_info);
- vec_oprnd = vect_get_vec_def_for_operand
- (vinfo, op, next_stmt_info);
- }
- }
- else
- {
- if (slp)
- vec_oprnd = vec_oprnds[j];
- else
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnd);
- }
+ vec_oprnd = vec_oprnds[j];
/* Pun the vector to extract from if necessary. */
if (lvectype != vectype)
{
@@ -8072,11 +7665,8 @@ vectorizable_store (vec_info *vinfo,
&& !slp)
{
if (j == 0 && i == 0)
- STMT_VINFO_VEC_STMT (stmt_info)
- = *vec_stmt = assign_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = assign_info;
- prev_stmt_info = assign_info;
+ *vec_stmt = assign_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign_info);
}
}
}
@@ -8147,8 +7737,7 @@ vectorizable_store (vec_info *vinfo,
/* In case the vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
more than one vector stmt - i.e - we need to "unroll" the
- vector stmt by a factor VF/nunits. For more details see documentation in
- vect_get_vec_def_for_copy_stmt. */
+ vector stmt by a factor VF/nunits. */
/* In case of interleaving (non-unit grouped access):
@@ -8183,8 +7772,11 @@ vectorizable_store (vec_info *vinfo,
STMT_VINFO_RELATED_STMT for the next copies.
*/
- prev_stmt_info = NULL;
- tree vec_mask = NULL_TREE;
+ auto_vec<tree> vec_masks;
+ tree vec_mask = NULL;
+ auto_vec<tree> vec_offsets;
+ auto_vec<vec<tree> > gvec_oprnds;
+ gvec_oprnds.safe_grow_cleared (group_size);
for (j = 0; j < ncopies; j++)
{
stmt_vec_info new_stmt_info;
@@ -8193,20 +7785,18 @@ vectorizable_store (vec_info *vinfo,
if (slp)
{
/* Get vectorized arguments for SLP_NODE. */
- vect_get_vec_defs (vinfo, op, NULL_TREE, stmt_info, &vec_oprnds,
- NULL, slp_node);
-
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
+ op, &vec_oprnds);
vec_oprnd = vec_oprnds[0];
}
else
{
/* For interleaved stores we collect vectorized defs for all the
stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
- used as an input to vect_permute_store_chain(), and OPRNDS as
- an input to vect_get_vec_def_for_stmt_copy() for the next copy.
+ used as an input to vect_permute_store_chain().
- If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and
- OPRNDS are of size 1. */
+ If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
+ and OPRNDS are of size 1. */
stmt_vec_info next_stmt_info = first_stmt_info;
for (i = 0; i < group_size; i++)
{
@@ -8216,15 +7806,19 @@ vectorizable_store (vec_info *vinfo,
that there is no interleaving, DR_GROUP_SIZE is 1,
and only one iteration of the loop will be executed. */
op = vect_get_store_rhs (next_stmt_info);
- vec_oprnd = vect_get_vec_def_for_operand
- (vinfo, op, next_stmt_info);
- dr_chain.quick_push (vec_oprnd);
- oprnds.quick_push (vec_oprnd);
+ vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
+ ncopies, op, &gvec_oprnds[i]);
+ vec_oprnd = gvec_oprnds[i][0];
+ dr_chain.quick_push (gvec_oprnds[i][0]);
+ oprnds.quick_push (gvec_oprnds[i][0]);
next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
}
if (mask)
- vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info,
- mask_vectype);
+ {
+ vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+ mask, &vec_masks, mask_vectype);
+ vec_mask = vec_masks[0];
+ }
}
/* We should have catched mismatched types earlier. */
@@ -8245,8 +7839,11 @@ vectorizable_store (vec_info *vinfo,
dataref_offset = build_int_cst (ref_type, 0);
}
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
- &dataref_ptr, &vec_offset);
+ {
+ vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
+ &dataref_ptr, &vec_offsets, ncopies);
+ vec_offset = vec_offsets[0];
+ }
else
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
@@ -8258,25 +7855,22 @@ vectorizable_store (vec_info *vinfo,
{
/* For interleaved stores we created vectorized defs for all the
defs stored in OPRNDS in the previous iteration (previous copy).
- DR_CHAIN is then used as an input to vect_permute_store_chain(),
- and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
- next copy.
+ DR_CHAIN is then used as an input to vect_permute_store_chain().
If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and
OPRNDS are of size 1. */
for (i = 0; i < group_size; i++)
{
- op = oprnds[i];
- vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, op);
- dr_chain[i] = vec_oprnd;
- oprnds[i] = vec_oprnd;
+ vec_oprnd = gvec_oprnds[i][j];
+ dr_chain[i] = gvec_oprnds[i][j];
+ oprnds[i] = gvec_oprnds[i][j];
}
if (mask)
- vec_mask = vect_get_vec_def_for_stmt_copy (vinfo, vec_mask);
+ vec_mask = vec_masks[j];
if (dataref_offset)
dataref_offset
= int_const_binop (PLUS_EXPR, dataref_offset, bump);
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vec_offset = vect_get_vec_def_for_stmt_copy (vinfo, vec_offset);
+ vec_offset = vec_offsets[j];
else
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, bump);
@@ -8475,13 +8069,16 @@ vectorizable_store (vec_info *vinfo,
if (!slp)
{
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
+ for (i = 0; i < group_size; ++i)
+ {
+ vec<tree> oprndsi = gvec_oprnds[i];
+ oprndsi.release ();
+ }
oprnds.release ();
result_chain.release ();
vec_oprnds.release ();
@@ -8616,7 +8213,6 @@ vectorizable_load (vec_info *vinfo,
tree scalar_dest;
tree vec_dest = NULL;
tree data_ref = NULL;
- stmt_vec_info prev_stmt_info;
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
@@ -8937,7 +8533,6 @@ vectorizable_load (vec_info *vinfo,
}
/* These copies are all equivalent, but currently the representation
requires a separate STMT_VINFO_VEC_STMT for each one. */
- prev_stmt_info = NULL;
gimple_stmt_iterator gsi2 = *gsi;
gsi_next (&gsi2);
for (j = 0; j < ncopies; j++)
@@ -8958,11 +8553,12 @@ vectorizable_load (vec_info *vinfo,
}
if (slp)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- else if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ {
+ if (j == 0)
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
+ }
}
return true;
}
@@ -9052,7 +8648,6 @@ vectorizable_load (vec_info *vinfo,
stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
- prev_stmt_info = NULL;
running_off = offvar;
alias_off = build_int_cst (ref_type, 0);
int nloads = const_nunits;
@@ -9176,10 +8771,8 @@ vectorizable_load (vec_info *vinfo,
else
{
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
if (slp_perm)
@@ -9209,7 +8802,7 @@ vectorizable_load (vec_info *vinfo,
first_stmt_info_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0];
/* Check if the chain of loads is already vectorized. */
- if (STMT_VINFO_VEC_STMT (first_stmt_info)
+ if (STMT_VINFO_VEC_STMTS (first_stmt_info).exists ()
/* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
??? But we can only do so if there is exactly one
as we have no way to get at the rest. Leave the CSE
@@ -9220,7 +8813,7 @@ vectorizable_load (vec_info *vinfo,
is even wrong code. See PR56270. */
&& !slp)
{
- *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
return true;
}
first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
@@ -9321,10 +8914,7 @@ vectorizable_load (vec_info *vinfo,
VS1_3: vx3 = memref3 - -
S1: x = load - VS1_0
S2: z = x + 1 - -
-
- See in documentation in vect_get_vec_def_for_stmt_copy for how the
- information we recorded in RELATED_STMT field is used to vectorize
- stmt S2. */
+ */
/* In case of interleaving (non-unit grouped access):
@@ -9455,8 +9045,12 @@ vectorizable_load (vec_info *vinfo,
memory_access_type);
}
+ vec<tree> vec_offsets = vNULL;
+ auto_vec<tree> vec_masks;
+ if (mask)
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ mask, &vec_masks, mask_vectype, NULL_TREE);
tree vec_mask = NULL_TREE;
- prev_stmt_info = NULL;
poly_uint64 group_elt = 0;
for (j = 0; j < ncopies; j++)
{
@@ -9507,8 +9101,11 @@ vectorizable_load (vec_info *vinfo,
}
}
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
- &dataref_ptr, &vec_offset);
+ {
+ vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
+ &dataref_ptr, &vec_offsets, ncopies);
+ vec_offset = vec_offsets[0];
+ }
else
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
@@ -9517,17 +9114,7 @@ vectorizable_load (vec_info *vinfo,
simd_lane_access_p,
byte_offset, bump);
if (mask)
- {
- if (slp_node)
- {
- auto_vec<vec<tree> > vec_defs (1);
- vect_get_slp_defs (vinfo, slp_node, &vec_defs);
- vec_mask = vec_defs[0][0];
- }
- else
- vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info,
- mask_vectype);
- }
+ vec_mask = vec_masks[0];
}
else
{
@@ -9535,12 +9122,12 @@ vectorizable_load (vec_info *vinfo,
dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
bump);
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vec_offset = vect_get_vec_def_for_stmt_copy (vinfo, vec_offset);
+ vec_offset = vec_offsets[j];
else
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, bump);
if (mask)
- vec_mask = vect_get_vec_def_for_stmt_copy (vinfo, vec_mask);
+ vec_mask = vec_masks[j];
}
if (grouped_load || slp_perm)
@@ -9960,15 +9547,13 @@ vectorizable_load (vec_info *vinfo,
if (memory_access_type != VMAT_LOAD_STORE_LANES)
vect_transform_grouped_load (vinfo, stmt_info, dr_chain,
group_size, gsi);
- *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
}
else
{
if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- prev_stmt_info = new_stmt_info;
+ *vec_stmt = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
}
dr_chain.release ();
@@ -10104,8 +9689,7 @@ vectorizable_condition (vec_info *vinfo,
int ncopies;
int vec_num;
enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
- stmt_vec_info prev_stmt_info = NULL;
- int i, j;
+ int i;
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
@@ -10354,258 +9938,202 @@ vectorizable_condition (vec_info *vinfo,
if (reduction_type != EXTRACT_LAST_REDUCTION)
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- /* Handle cond expr. */
- for (j = 0; j < ncopies; j++)
- {
- bool swap_cond_operands = false;
+ bool swap_cond_operands = false;
- /* See whether another part of the vectorized code applies a loop
- mask to the condition, or to its inverse. */
+ /* See whether another part of the vectorized code applies a loop
+ mask to the condition, or to its inverse. */
- vec_loop_masks *masks = NULL;
- if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ vec_loop_masks *masks = NULL;
+ if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ {
+ if (reduction_type == EXTRACT_LAST_REDUCTION)
+ masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ else
{
- if (reduction_type == EXTRACT_LAST_REDUCTION)
+ scalar_cond_masked_key cond (cond_expr, ncopies);
+ if (loop_vinfo->scalar_cond_masked_set.contains (cond))
masks = &LOOP_VINFO_MASKS (loop_vinfo);
else
{
- scalar_cond_masked_key cond (cond_expr, ncopies);
+ bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
+ cond.code = invert_tree_comparison (cond.code, honor_nans);
if (loop_vinfo->scalar_cond_masked_set.contains (cond))
- masks = &LOOP_VINFO_MASKS (loop_vinfo);
- else
- {
- bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
- cond.code = invert_tree_comparison (cond.code, honor_nans);
- if (loop_vinfo->scalar_cond_masked_set.contains (cond))
- {
- masks = &LOOP_VINFO_MASKS (loop_vinfo);
- cond_code = cond.code;
- swap_cond_operands = true;
- }
- }
- }
- }
-
- stmt_vec_info new_stmt_info = NULL;
- if (j == 0)
- {
- if (slp_node)
- {
- auto_vec<vec<tree>, 4> vec_defs;
- vect_get_slp_defs (vinfo, slp_node, &vec_defs);
- vec_oprnds3 = vec_defs.pop ();
- vec_oprnds2 = vec_defs.pop ();
- if (!masked)
- vec_oprnds1 = vec_defs.pop ();
- vec_oprnds0 = vec_defs.pop ();
- }
- else
- {
- if (masked)
- {
- vec_cond_lhs
- = vect_get_vec_def_for_operand (vinfo, cond_expr, stmt_info,
- comp_vectype);
- }
- else
{
- vec_cond_lhs
- = vect_get_vec_def_for_operand (vinfo, cond_expr0,
- stmt_info, comp_vectype);
- vec_cond_rhs
- = vect_get_vec_def_for_operand (vinfo, cond_expr1,
- stmt_info, comp_vectype);
+ masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ cond_code = cond.code;
+ swap_cond_operands = true;
}
- vec_then_clause = vect_get_vec_def_for_operand (vinfo,
- then_clause,
- stmt_info);
- if (reduction_type != EXTRACT_LAST_REDUCTION)
- vec_else_clause = vect_get_vec_def_for_operand (vinfo,
- else_clause,
- stmt_info);
}
}
- else
- {
- vec_cond_lhs
- = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnds0.pop ());
- if (!masked)
- vec_cond_rhs
- = vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnds1.pop ());
-
- vec_then_clause = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnds2.pop ());
- vec_else_clause = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnds3.pop ());
- }
+ }
- if (!slp_node)
- {
- vec_oprnds0.quick_push (vec_cond_lhs);
- if (!masked)
- vec_oprnds1.quick_push (vec_cond_rhs);
- vec_oprnds2.quick_push (vec_then_clause);
- vec_oprnds3.quick_push (vec_else_clause);
- }
+ /* Handle cond expr. */
+ if (masked)
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ cond_expr, &vec_oprnds0, comp_vectype,
+ then_clause, &vec_oprnds2, vectype,
+ reduction_type != EXTRACT_LAST_REDUCTION
+ ? else_clause : NULL, &vec_oprnds3, vectype);
+ else
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ cond_expr0, &vec_oprnds0, comp_vectype,
+ cond_expr1, &vec_oprnds1, comp_vectype,
+ then_clause, &vec_oprnds2, vectype,
+ reduction_type != EXTRACT_LAST_REDUCTION
+ ? else_clause : NULL, &vec_oprnds3, vectype);
- /* Arguments are ready. Create the new vector stmt. */
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
- {
- vec_then_clause = vec_oprnds2[i];
- vec_else_clause = vec_oprnds3[i];
+ /* Arguments are ready. Create the new vector stmt. */
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
+ {
+ vec_then_clause = vec_oprnds2[i];
+ vec_else_clause = vec_oprnds3[i];
- if (swap_cond_operands)
- std::swap (vec_then_clause, vec_else_clause);
+ if (swap_cond_operands)
+ std::swap (vec_then_clause, vec_else_clause);
- if (masked)
- vec_compare = vec_cond_lhs;
+ if (masked)
+ vec_compare = vec_cond_lhs;
+ else
+ {
+ vec_cond_rhs = vec_oprnds1[i];
+ if (bitop1 == NOP_EXPR)
+ vec_compare = build2 (cond_code, vec_cmp_type,
+ vec_cond_lhs, vec_cond_rhs);
else
{
- vec_cond_rhs = vec_oprnds1[i];
- if (bitop1 == NOP_EXPR)
- vec_compare = build2 (cond_code, vec_cmp_type,
- vec_cond_lhs, vec_cond_rhs);
+ new_temp = make_ssa_name (vec_cmp_type);
+ gassign *new_stmt;
+ if (bitop1 == BIT_NOT_EXPR)
+ new_stmt = gimple_build_assign (new_temp, bitop1,
+ vec_cond_rhs);
else
+ new_stmt
+ = gimple_build_assign (new_temp, bitop1, vec_cond_lhs,
+ vec_cond_rhs);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ if (bitop2 == NOP_EXPR)
+ vec_compare = new_temp;
+ else if (bitop2 == BIT_NOT_EXPR)
{
- new_temp = make_ssa_name (vec_cmp_type);
- gassign *new_stmt;
- if (bitop1 == BIT_NOT_EXPR)
- new_stmt = gimple_build_assign (new_temp, bitop1,
- vec_cond_rhs);
- else
- new_stmt
- = gimple_build_assign (new_temp, bitop1, vec_cond_lhs,
- vec_cond_rhs);
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (bitop2 == NOP_EXPR)
- vec_compare = new_temp;
- else if (bitop2 == BIT_NOT_EXPR)
- {
- /* Instead of doing ~x ? y : z do x ? z : y. */
- vec_compare = new_temp;
- std::swap (vec_then_clause, vec_else_clause);
- }
- else
- {
- vec_compare = make_ssa_name (vec_cmp_type);
- new_stmt
- = gimple_build_assign (vec_compare, bitop2,
- vec_cond_lhs, new_temp);
- vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
- }
+ /* Instead of doing ~x ? y : z do x ? z : y. */
+ vec_compare = new_temp;
+ std::swap (vec_then_clause, vec_else_clause);
+ }
+ else
+ {
+ vec_compare = make_ssa_name (vec_cmp_type);
+ new_stmt
+ = gimple_build_assign (vec_compare, bitop2,
+ vec_cond_lhs, new_temp);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
+ }
- /* If we decided to apply a loop mask to the result of the vector
- comparison, AND the comparison with the mask now. Later passes
- should then be able to reuse the AND results between mulitple
- vector statements.
+ /* If we decided to apply a loop mask to the result of the vector
+ comparison, AND the comparison with the mask now. Later passes
+ should then be able to reuse the AND results between mulitple
+ vector statements.
- For example:
- for (int i = 0; i < 100; ++i)
- x[i] = y[i] ? z[i] : 10;
+ For example:
+ for (int i = 0; i < 100; ++i)
+ x[i] = y[i] ? z[i] : 10;
- results in following optimized GIMPLE:
+ results in following optimized GIMPLE:
- mask__35.8_43 = vect__4.7_41 != { 0, ... };
- vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
- _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
- vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
- vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
- vect_iftmp.11_47, { 10, ... }>;
+ mask__35.8_43 = vect__4.7_41 != { 0, ... };
+ vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
+ _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
+ vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
+ vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
+ vect_iftmp.11_47, { 10, ... }>;
- instead of using a masked and unmasked forms of
- vec != { 0, ... } (masked in the MASK_LOAD,
- unmasked in the VEC_COND_EXPR). */
+ instead of using a masked and unmasked forms of
+ vec != { 0, ... } (masked in the MASK_LOAD,
+ unmasked in the VEC_COND_EXPR). */
- /* Force vec_compare to be an SSA_NAME rather than a comparison,
- in cases where that's necessary. */
+ /* Force vec_compare to be an SSA_NAME rather than a comparison,
+ in cases where that's necessary. */
- if (masks || reduction_type == EXTRACT_LAST_REDUCTION)
+ if (masks || reduction_type == EXTRACT_LAST_REDUCTION)
+ {
+ if (!is_gimple_val (vec_compare))
{
- if (!is_gimple_val (vec_compare))
- {
- tree vec_compare_name = make_ssa_name (vec_cmp_type);
- gassign *new_stmt = gimple_build_assign (vec_compare_name,
- vec_compare);
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- vec_compare = vec_compare_name;
- }
-
- if (must_invert_cmp_result)
- {
- tree vec_compare_name = make_ssa_name (vec_cmp_type);
- gassign *new_stmt = gimple_build_assign (vec_compare_name,
- BIT_NOT_EXPR,
- vec_compare);
- vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- vec_compare = vec_compare_name;
- }
+ tree vec_compare_name = make_ssa_name (vec_cmp_type);
+ gassign *new_stmt = gimple_build_assign (vec_compare_name,
+ vec_compare);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vec_compare = vec_compare_name;
+ }
- if (masks)
- {
- unsigned vec_num = vec_oprnds0.length ();
- tree loop_mask
- = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
- vectype, vec_num * j + i);
- tree tmp2 = make_ssa_name (vec_cmp_type);
- gassign *g
- = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
- loop_mask);
- vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
- vec_compare = tmp2;
- }
+ if (must_invert_cmp_result)
+ {
+ tree vec_compare_name = make_ssa_name (vec_cmp_type);
+ gassign *new_stmt = gimple_build_assign (vec_compare_name,
+ BIT_NOT_EXPR,
+ vec_compare);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vec_compare = vec_compare_name;
}
- if (reduction_type == EXTRACT_LAST_REDUCTION)
+ if (masks)
{
- gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
- tree lhs = gimple_get_lhs (old_stmt);
- gcall *new_stmt = gimple_build_call_internal
- (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
- vec_then_clause);
- gimple_call_set_lhs (new_stmt, lhs);
- SSA_NAME_DEF_STMT (lhs) = new_stmt;
- if (old_stmt == gsi_stmt (*gsi))
- new_stmt_info = vect_finish_replace_stmt (vinfo,
- stmt_info, new_stmt);
- else
- {
- /* In this case we're moving the definition to later in the
- block. That doesn't matter because the only uses of the
- lhs are in phi statements. */
- gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
- gsi_remove (&old_gsi, true);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
- }
+ unsigned vec_num = vec_oprnds0.length ();
+ tree loop_mask
+ = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+ vectype, i);
+ tree tmp2 = make_ssa_name (vec_cmp_type);
+ gassign *g
+ = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
+ loop_mask);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
+ vec_compare = tmp2;
}
+ }
+
+ stmt_vec_info new_stmt_info;
+ if (reduction_type == EXTRACT_LAST_REDUCTION)
+ {
+ gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
+ tree lhs = gimple_get_lhs (old_stmt);
+ gcall *new_stmt = gimple_build_call_internal
+ (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
+ vec_then_clause);
+ gimple_call_set_lhs (new_stmt, lhs);
+ SSA_NAME_DEF_STMT (lhs) = new_stmt;
+ if (old_stmt == gsi_stmt (*gsi))
+ new_stmt_info = vect_finish_replace_stmt (vinfo,
+ stmt_info, new_stmt);
else
{
- new_temp = make_ssa_name (vec_dest);
- gassign *new_stmt
- = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
- vec_then_clause, vec_else_clause);
+ /* In this case we're moving the definition to later in the
+ block. That doesn't matter because the only uses of the
+ lhs are in phi statements. */
+ gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
+ gsi_remove (&old_gsi, true);
new_stmt_info
= vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
- }
-
- if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
- else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ }
+ else
+ {
+ new_temp = make_ssa_name (vec_dest);
+ gassign *new_stmt
+ = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
+ vec_then_clause, vec_else_clause);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ }
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
vec_oprnds0.release ();
vec_oprnds1.release ();
vec_oprnds2.release ();
@@ -10639,8 +10167,7 @@ vectorizable_comparison (vec_info *vinfo,
poly_uint64 nunits;
int ncopies;
enum tree_code code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
- stmt_vec_info prev_stmt_info = NULL;
- int i, j;
+ int i;
bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
@@ -10810,96 +10337,57 @@ vectorizable_comparison (vec_info *vinfo,
lhs = gimple_assign_lhs (stmt);
mask = vect_create_destination_var (lhs, mask_type);
- /* Handle cmp expr. */
- for (j = 0; j < ncopies; j++)
+ vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+ rhs1, &vec_oprnds0, vectype,
+ rhs2, &vec_oprnds1, vectype);
+ if (swap_p)
+ std::swap (vec_oprnds0, vec_oprnds1);
+
+ /* Arguments are ready. Create the new vector stmt. */
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
{
- stmt_vec_info new_stmt_info = NULL;
- if (j == 0)
- {
- if (slp_node)
- {
- auto_vec<vec<tree>, 2> vec_defs;
- vect_get_slp_defs (vinfo, slp_node, &vec_defs);
- vec_oprnds1 = vec_defs.pop ();
- vec_oprnds0 = vec_defs.pop ();
- if (swap_p)
- std::swap (vec_oprnds0, vec_oprnds1);
- }
- else
- {
- vec_rhs1 = vect_get_vec_def_for_operand (vinfo, rhs1, stmt_info,
- vectype);
- vec_rhs2 = vect_get_vec_def_for_operand (vinfo, rhs2, stmt_info,
- vectype);
- }
- }
- else
- {
- vec_rhs1 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnds0.pop ());
- vec_rhs2 = vect_get_vec_def_for_stmt_copy (vinfo,
- vec_oprnds1.pop ());
- }
+ stmt_vec_info new_stmt_info;
+ vec_rhs2 = vec_oprnds1[i];
- if (!slp_node)
+ new_temp = make_ssa_name (mask);
+ if (bitop1 == NOP_EXPR)
{
- if (swap_p && j == 0)
- std::swap (vec_rhs1, vec_rhs2);
- vec_oprnds0.quick_push (vec_rhs1);
- vec_oprnds1.quick_push (vec_rhs2);
+ gassign *new_stmt = gimple_build_assign (new_temp, code,
+ vec_rhs1, vec_rhs2);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
-
- /* Arguments are ready. Create the new vector stmt. */
- FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
+ else
{
- vec_rhs2 = vec_oprnds1[i];
-
- new_temp = make_ssa_name (mask);
- if (bitop1 == NOP_EXPR)
- {
- gassign *new_stmt = gimple_build_assign (new_temp, code,
- vec_rhs1, vec_rhs2);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- }
+ gassign *new_stmt;
+ if (bitop1 == BIT_NOT_EXPR)
+ new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
else
+ new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
+ vec_rhs2);
+ new_stmt_info
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ if (bitop2 != NOP_EXPR)
{
- gassign *new_stmt;
- if (bitop1 == BIT_NOT_EXPR)
- new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
+ tree res = make_ssa_name (mask);
+ if (bitop2 == BIT_NOT_EXPR)
+ new_stmt = gimple_build_assign (res, bitop2, new_temp);
else
- new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
- vec_rhs2);
+ new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
+ new_temp);
new_stmt_info
= vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
- if (bitop2 != NOP_EXPR)
- {
- tree res = make_ssa_name (mask);
- if (bitop2 == BIT_NOT_EXPR)
- new_stmt = gimple_build_assign (res, bitop2, new_temp);
- else
- new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
- new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (vinfo, stmt_info,
- new_stmt, gsi);
- }
}
- if (slp_node)
- SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
-
if (slp_node)
- continue;
-
- if (j == 0)
- STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
else
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-
- prev_stmt_info = new_stmt_info;
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info);
}
+ if (!slp_node)
+ *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+
vec_oprnds0.release ();
vec_oprnds1.release ();
@@ -11184,7 +10672,6 @@ vect_transform_stmt (vec_info *vinfo,
bool done;
gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
- stmt_vec_info old_vec_stmt_info = STMT_VINFO_VEC_STMT (stmt_info);
switch (STMT_VINFO_TYPE (stmt_info))
{
@@ -11296,14 +10783,8 @@ vect_transform_stmt (vec_info *vinfo,
done = true;
}
- /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
- This would break hybrid SLP vectorization. */
- if (slp_node)
- gcc_assert (!vec_stmt
- && STMT_VINFO_VEC_STMT (stmt_info) == old_vec_stmt_info);
-
- if (vec_stmt)
- STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
+ if (!slp_node && vec_stmt)
+ gcc_assert (STMT_VINFO_VEC_STMTS (stmt_info).exists ());
if (STMT_VINFO_TYPE (stmt_info) == store_vec_info_type)
return is_store;
@@ -11329,19 +10810,15 @@ vect_transform_stmt (vec_info *vinfo,
&& (PHI_ARG_DEF_FROM_EDGE (phi, e)
== gimple_get_lhs (orig_stmt_info->stmt)))
{
- stmt_vec_info phi_info
- = STMT_VINFO_VEC_STMT (STMT_VINFO_REDUC_DEF (orig_stmt_info));
- stmt_vec_info vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
- do
- {
- add_phi_arg (as_a <gphi *> (phi_info->stmt),
- gimple_get_lhs (vec_stmt->stmt), e,
- gimple_phi_arg_location (phi, e->dest_idx));
- phi_info = STMT_VINFO_RELATED_STMT (phi_info);
- vec_stmt = STMT_VINFO_RELATED_STMT (vec_stmt);
- }
- while (phi_info);
- gcc_assert (!vec_stmt);
+ vec<stmt_vec_info> &phi_info
+ = STMT_VINFO_VEC_STMTS (STMT_VINFO_REDUC_DEF (orig_stmt_info));
+ vec<stmt_vec_info> &vec_stmt
+ = STMT_VINFO_VEC_STMTS (stmt_info);
+ gcc_assert (phi_info.length () == vec_stmt.length ());
+ for (unsigned i = 0; i < phi_info.length (); ++i)
+ add_phi_arg (as_a <gphi *> (phi_info[i]->stmt),
+ gimple_get_lhs (vec_stmt[i]->stmt), e,
+ gimple_phi_arg_location (phi, e->dest_idx));
}
else if (slp_node
&& slp_node != slp_node_instance->reduc_phis)
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 584f13095ab..e9fe3840e1e 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -641,6 +641,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
STMT_VINFO_REDUC_FN (res) = IFN_LAST;
STMT_VINFO_REDUC_IDX (res) = -1;
STMT_VINFO_SLP_VECT_ONLY (res) = false;
+ STMT_VINFO_VEC_STMTS (res) = vNULL;
if (gimple_code (stmt) == GIMPLE_PHI
&& is_loop_header_bb_p (gimple_bb (stmt)))
@@ -705,6 +706,7 @@ vec_info::free_stmt_vec_info (stmt_vec_info stmt_info)
STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release ();
STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
+ STMT_VINFO_VEC_STMTS (stmt_info).release ();
free (stmt_info);
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 9bb82a546f6..df056381531 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -964,9 +964,8 @@ public:
/* The vector type to be used for the LHS of this statement. */
tree vectype;
- /* The vectorized version of the stmt. */
- stmt_vec_info vectorized_stmt;
-
+ /* The vectorized stmts. */
+ vec<stmt_vec_info> vec_stmts;
/* The following is relevant only for stmts that contain a non-scalar
data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
@@ -1127,6 +1126,8 @@ public:
/* True if this is only suitable for SLP vectorization. */
bool slp_vect_only_p;
+
+ bool vector_stmt;
};
/* Information about a gather/scatter call. */
@@ -1168,7 +1169,7 @@ struct gather_scatter_info {
#define STMT_VINFO_RELEVANT(S) (S)->relevant
#define STMT_VINFO_LIVE_P(S) (S)->live
#define STMT_VINFO_VECTYPE(S) (S)->vectype
-#define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
+#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
@@ -1743,16 +1744,20 @@ extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
gimple_stmt_iterator *);
extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
extern tree vect_get_store_rhs (stmt_vec_info);
-extern tree vect_get_vec_def_for_operand_1 (stmt_vec_info, enum vect_def_type);
-extern tree vect_get_vec_def_for_operand (vec_info *, tree,
- stmt_vec_info, tree = NULL);
-extern void vect_get_vec_defs (vec_info *, tree, tree, stmt_vec_info,
- vec<tree> *, vec<tree> *, slp_tree);
-extern void vect_get_vec_defs_for_stmt_copy (vec_info *,
- vec<tree> *, vec<tree> *);
+void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned,
+ tree op, vec<tree> *, tree = NULL);
+void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
+ tree, vec<tree> *,
+ tree = NULL, vec<tree> * = NULL,
+ tree = NULL, vec<tree> * = NULL,
+ tree = NULL, vec<tree> * = NULL);
+void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
+ tree, vec<tree> *, tree,
+ tree = NULL, vec<tree> * = NULL, tree = NULL,
+ tree = NULL, vec<tree> * = NULL, tree = NULL,
+ tree = NULL, vec<tree> * = NULL, tree = NULL);
extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
gimple_stmt_iterator *);
-extern tree vect_get_vec_def_for_stmt_copy (vec_info *, tree);
extern tree vect_get_slp_vect_def (slp_tree, unsigned);
extern bool vect_transform_stmt (vec_info *, stmt_vec_info,
gimple_stmt_iterator *,
@@ -1895,6 +1900,7 @@ extern opt_result vect_analyze_slp (vec_info *, unsigned);
extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_optimize_slp (vec_info *);
+extern void vect_get_slp_defs (slp_tree, vec<tree> *);
extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
unsigned n = -1U);
extern bool vect_slp_bb (basic_block);
--
2.26.2
More information about the Gcc-patches
mailing list