This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
[PATCH] More reduction vectorization refactoring
- From: Richard Biener <rguenther at suse dot de>
- To: gcc-patches at gcc dot gnu dot org
- Date: Tue, 24 Sep 2019 13:52:21 +0200 (CEST)
- Subject: [PATCH] More reduction vectorization refactoring
Baby-steps towards sanity. My focus is currently vectorizable_reduction
vs. vect_create_epilog_for_reduction - the following aims at simplifying
all the condition reduction special-casing. The real next intermediate
goal is to move all code-generation that is not "epilogue" from
vect_create_epilog_for_reduction to vectorizable_reduction, first
completing the last big refactoring and move all PHI creation to
the point when vectorizable_reduction is called on the scalar
reduction PHI.
Anyway...
Bootstrapped on x86_64-unknown-linux-gnu, testing in progress.
Richard.
2019-09-24 Richard Biener <rguenther@suse.de>
* tree-vectorizer.h (_stmt_vec_info::const_cond_reduc_code):
Rename to...
(_stmt_vec_info::cond_reduc_code): ... this.
(_stmt_vec_info::induc_cond_initial_val): Add.
(STMT_VINFO_VEC_CONST_COND_REDUC_CODE): Rename to...
(STMT_VINFO_VEC_COND_REDUC_CODE): ... this.
(STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL): Add.
* tree-vectorizer.c (vec_info::new_stmt_vec_info): Adjust.
* tree-vect-loop.c (get_initial_def_for_reduction): Pass in
the reduction code.
(vect_create_epilog_for_reduction): Drop special
induction condition reduction params, pass in reduction code
and simplify.
(vectorizable_reduction): Perform condition reduction kind
selection only at analysis time. Adjust passing on state.
Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c (revision 276092)
+++ gcc/tree-vect-loop.c (working copy)
@@ -3981,14 +3981,14 @@ vect_model_induction_cost (stmt_vec_info
A cost model should help decide between these two schemes. */
static tree
-get_initial_def_for_reduction (stmt_vec_info stmt_vinfo, tree init_val,
+get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
+ enum tree_code code, tree init_val,
tree *adjustment_def)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree scalar_type = TREE_TYPE (init_val);
tree vectype = get_vectype_for_scalar_type (scalar_type);
- enum tree_code code = gimple_assign_rhs_code (stmt_vinfo->stmt);
tree def_for_init;
tree init_def;
REAL_VALUE_TYPE real_init_val = dconst0;
@@ -4273,14 +4273,15 @@ static void
vect_create_epilog_for_reduction (vec<tree> vect_defs,
stmt_vec_info stmt_info,
gimple *reduc_def_stmt,
+ enum tree_code code,
int ncopies, internal_fn reduc_fn,
vec<stmt_vec_info> reduction_phis,
bool double_reduc,
slp_tree slp_node,
slp_instance slp_node_instance,
- tree induc_val, enum tree_code induc_code,
tree neutral_op)
{
+ tree induc_val = NULL_TREE;
stmt_vec_info prev_phi_info;
tree vectype;
machine_mode mode;
@@ -4370,17 +4371,22 @@ vect_create_epilog_for_reduction (vec<tr
/* Optimize: if initial_def is for REDUC_MAX smaller than the base
and we can't use zero for induc_val, use initial_def. Similarly
for REDUC_MIN and initial_def larger than the base. */
- if (TREE_CODE (initial_def) == INTEGER_CST
- && (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
- == INTEGER_INDUC_COND_REDUCTION)
- && !integer_zerop (induc_val)
- && ((induc_code == MAX_EXPR
- && tree_int_cst_lt (initial_def, induc_val))
- || (induc_code == MIN_EXPR
- && tree_int_cst_lt (induc_val, initial_def))))
- induc_val = initial_def;
-
- if (double_reduc)
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
+ == INTEGER_INDUC_COND_REDUCTION)
+ {
+ induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info);
+ if (TREE_CODE (initial_def) == INTEGER_CST
+ && (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
+ == INTEGER_INDUC_COND_REDUCTION)
+ && !integer_zerop (induc_val)
+ && ((code == MAX_EXPR
+ && tree_int_cst_lt (initial_def, induc_val))
+ || (code == MIN_EXPR
+ && tree_int_cst_lt (induc_val, initial_def))))
+ induc_val = initial_def;
+ vec_initial_def = build_vector_from_val (vectype, induc_val);
+ }
+ else if (double_reduc)
/* In case of double reduction we only create a vector variable
to be put in the reduction phi node. The actual statement
creation is done later in this function. */
@@ -4394,7 +4400,7 @@ vect_create_epilog_for_reduction (vec<tr
}
else
vec_initial_def
- = get_initial_def_for_reduction (stmt_info, initial_def,
+ = get_initial_def_for_reduction (stmt_info, code, initial_def,
&adjustment_def);
vec_initial_defs.create (1);
vec_initial_defs.quick_push (vec_initial_def);
@@ -4418,24 +4424,8 @@ vect_create_epilog_for_reduction (vec<tr
/* Set the loop-entry arg of the reduction-phi. */
gphi *phi = as_a <gphi *> (phi_info->stmt);
- if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
- == INTEGER_INDUC_COND_REDUCTION)
- {
- /* Initialise the reduction phi to zero. This prevents initial
- values of non-zero interferring with the reduction op. */
- gcc_assert (ncopies == 1);
- gcc_assert (i == 0);
-
- tree vec_init_def_type = TREE_TYPE (vec_init_def);
- tree induc_val_vec
- = build_vector_from_val (vec_init_def_type, induc_val);
-
- add_phi_arg (phi, induc_val_vec, loop_preheader_edge (loop),
- UNKNOWN_LOCATION);
- }
- else
- add_phi_arg (phi, vec_init_def, loop_preheader_edge (loop),
- UNKNOWN_LOCATION);
+ add_phi_arg (phi, vec_init_def, loop_preheader_edge (loop),
+ UNKNOWN_LOCATION);
/* Set the loop-latch arg for the reduction-phi. */
if (j > 0)
@@ -4652,12 +4642,6 @@ vect_create_epilog_for_reduction (vec<tr
;
else
{
- enum tree_code code = gimple_assign_rhs_code (orig_stmt_info->stmt);
- /* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore,
- partial results are added and not subtracted. */
- if (code == MINUS_EXPR)
- code = PLUS_EXPR;
-
/* SLP reduction without reduction chain, e.g.,
# a1 = phi <a2, a0>
# b1 = phi <b2, b0>
@@ -5049,20 +5033,6 @@ vect_create_epilog_for_reduction (vec<tr
bool reduce_with_shift;
tree vec_temp;
- /* COND reductions all do the final reduction with MAX_EXPR
- or MIN_EXPR. */
- if (code == COND_EXPR)
- {
- if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
- == INTEGER_INDUC_COND_REDUCTION)
- code = induc_code;
- else if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
- == CONST_COND_REDUCTION)
- code = STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info);
- else
- code = MAX_EXPR;
- }
-
/* See if the target wants to do the final (shift) reduction
in a vector mode of smaller size and first reduce upper/lower
halves against each other. */
@@ -5543,7 +5513,7 @@ vect_create_epilog_for_reduction (vec<tr
preheader_arg = PHI_ARG_DEF_FROM_EDGE (use_stmt,
loop_preheader_edge (outer_loop));
vect_phi_init = get_initial_def_for_reduction
- (stmt_info, preheader_arg, NULL);
+ (stmt_info, code, preheader_arg, NULL);
/* Update phi node arguments with vs0 and vs2. */
add_phi_arg (vect_phi, vect_phi_init,
@@ -6021,7 +5991,7 @@ vectorizable_reduction (stmt_vec_info st
tree vectype_in = NULL_TREE;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- enum tree_code code, orig_code;
+ enum tree_code code;
internal_fn reduc_fn;
machine_mode vec_mode;
int op_type;
@@ -6029,7 +5999,6 @@ vectorizable_reduction (stmt_vec_info st
tree new_temp = NULL_TREE;
enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type;
stmt_vec_info cond_stmt_vinfo = NULL;
- enum tree_code cond_reduc_op_code = ERROR_MARK;
tree scalar_type;
bool is_simple_use;
int i;
@@ -6362,9 +6331,11 @@ vectorizable_reduction (stmt_vec_info st
= STMT_VINFO_REDUC_TYPE (reduc_def_info);
stmt_vec_info tmp = STMT_VINFO_REDUC_DEF (reduc_def_info);
- STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = v_reduc_type;
+ if (!vec_stmt)
+ STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = v_reduc_type;
/* If we have a condition reduction, see if we can simplify it further. */
- if (v_reduc_type == COND_REDUCTION)
+ if (v_reduc_type == COND_REDUCTION
+ && !vec_stmt)
{
/* TODO: We can't yet handle reduction chains, since we need to treat
each COND_EXPR in the chain specially, not just the last one.
@@ -6386,20 +6357,8 @@ vectorizable_reduction (stmt_vec_info st
return false;
}
- /* Loop peeling modifies initial value of reduction PHI, which
- makes the reduction stmt to be transformed different to the
- original stmt analyzed. We need to record reduction code for
- CONST_COND_REDUCTION type reduction at analyzing stage, thus
- it can be used directly at transform stage. */
- if (STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info) == MAX_EXPR
- || STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info) == MIN_EXPR)
- {
- /* Also set the reduction type to CONST_COND_REDUCTION. */
- gcc_assert (cond_reduc_dt == vect_constant_def);
- STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = CONST_COND_REDUCTION;
- }
- else if (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST,
- vectype_in, OPTIMIZE_FOR_SPEED))
+ if (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST,
+ vectype_in, OPTIMIZE_FOR_SPEED))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6416,6 +6375,7 @@ vectorizable_reduction (stmt_vec_info st
gcc_assert (TREE_CODE (base) == INTEGER_CST
&& TREE_CODE (step) == INTEGER_CST);
cond_reduc_val = NULL_TREE;
+ enum tree_code cond_reduc_op_code = ERROR_MARK;
tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
;
@@ -6444,10 +6404,10 @@ vectorizable_reduction (stmt_vec_info st
}
if (cond_reduc_val)
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "condition expression based on "
- "integer induction.\n");
+ STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
+ = cond_reduc_op_code;
+ STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info)
+ = cond_reduc_val;
STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
= INTEGER_INDUC_COND_REDUCTION;
}
@@ -6474,7 +6434,7 @@ vectorizable_reduction (stmt_vec_info st
"condition expression based on "
"compile time constant.\n");
/* Record reduction code at analysis stage. */
- STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info)
+ STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
= integer_onep (e) ? MAX_EXPR : MIN_EXPR;
STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
= CONST_COND_REDUCTION;
@@ -6482,6 +6442,11 @@ vectorizable_reduction (stmt_vec_info st
}
}
}
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == INTEGER_INDUC_COND_REDUCTION
+ && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "condition expression based on "
+ "integer induction.\n");
if (orig_stmt_info)
gcc_assert (tmp == orig_stmt_info
@@ -6637,6 +6602,7 @@ vectorizable_reduction (stmt_vec_info st
(and also the same tree-code) when generating the epilog code and
when generating the code inside the loop. */
+ enum tree_code orig_code;
if (orig_stmt_info
&& (reduction_type == TREE_CODE_REDUCTION
|| reduction_type == FOLD_LEFT_REDUCTION))
@@ -6658,13 +6624,12 @@ vectorizable_reduction (stmt_vec_info st
/* For simple condition reductions, replace with the actual expression
we want to base our reduction around. */
- if (reduction_type == CONST_COND_REDUCTION)
+ if (reduction_type == CONST_COND_REDUCTION
+ || reduction_type == INTEGER_INDUC_COND_REDUCTION)
{
- orig_code = STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info);
+ orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info);
gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
}
- else if (reduction_type == INTEGER_INDUC_COND_REDUCTION)
- orig_code = cond_reduc_op_code;
}
reduc_fn = IFN_LAST;
@@ -7171,9 +7136,8 @@ vectorizable_reduction (stmt_vec_info st
vect_defs[0] = gimple_get_lhs ((*vec_stmt)->stmt);
vect_create_epilog_for_reduction (vect_defs, stmt_info, reduc_def_phi,
- epilog_copies, reduc_fn, phis,
+ orig_code, epilog_copies, reduc_fn, phis,
double_reduc, slp_node, slp_node_instance,
- cond_reduc_val, cond_reduc_op_code,
neutral_op);
return true;
Index: gcc/tree-vectorizer.c
===================================================================
--- gcc/tree-vectorizer.c (revision 276092)
+++ gcc/tree-vectorizer.c (working copy)
@@ -638,7 +638,7 @@ vec_info::new_stmt_vec_info (gimple *stm
STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
STMT_VINFO_VECTORIZABLE (res) = true;
STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
- STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
+ STMT_VINFO_VEC_COND_REDUC_CODE (res) = ERROR_MARK;
STMT_VINFO_REDUC_IDX (res) = -1;
STMT_VINFO_SLP_VECT_ONLY (res) = false;
Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h (revision 276092)
+++ gcc/tree-vectorizer.h (working copy)
@@ -934,8 +934,12 @@ public:
/* For reduction loops, this is the type of reduction. */
enum vect_reduction_type v_reduc_type;
- /* For CONST_COND_REDUCTION, record the reduc code. */
- enum tree_code const_cond_reduc_code;
+ /* For CONST_COND_REDUCTION and INTEGER_INDUC_COND_REDUCTION, the
+ reduction code. */
+ enum tree_code cond_reduc_code;
+
+ /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
+ tree induc_cond_initial_val;
/* On a reduction PHI the reduction type as detected by
vect_force_simple_reduction. */
@@ -1033,7 +1037,8 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_
#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
#define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type
-#define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code
+#define STMT_VINFO_VEC_COND_REDUC_CODE(S) (S)->cond_reduc_code
+#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop