[8/n] PR85694: Make patterns check for target support
Richard Sandiford
richard.sandiford@arm.com
Mon Jun 18 15:01:00 GMT 2018
This patch makes pattern recognisers do their own checking for vector
types and target support. Previously some recognisers did this
themselves and some left it to vect_pattern_recog_1.
Doing this means we can get rid of the type_in argument, which was
ignored if the recogniser did its own checking. It also means
we create fewer junk statements.
Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
Richard
2018-06-18 Richard Sandiford <richard.sandiford@arm.com>
gcc/
* tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to
tree-vect-patterns.c.
* tree-vect-patterns.c (vect_supportable_direct_optab_p): New function.
(vect_recog_dot_prod_pattern): Use it. Remove the type_in argument.
(vect_recog_sad_pattern): Likewise.
(vect_recog_widen_sum_pattern): Likewise.
(vect_recog_pow_pattern): Likewise. Check for a null vectype.
(vect_recog_widen_shift_pattern): Remove the type_in argument.
(vect_recog_rotate_pattern): Likewise.
(vect_recog_mult_pattern): Likewise.
(vect_recog_vector_vector_shift_pattern): Likewise.
(vect_recog_divmod_pattern): Likewise.
(vect_recog_mixed_size_cond_pattern): Likewise.
(vect_recog_bool_pattern): Likewise.
(vect_recog_mask_conversion_pattern): Likewise.
(vect_try_gather_scatter_pattern): Likewise.
(vect_recog_widen_mult_pattern): Likewise. Check for a null vectype.
(vect_recog_over_widening_pattern): Likewise.
(vect_recog_gather_scatter_pattern): Likewise.
(vect_recog_func_ptr): Move from tree-vectorizer.h
(vect_vect_recog_func_ptrs): Move further down the file.
(vect_recog_func): Likewise. Remove the third argument.
(NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs.
(vect_pattern_recog_1): Expect the pattern function to do any
necessary target tests. Also expect it to provide a vector type.
Remove the type_in handling.
Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h 2018-06-14 12:27:34.104084473 +0100
+++ gcc/tree-vectorizer.h 2018-06-18 15:43:52.951038712 +0100
@@ -1616,8 +1616,6 @@ extern int vect_get_place_in_interleavin
/* Pattern recognition functions.
Additional pattern recognition functions can (and will) be added
in the future. */
-typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *);
-#define NUM_PATTERNS 15
void vect_pattern_recog (vec_info *);
/* In tree-vectorizer.c. */
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c 2018-06-18 15:43:52.000000000 +0100
+++ gcc/tree-vect-patterns.c 2018-06-18 15:43:52.951038712 +0100
@@ -47,66 +47,6 @@ Software Foundation; either version 3, o
#include "omp-simd-clone.h"
#include "predict.h"
-/* Pattern recognition functions */
-static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_divmod_pattern (vec<gimple *> *,
- tree *, tree *);
-
-static gimple *vect_recog_mult_pattern (vec<gimple *> *,
- tree *, tree *);
-
-static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *,
- tree *);
-
-struct vect_recog_func
-{
- vect_recog_func_ptr fn;
- const char *name;
-};
-
-/* Note that ordering matters - the first pattern matching on a stmt
- is taken which means usually the more complex one needs to preceed
- the less comples onex (widen_sum only after dot_prod or sad for example). */
-static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
- { vect_recog_widen_mult_pattern, "widen_mult" },
- { vect_recog_dot_prod_pattern, "dot_prod" },
- { vect_recog_sad_pattern, "sad" },
- { vect_recog_widen_sum_pattern, "widen_sum" },
- { vect_recog_pow_pattern, "pow" },
- { vect_recog_widen_shift_pattern, "widen_shift" },
- { vect_recog_over_widening_pattern, "over_widening" },
- { vect_recog_rotate_pattern, "rotate" },
- { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
- { vect_recog_divmod_pattern, "divmod" },
- { vect_recog_mult_pattern, "mult" },
- { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
- { vect_recog_bool_pattern, "bool" },
- /* This must come before mask conversion, and includes the parts
- of mask conversion that are needed for gather and scatter
- internal functions. */
- { vect_recog_gather_scatter_pattern, "gather_scatter" },
- { vect_recog_mask_conversion_pattern, "mask_conversion" }
-};
-
/* Report that we've found an instance of pattern PATTERN in
statement STMT. */
@@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_
append_pattern_def_seq (stmt_info, stmt);
}
+/* Return true if the target supports a vector version of CODE,
+ where CODE is known to map to a direct optab. ITYPE specifies
+ the type of (some of) the scalar inputs and OTYPE specifies the
+ type of the scalar result.
+
+ If CODE allows the inputs and outputs to have different type
+ (such as for WIDEN_SUM_EXPR), it is the input mode rather
+ than the output mode that determines the appropriate target pattern.
+ Operand 0 of the target pattern then specifies the mode that the output
+ must have.
+
+ When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
+ Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
+ is nonnull. */
+
+static bool
+vect_supportable_direct_optab_p (tree otype, tree_code code,
+ tree itype, tree *vecotype_out,
+ tree *vecitype_out = NULL)
+{
+ tree vecitype = get_vectype_for_scalar_type (itype);
+ if (!vecitype)
+ return false;
+
+ tree vecotype = get_vectype_for_scalar_type (otype);
+ if (!vecotype)
+ return false;
+
+ optab optab = optab_for_tree_code (code, vecitype, optab_default);
+ if (!optab)
+ return false;
+
+ insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
+ if (icode == CODE_FOR_nothing
+ || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
+ return false;
+
+ *vecotype_out = vecotype;
+ if (vecitype_out)
+ *vecitype_out = vecitype;
+ return true;
+}
+
/* Check whether STMT2 is in the same loop or basic block as STMT1.
Which of the two applies depends on whether we're currently doing
loop-based or basic-block-based vectorization, as determined by
@@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt, *last_stmt = (*stmts)[0];
tree oprnd0, oprnd1;
@@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple
vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
half_type = TREE_TYPE (oprnd00);
- *type_in = half_type;
- *type_out = type;
+ if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type,
+ type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
@@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple
*/
static gimple *
-vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree sad_oprnd0, sad_oprnd1;
@@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *s
vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
- *type_in = TREE_TYPE (sad_oprnd0);
- *type_out = sum_type;
+ if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR,
+ TREE_TYPE (sad_oprnd0), type_out))
+ return NULL;
tree var = vect_recog_temp_ssa_var (sum_type, NULL);
gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0,
@@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *s
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *s
*/
static gimple *
-vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
gimple *def_stmt0, *def_stmt1;
@@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimpl
&dummy_int, &dummy_vec))
return NULL;
- *type_in = vectype;
*type_out = get_vectype_for_scalar_type (type);
+ if (!*type_out)
+ return NULL;
/* Pattern supported. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (itype, NULL);
@@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimpl
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimpl
*/
static gimple *
-vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree base, exp;
@@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *s
if (node->simd_clones == NULL)
return NULL;
}
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+ if (!*type_out)
+ return NULL;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
new_pattern_def_seq (stmt_vinfo, g);
- *type_in = TREE_TYPE (base);
- *type_out = NULL_TREE;
tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
g = gimple_build_call (exp_decl, 1, def);
gimple_call_set_lhs (g, res);
@@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *s
/* We now have a pow or powi builtin function call with a constant
exponent. */
- *type_out = NULL_TREE;
-
/* Catch squaring. */
if ((tree_fits_shwi_p (exp)
&& tree_to_shwi (exp) == 2)
|| (TREE_CODE (exp) == REAL_CST
&& real_equal (&TREE_REAL_CST (exp), &dconst2)))
{
- *type_in = TREE_TYPE (base);
+ if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR,
+ TREE_TYPE (base), type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
stmt = gimple_build_assign (var, MULT_EXPR, base, base);
@@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *s
if (TREE_CODE (exp) == REAL_CST
&& real_equal (&TREE_REAL_CST (exp), &dconsthalf))
{
- *type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
- if (*type_in
- && direct_internal_fn_supported_p (IFN_SQRT, *type_in,
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+ if (*type_out
+ && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
OPTIMIZE_FOR_SPEED))
{
gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
@@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *s
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *s
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt, *last_stmt = (*stmts)[0];
tree oprnd0, oprnd1;
@@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple
if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt,
&promotion)
|| !promotion)
- return NULL;
+ return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
- *type_in = half_type;
- *type_out = type;
+ if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type,
+ type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1);
@@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple
demotion operation. We also check that S3 and S4 have only one use. */
static gimple *
-vect_recog_over_widening_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt = stmts->pop ();
gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL,
@@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gi
if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
|| TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type))
{
+ *type_out = get_vectype_for_scalar_type (use_type);
+ if (!*type_out)
+ return NULL;
+
/* Create NEW_TYPE->USE_TYPE conversion. */
new_oprnd = make_ssa_name (use_type);
pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var);
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt;
- *type_in = get_vectype_for_scalar_type (new_type);
- *type_out = get_vectype_for_scalar_type (use_type);
-
/* We created a pattern statement for the last statement in the
sequence, so we don't need to associate it with the pattern
statement created for PREV_STMT. Therefore, we add PREV_STMT
@@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gi
STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
= STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
- *type_in = vectype;
- *type_out = NULL_TREE;
+ *type_out = vectype;
}
stmts->safe_push (use_stmt);
@@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gi
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gi
WIDEN_LSHIFT_EXPR <a_t, CONST>. */
static gimple *
-vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
gimple *def_stmt0;
@@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimp
&dummy_int, &dummy_vec))
return NULL;
- *type_in = vectype;
*type_out = vectype_out;
/* Pattern supported. Create a stmt to be used to replace the pattern. */
@@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimp
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the rotate
S0 stmt. */
static gimple *
-vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out)
+vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
@@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *>
return NULL;
}
- *type_in = vectype;
*type_out = vectype;
- if (*type_in == NULL_TREE)
- return NULL;
if (dt == vect_external_def
&& TREE_CODE (oprnd1) == SSA_NAME
@@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *>
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the shift/rotate
S3 stmt. */
static gimple *
-vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, lhs, var;
@@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern (
if (!def_vinfo)
return NULL;
- *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
- *type_out = *type_in;
- if (*type_in == NULL_TREE)
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
+ if (*type_out == NULL_TREE)
return NULL;
tree def = NULL_TREE;
@@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tr
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace
the multiplication. */
static gimple *
-vect_recog_mult_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, vectype, itype;
@@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> *
vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
@@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> *
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the division
S1 or modulo S4 stmt. */
static gimple *
-vect_recog_divmod_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, vectype, itype, cond;
@@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *>
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
}
@@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *>
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
}
@@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *>
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the pattern.
@@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *>
a_T = (TYPE) a_it; */
static gimple *
-vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree cond_expr, then_clause, else_clause;
@@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec<
def_stmt_info = new_stmt_vec_info (def_stmt, vinfo);
set_vinfo_for_stmt (def_stmt, def_stmt_info);
STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
- *type_in = vecitype;
*type_out = vectype;
vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt);
@@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the pattern.
@@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info
but the above is more efficient. */
static gimple *
-vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
enum tree_code rhs_code;
@@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> *
}
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> *
gimple_assign_rhs2 (last_stmt),
gimple_assign_rhs3 (last_stmt));
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> *
STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree v
S4' c_1' = m_3'' ? c_2 : c_3; */
static gimple *
-vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
enum tree_code rhs_code;
@@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec<
= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec<
gimple_assign_rhs3 (last_stmt));
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec<
pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree typ
/* Try to convert STMT into a call to a gather load or scatter store
internal function. Return the final statement on success and set
- *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored.
+ *TYPE_OUT to the vector type being loaded or stored.
This function only handles gathers and scatters that were recognized
as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
static gimple *
vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
- tree *type_in, tree *type_out)
+ tree *type_out)
{
/* Currently we only support this for loop vectorization. */
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
*type_out = vectype;
- *type_in = vectype;
vect_pattern_detected ("gather/scatter pattern", stmt);
return pattern_stmt;
@@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple
/* Pattern wrapper around vect_try_gather_scatter_pattern. */
static gimple *
-vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt);
gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt,
last_stmt_info,
- type_in, type_out);
+ type_out);
if (pattern_stmt)
stmts->safe_push (last_stmt);
return pattern_stmt;
}
+typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *);
+
+struct vect_recog_func
+{
+ vect_recog_func_ptr fn;
+ const char *name;
+};
+
+/* Note that ordering matters - the first pattern matching on a stmt is
+ taken which means usually the more complex one needs to preceed the
+ less comples onex (widen_sum only after dot_prod or sad for example). */
+static vect_recog_func vect_vect_recog_func_ptrs[] = {
+ { vect_recog_widen_mult_pattern, "widen_mult" },
+ { vect_recog_dot_prod_pattern, "dot_prod" },
+ { vect_recog_sad_pattern, "sad" },
+ { vect_recog_widen_sum_pattern, "widen_sum" },
+ { vect_recog_pow_pattern, "pow" },
+ { vect_recog_widen_shift_pattern, "widen_shift" },
+ { vect_recog_over_widening_pattern, "over_widening" },
+ { vect_recog_rotate_pattern, "rotate" },
+ { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
+ { vect_recog_divmod_pattern, "divmod" },
+ { vect_recog_mult_pattern, "mult" },
+ { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
+ { vect_recog_bool_pattern, "bool" },
+ /* This must come before mask conversion, and includes the parts
+ of mask conversion that are needed for gather and scatter
+ internal functions. */
+ { vect_recog_gather_scatter_pattern, "gather_scatter" },
+ { vect_recog_mask_conversion_pattern, "mask_conversion" }
+};
+
+const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
+
/* Mark statements that are involved in a pattern. */
static inline void
@@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_st
computation pattern.
STMT: A stmt from which the pattern search should start.
- If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
- expression that computes the same functionality and can be used to
- replace the sequence of stmts that are involved in the pattern.
-
- Output:
- This function checks if the expression returned by PATTERN_RECOG_FUNC is
- supported in vector form by the target. We use 'TYPE_IN' to obtain the
- relevant vector type. If 'TYPE_IN' is already a vector type, then this
- indicates that target support had already been checked by PATTERN_RECOG_FUNC.
- If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits
- to the available target pattern.
+ If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
+ a sequence of statements that has the same functionality and can be
+ used to replace STMT. It returns the last statement in the sequence
+ and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ.
+ PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
+ statement, having first checked that the target supports the new operation
+ in that type.
This function also does some bookkeeping, as explained in the documentation
for vect_recog_pattern. */
@@ -4361,56 +4325,19 @@ vect_pattern_recog_1 (vect_recog_func *r
stmt_vec_info stmt_info;
loop_vec_info loop_vinfo;
tree pattern_vectype;
- tree type_in, type_out;
- enum tree_code code;
int i;
stmts_to_replace->truncate (0);
stmts_to_replace->quick_push (stmt);
- pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out);
+ pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype);
if (!pattern_stmt)
return false;
stmt = stmts_to_replace->last ();
stmt_info = vinfo_for_stmt (stmt);
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ gcc_assert (pattern_vectype);
- if (VECTOR_BOOLEAN_TYPE_P (type_in)
- || VECTOR_TYPE_P (type_in))
- {
- /* No need to check target support (already checked by the pattern
- recognition function). */
- pattern_vectype = type_out ? type_out : type_in;
- }
- else
- {
- /* Check target support */
- type_in = get_vectype_for_scalar_type (type_in);
- if (!type_in)
- return false;
- if (type_out)
- type_out = get_vectype_for_scalar_type (type_out);
- else
- type_out = type_in;
- if (!type_out)
- return false;
- pattern_vectype = type_out;
-
- if (is_gimple_assign (pattern_stmt))
- {
- enum insn_code icode;
- code = gimple_assign_rhs_code (pattern_stmt);
- optab optab = optab_for_tree_code (code, type_in, optab_default);
- machine_mode vec_mode = TYPE_MODE (type_in);
- if (!optab
- || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
- || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
- return false;
- }
- else
- gcc_assert (is_gimple_call (pattern_stmt));
- }
-
/* Found a vectorizable pattern. */
if (dump_enabled_p ())
{
More information about the Gcc-patches
mailing list