This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
[8/n] PR85694: Make patterns check for target support
- From: Richard Sandiford <richard dot sandiford at arm dot com>
- To: gcc-patches at gcc dot gnu dot org
- Date: Mon, 18 Jun 2018 16:01:04 +0100
- Subject: [8/n] PR85694: Make patterns check for target support
This patch makes pattern recognisers do their own checking for vector
types and target support. Previously some recognisers did this
themselves and some left it to vect_pattern_recog_1.
Doing this means we can get rid of the type_in argument, which was
ignored if the recogniser did its own checking. It also means
we create fewer junk statements.
Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install?
Richard
2018-06-18 Richard Sandiford <richard.sandiford@arm.com>
gcc/
* tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to
tree-vect-patterns.c.
* tree-vect-patterns.c (vect_supportable_direct_optab_p): New function.
(vect_recog_dot_prod_pattern): Use it. Remove the type_in argument.
(vect_recog_sad_pattern): Likewise.
(vect_recog_widen_sum_pattern): Likewise.
(vect_recog_pow_pattern): Likewise. Check for a null vectype.
(vect_recog_widen_shift_pattern): Remove the type_in argument.
(vect_recog_rotate_pattern): Likewise.
(vect_recog_mult_pattern): Likewise.
(vect_recog_vector_vector_shift_pattern): Likewise.
(vect_recog_divmod_pattern): Likewise.
(vect_recog_mixed_size_cond_pattern): Likewise.
(vect_recog_bool_pattern): Likewise.
(vect_recog_mask_conversion_pattern): Likewise.
(vect_try_gather_scatter_pattern): Likewise.
(vect_recog_widen_mult_pattern): Likewise. Check for a null vectype.
(vect_recog_over_widening_pattern): Likewise.
(vect_recog_gather_scatter_pattern): Likewise.
(vect_recog_func_ptr): Move from tree-vectorizer.h
(vect_vect_recog_func_ptrs): Move further down the file.
(vect_recog_func): Likewise. Remove the third argument.
(NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs.
(vect_pattern_recog_1): Expect the pattern function to do any
necessary target tests. Also expect it to provide a vector type.
Remove the type_in handling.
Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h 2018-06-14 12:27:34.104084473 +0100
+++ gcc/tree-vectorizer.h 2018-06-18 15:43:52.951038712 +0100
@@ -1616,8 +1616,6 @@ extern int vect_get_place_in_interleavin
/* Pattern recognition functions.
Additional pattern recognition functions can (and will) be added
in the future. */
-typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *);
-#define NUM_PATTERNS 15
void vect_pattern_recog (vec_info *);
/* In tree-vectorizer.c. */
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c 2018-06-18 15:43:52.000000000 +0100
+++ gcc/tree-vect-patterns.c 2018-06-18 15:43:52.951038712 +0100
@@ -47,66 +47,6 @@ Software Foundation; either version 3, o
#include "omp-simd-clone.h"
#include "predict.h"
-/* Pattern recognition functions */
-static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *,
- tree *);
-static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_divmod_pattern (vec<gimple *> *,
- tree *, tree *);
-
-static gimple *vect_recog_mult_pattern (vec<gimple *> *,
- tree *, tree *);
-
-static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *,
- tree *, tree *);
-static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *,
- tree *);
-
-struct vect_recog_func
-{
- vect_recog_func_ptr fn;
- const char *name;
-};
-
-/* Note that ordering matters - the first pattern matching on a stmt
- is taken which means usually the more complex one needs to preceed
- the less comples onex (widen_sum only after dot_prod or sad for example). */
-static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
- { vect_recog_widen_mult_pattern, "widen_mult" },
- { vect_recog_dot_prod_pattern, "dot_prod" },
- { vect_recog_sad_pattern, "sad" },
- { vect_recog_widen_sum_pattern, "widen_sum" },
- { vect_recog_pow_pattern, "pow" },
- { vect_recog_widen_shift_pattern, "widen_shift" },
- { vect_recog_over_widening_pattern, "over_widening" },
- { vect_recog_rotate_pattern, "rotate" },
- { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
- { vect_recog_divmod_pattern, "divmod" },
- { vect_recog_mult_pattern, "mult" },
- { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
- { vect_recog_bool_pattern, "bool" },
- /* This must come before mask conversion, and includes the parts
- of mask conversion that are needed for gather and scatter
- internal functions. */
- { vect_recog_gather_scatter_pattern, "gather_scatter" },
- { vect_recog_mask_conversion_pattern, "mask_conversion" }
-};
-
/* Report that we've found an instance of pattern PATTERN in
statement STMT. */
@@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_
append_pattern_def_seq (stmt_info, stmt);
}
+/* Return true if the target supports a vector version of CODE,
+ where CODE is known to map to a direct optab. ITYPE specifies
+ the type of (some of) the scalar inputs and OTYPE specifies the
+ type of the scalar result.
+
+ If CODE allows the inputs and outputs to have different type
+ (such as for WIDEN_SUM_EXPR), it is the input mode rather
+ than the output mode that determines the appropriate target pattern.
+ Operand 0 of the target pattern then specifies the mode that the output
+ must have.
+
+ When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
+ Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
+ is nonnull. */
+
+static bool
+vect_supportable_direct_optab_p (tree otype, tree_code code,
+ tree itype, tree *vecotype_out,
+ tree *vecitype_out = NULL)
+{
+ tree vecitype = get_vectype_for_scalar_type (itype);
+ if (!vecitype)
+ return false;
+
+ tree vecotype = get_vectype_for_scalar_type (otype);
+ if (!vecotype)
+ return false;
+
+ optab optab = optab_for_tree_code (code, vecitype, optab_default);
+ if (!optab)
+ return false;
+
+ insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
+ if (icode == CODE_FOR_nothing
+ || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
+ return false;
+
+ *vecotype_out = vecotype;
+ if (vecitype_out)
+ *vecitype_out = vecitype;
+ return true;
+}
+
/* Check whether STMT2 is in the same loop or basic block as STMT1.
Which of the two applies depends on whether we're currently doing
loop-based or basic-block-based vectorization, as determined by
@@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt, *last_stmt = (*stmts)[0];
tree oprnd0, oprnd1;
@@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple
vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
half_type = TREE_TYPE (oprnd00);
- *type_in = half_type;
- *type_out = type;
+ if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type,
+ type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
@@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple
*/
static gimple *
-vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree sad_oprnd0, sad_oprnd1;
@@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *s
vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
- *type_in = TREE_TYPE (sad_oprnd0);
- *type_out = sum_type;
+ if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR,
+ TREE_TYPE (sad_oprnd0), type_out))
+ return NULL;
tree var = vect_recog_temp_ssa_var (sum_type, NULL);
gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0,
@@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *s
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *s
*/
static gimple *
-vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
gimple *def_stmt0, *def_stmt1;
@@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimpl
&dummy_int, &dummy_vec))
return NULL;
- *type_in = vectype;
*type_out = get_vectype_for_scalar_type (type);
+ if (!*type_out)
+ return NULL;
/* Pattern supported. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (itype, NULL);
@@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimpl
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimpl
*/
static gimple *
-vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree base, exp;
@@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *s
if (node->simd_clones == NULL)
return NULL;
}
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+ if (!*type_out)
+ return NULL;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
new_pattern_def_seq (stmt_vinfo, g);
- *type_in = TREE_TYPE (base);
- *type_out = NULL_TREE;
tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
g = gimple_build_call (exp_decl, 1, def);
gimple_call_set_lhs (g, res);
@@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *s
/* We now have a pow or powi builtin function call with a constant
exponent. */
- *type_out = NULL_TREE;
-
/* Catch squaring. */
if ((tree_fits_shwi_p (exp)
&& tree_to_shwi (exp) == 2)
|| (TREE_CODE (exp) == REAL_CST
&& real_equal (&TREE_REAL_CST (exp), &dconst2)))
{
- *type_in = TREE_TYPE (base);
+ if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR,
+ TREE_TYPE (base), type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
stmt = gimple_build_assign (var, MULT_EXPR, base, base);
@@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *s
if (TREE_CODE (exp) == REAL_CST
&& real_equal (&TREE_REAL_CST (exp), &dconsthalf))
{
- *type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
- if (*type_in
- && direct_internal_fn_supported_p (IFN_SQRT, *type_in,
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+ if (*type_out
+ && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
OPTIMIZE_FOR_SPEED))
{
gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
@@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *s
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *s
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt, *last_stmt = (*stmts)[0];
tree oprnd0, oprnd1;
@@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple
if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt,
&promotion)
|| !promotion)
- return NULL;
+ return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
- *type_in = half_type;
- *type_out = type;
+ if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type,
+ type_out))
+ return NULL;
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1);
@@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple
demotion operation. We also check that S3 and S4 have only one use. */
static gimple *
-vect_recog_over_widening_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *stmt = stmts->pop ();
gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL,
@@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gi
if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
|| TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type))
{
+ *type_out = get_vectype_for_scalar_type (use_type);
+ if (!*type_out)
+ return NULL;
+
/* Create NEW_TYPE->USE_TYPE conversion. */
new_oprnd = make_ssa_name (use_type);
pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var);
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt;
- *type_in = get_vectype_for_scalar_type (new_type);
- *type_out = get_vectype_for_scalar_type (use_type);
-
/* We created a pattern statement for the last statement in the
sequence, so we don't need to associate it with the pattern
statement created for PREV_STMT. Therefore, we add PREV_STMT
@@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gi
STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
= STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
- *type_in = vectype;
- *type_out = NULL_TREE;
+ *type_out = vectype;
}
stmts->safe_push (use_stmt);
@@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gi
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the sequence of
@@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gi
WIDEN_LSHIFT_EXPR <a_t, CONST>. */
static gimple *
-vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
gimple *def_stmt0;
@@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimp
&dummy_int, &dummy_vec))
return NULL;
- *type_in = vectype;
*type_out = vectype_out;
/* Pattern supported. Create a stmt to be used to replace the pattern. */
@@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimp
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the rotate
S0 stmt. */
static gimple *
-vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out)
+vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
@@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *>
return NULL;
}
- *type_in = vectype;
*type_out = vectype;
- if (*type_in == NULL_TREE)
- return NULL;
if (dt == vect_external_def
&& TREE_CODE (oprnd1) == SSA_NAME
@@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *>
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the shift/rotate
S3 stmt. */
static gimple *
-vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, lhs, var;
@@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern (
if (!def_vinfo)
return NULL;
- *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
- *type_out = *type_in;
- if (*type_in == NULL_TREE)
+ *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
+ if (*type_out == NULL_TREE)
return NULL;
tree def = NULL_TREE;
@@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tr
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace
the multiplication. */
static gimple *
-vect_recog_mult_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, vectype, itype;
@@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> *
vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
@@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> *
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the division
S1 or modulo S4 stmt. */
static gimple *
-vect_recog_divmod_pattern (vec<gimple *> *stmts,
- tree *type_in, tree *type_out)
+vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
tree oprnd0, oprnd1, vectype, itype, cond;
@@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *>
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
}
@@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *>
stmts->safe_push (last_stmt);
- *type_in = vectype;
*type_out = vectype;
return pattern_stmt;
}
@@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *>
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the pattern.
@@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *>
a_T = (TYPE) a_it; */
static gimple *
-vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = (*stmts)[0];
tree cond_expr, then_clause, else_clause;
@@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec<
def_stmt_info = new_stmt_vec_info (def_stmt, vinfo);
set_vinfo_for_stmt (def_stmt, def_stmt_info);
STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
- *type_in = vecitype;
*type_out = vectype;
vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt);
@@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info
Output:
- * TYPE_IN: The type of the input arguments to the pattern.
-
* TYPE_OUT: The type of the output of this pattern.
* Return value: A new stmt that will be used to replace the pattern.
@@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info
but the above is more efficient. */
static gimple *
-vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
enum tree_code rhs_code;
@@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> *
}
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> *
gimple_assign_rhs2 (last_stmt),
gimple_assign_rhs3 (last_stmt));
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> *
STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
*type_out = vectype;
- *type_in = vectype;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
@@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree v
S4' c_1' = m_3'' ? c_2 : c_3; */
static gimple *
-vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
enum tree_code rhs_code;
@@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec<
= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec<
gimple_assign_rhs3 (last_stmt));
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec<
pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
*type_out = vectype1;
- *type_in = vectype1;
stmts->safe_push (last_stmt);
vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
@@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree typ
/* Try to convert STMT into a call to a gather load or scatter store
internal function. Return the final statement on success and set
- *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored.
+ *TYPE_OUT to the vector type being loaded or stored.
This function only handles gathers and scatters that were recognized
as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
static gimple *
vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
- tree *type_in, tree *type_out)
+ tree *type_out)
{
/* Currently we only support this for loop vectorization. */
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
*type_out = vectype;
- *type_in = vectype;
vect_pattern_detected ("gather/scatter pattern", stmt);
return pattern_stmt;
@@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple
/* Pattern wrapper around vect_try_gather_scatter_pattern. */
static gimple *
-vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in,
- tree *type_out)
+vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out)
{
gimple *last_stmt = stmts->pop ();
stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt);
gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt,
last_stmt_info,
- type_in, type_out);
+ type_out);
if (pattern_stmt)
stmts->safe_push (last_stmt);
return pattern_stmt;
}
+typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *);
+
+struct vect_recog_func
+{
+ vect_recog_func_ptr fn;
+ const char *name;
+};
+
+/* Note that ordering matters - the first pattern matching on a stmt is
+ taken which means usually the more complex one needs to preceed the
+ less comples onex (widen_sum only after dot_prod or sad for example). */
+static vect_recog_func vect_vect_recog_func_ptrs[] = {
+ { vect_recog_widen_mult_pattern, "widen_mult" },
+ { vect_recog_dot_prod_pattern, "dot_prod" },
+ { vect_recog_sad_pattern, "sad" },
+ { vect_recog_widen_sum_pattern, "widen_sum" },
+ { vect_recog_pow_pattern, "pow" },
+ { vect_recog_widen_shift_pattern, "widen_shift" },
+ { vect_recog_over_widening_pattern, "over_widening" },
+ { vect_recog_rotate_pattern, "rotate" },
+ { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
+ { vect_recog_divmod_pattern, "divmod" },
+ { vect_recog_mult_pattern, "mult" },
+ { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
+ { vect_recog_bool_pattern, "bool" },
+ /* This must come before mask conversion, and includes the parts
+ of mask conversion that are needed for gather and scatter
+ internal functions. */
+ { vect_recog_gather_scatter_pattern, "gather_scatter" },
+ { vect_recog_mask_conversion_pattern, "mask_conversion" }
+};
+
+const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
+
/* Mark statements that are involved in a pattern. */
static inline void
@@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_st
computation pattern.
STMT: A stmt from which the pattern search should start.
- If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
- expression that computes the same functionality and can be used to
- replace the sequence of stmts that are involved in the pattern.
-
- Output:
- This function checks if the expression returned by PATTERN_RECOG_FUNC is
- supported in vector form by the target. We use 'TYPE_IN' to obtain the
- relevant vector type. If 'TYPE_IN' is already a vector type, then this
- indicates that target support had already been checked by PATTERN_RECOG_FUNC.
- If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits
- to the available target pattern.
+ If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
+ a sequence of statements that has the same functionality and can be
+ used to replace STMT. It returns the last statement in the sequence
+ and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ.
+ PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
+ statement, having first checked that the target supports the new operation
+ in that type.
This function also does some bookkeeping, as explained in the documentation
for vect_recog_pattern. */
@@ -4361,56 +4325,19 @@ vect_pattern_recog_1 (vect_recog_func *r
stmt_vec_info stmt_info;
loop_vec_info loop_vinfo;
tree pattern_vectype;
- tree type_in, type_out;
- enum tree_code code;
int i;
stmts_to_replace->truncate (0);
stmts_to_replace->quick_push (stmt);
- pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out);
+ pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype);
if (!pattern_stmt)
return false;
stmt = stmts_to_replace->last ();
stmt_info = vinfo_for_stmt (stmt);
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ gcc_assert (pattern_vectype);
- if (VECTOR_BOOLEAN_TYPE_P (type_in)
- || VECTOR_TYPE_P (type_in))
- {
- /* No need to check target support (already checked by the pattern
- recognition function). */
- pattern_vectype = type_out ? type_out : type_in;
- }
- else
- {
- /* Check target support */
- type_in = get_vectype_for_scalar_type (type_in);
- if (!type_in)
- return false;
- if (type_out)
- type_out = get_vectype_for_scalar_type (type_out);
- else
- type_out = type_in;
- if (!type_out)
- return false;
- pattern_vectype = type_out;
-
- if (is_gimple_assign (pattern_stmt))
- {
- enum insn_code icode;
- code = gimple_assign_rhs_code (pattern_stmt);
- optab optab = optab_for_tree_code (code, type_in, optab_default);
- machine_mode vec_mode = TYPE_MODE (type_in);
- if (!optab
- || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
- || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
- return false;
- }
- else
- gcc_assert (is_gimple_call (pattern_stmt));
- }
-
/* Found a vectorizable pattern. */
if (dump_enabled_p ())
{