[PATCH 4/5] vect: Make reduction code handle calls

Richard Sandiford richard.sandiford@arm.com
Tue Nov 16 16:24:40 GMT 2021


Richard Biener via Gcc-patches <gcc-patches@gcc.gnu.org> writes:
> On Wed, Nov 10, 2021 at 1:48 PM Richard Sandiford via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
>>
>> This patch extends the reduction code to handle calls.  So far
>> it's a structural change only; a later patch adds support for
>> specific function reductions.
>>
>> Most of the patch consists of using code_helper and gimple_match_op
>> to describe the reduction operations.  The other main change is that
>> vectorizable_call now needs to handle fully-predicated reductions.
>>
>> Tested on aarch64-linux-gnu and x86_64-linux-gnu.  OK to install?
>>
>> Richard
>>
>>
>> gcc/
>>         * builtins.h (associated_internal_fn): Declare overload that
>>         takes a (combined_cfn, return type) pair.
>>         * builtins.c (associated_internal_fn): Split new overload out
>>         of original fndecl version.  Also provide an overload that takes
>>         a (combined_cfn, return type) pair.
>>         * internal-fn.h (commutative_binary_fn_p): Declare.
>>         (associative_binary_fn_p): Likewise.
>>         * internal-fn.c (commutative_binary_fn_p): New function,
>>         split out from...
>>         (first_commutative_argument): ...here.
>>         (associative_binary_fn_p): New function.
>>         * gimple-match.h (code_helper): Add a constructor that takes
>>         internal functions.
>>         (commutative_binary_op_p): Declare.
>>         (associative_binary_op_p): Likewise.
>>         (canonicalize_code): Likewise.
>>         (directly_supported_p): Likewise.
>>         (get_conditional_internal_fn): Likewise.
>>         (gimple_build): New overload that takes a code_helper.
>>         * gimple-fold.c (gimple_build): Likewise.
>>         * gimple-match-head.c (commutative_binary_op_p): New function.
>>         (associative_binary_op_p): Likewise.
>>         (canonicalize_code): Likewise.
>>         (directly_supported_p): Likewise.
>>         (get_conditional_internal_fn): Likewise.
>>         * tree-vectorizer.h: Include gimple-match.h.
>>         (neutral_op_for_reduction): Take a code_helper instead of a tree_code.
>>         (needs_fold_left_reduction_p): Likewise.
>>         (reduction_fn_for_scalar_code): Likewise.
>>         (vect_can_vectorize_without_simd_p): Declare a nNew overload that takes
>>         a code_helper.
>>         * tree-vect-loop.c: Include case-cfn-macros.h.
>>         (fold_left_reduction_fn): Take a code_helper instead of a tree_code.
>>         (reduction_fn_for_scalar_code): Likewise.
>>         (neutral_op_for_reduction): Likewise.
>>         (needs_fold_left_reduction_p): Likewise.
>>         (use_mask_by_cond_expr_p): Likewise.
>>         (build_vect_cond_expr): Likewise.
>>         (vect_create_partial_epilog): Likewise.  Use gimple_build rather
>>         than gimple_build_assign.
>>         (check_reduction_path): Handle calls and operate on code_helpers
>>         rather than tree_codes.
>>         (vect_is_simple_reduction): Likewise.
>>         (vect_model_reduction_cost): Likewise.
>>         (vect_find_reusable_accumulator): Likewise.
>>         (vect_create_epilog_for_reduction): Likewise.
>>         (vect_transform_cycle_phi): Likewise.
>>         (vectorizable_reduction): Likewise.  Make more use of
>>         lane_reduc_code_p.
>>         (vect_transform_reduction): Use gimple_extract_op but expect
>>         a tree_code for now.
>>         (vect_can_vectorize_without_simd_p): New overload that takes
>>         a code_helper.
>>         * tree-vect-stmts.c (vectorizable_call): Handle reductions in
>>         fully-masked loops.
>>         * tree-vect-patterns.c (vect_mark_pattern_stmts): Use
>>         gimple_extract_op when updating STMT_VINFO_REDUC_IDX.
>> ---
>>  gcc/builtins.c           |  46 ++++-
>>  gcc/builtins.h           |   1 +
>>  gcc/gimple-fold.c        |   9 +
>>  gcc/gimple-match-head.c  |  70 +++++++
>>  gcc/gimple-match.h       |  20 ++
>>  gcc/internal-fn.c        |  46 ++++-
>>  gcc/internal-fn.h        |   2 +
>>  gcc/tree-vect-loop.c     | 420 +++++++++++++++++++--------------------
>>  gcc/tree-vect-patterns.c |  23 ++-
>>  gcc/tree-vect-stmts.c    |  66 ++++--
>>  gcc/tree-vectorizer.h    |  10 +-
>>  11 files changed, 455 insertions(+), 258 deletions(-)
>>
>> diff --git a/gcc/builtins.c b/gcc/builtins.c
>> index 384864bfb3a..03829c03a5a 100644
>> --- a/gcc/builtins.c
>> +++ b/gcc/builtins.c
>> @@ -2139,17 +2139,17 @@ mathfn_built_in_type (combined_fn fn)
>>  #undef SEQ_OF_CASE_MATHFN
>>  }
>>
>> -/* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
>> -   return its code, otherwise return IFN_LAST.  Note that this function
>> -   only tests whether the function is defined in internals.def, not whether
>> -   it is actually available on the target.  */
>> +/* Check whether there is an internal function associated with function FN
>> +   and return type RETURN_TYPE.  Return the function if so, otherwise return
>> +   IFN_LAST.
>>
>> -internal_fn
>> -associated_internal_fn (tree fndecl)
>> +   Note that this function only tests whether the function is defined in
>> +   internals.def, not whether it is actually available on the target.  */
>> +
>> +static internal_fn
>> +associated_internal_fn (built_in_function fn, tree return_type)
>>  {
>> -  gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
>> -  tree return_type = TREE_TYPE (TREE_TYPE (fndecl));
>> -  switch (DECL_FUNCTION_CODE (fndecl))
>> +  switch (fn)
>>      {
>>  #define DEF_INTERNAL_FLT_FN(NAME, FLAGS, OPTAB, TYPE) \
>>      CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME;
>> @@ -2177,6 +2177,34 @@ associated_internal_fn (tree fndecl)
>>      }
>>  }
>>
>> +/* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
>> +   return its code, otherwise return IFN_LAST.  Note that this function
>> +   only tests whether the function is defined in internals.def, not whether
>> +   it is actually available on the target.  */
>> +
>> +internal_fn
>> +associated_internal_fn (tree fndecl)
>> +{
>> +  gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
>> +  return associated_internal_fn (DECL_FUNCTION_CODE (fndecl),
>> +                                TREE_TYPE (TREE_TYPE (fndecl)));
>> +}
>> +
>> +/* Check whether there is an internal function associated with function CFN
>> +   and return type RETURN_TYPE.  Return the function if so, otherwise return
>> +   IFN_LAST.
>> +
>> +   Note that this function only tests whether the function is defined in
>> +   internals.def, not whether it is actually available on the target.  */
>> +
>> +internal_fn
>> +associated_internal_fn (combined_fn cfn, tree return_type)
>> +{
>> +  if (internal_fn_p (cfn))
>> +    return as_internal_fn (cfn);
>> +  return associated_internal_fn (as_builtin_fn (cfn), return_type);
>> +}
>> +
>>  /* If CALL is a call to a BUILT_IN_NORMAL function that could be replaced
>>     on the current target by a call to an internal function, return the
>>     code of that internal function, otherwise return IFN_LAST.  The caller
>> diff --git a/gcc/builtins.h b/gcc/builtins.h
>> index 5e4d86e9c37..c99670b12f1 100644
>> --- a/gcc/builtins.h
>> +++ b/gcc/builtins.h
>> @@ -148,6 +148,7 @@ extern char target_percent_s_newline[4];
>>  extern bool target_char_cst_p (tree t, char *p);
>>  extern rtx get_memory_rtx (tree exp, tree len);
>>
>> +extern internal_fn associated_internal_fn (combined_fn, tree);
>>  extern internal_fn associated_internal_fn (tree);
>>  extern internal_fn replacement_internal_fn (gcall *);
>>
>> diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c
>> index 9daf2cc590c..a937f130815 100644
>> --- a/gcc/gimple-fold.c
>> +++ b/gcc/gimple-fold.c
>> @@ -8808,6 +8808,15 @@ gimple_build (gimple_seq *seq, location_t loc, combined_fn fn,
>>    return res;
>>  }
>
> Toplevel comment missing.  You add this for two operands, please
> also add it for one and three (even if unused).

OK.  On the comment side: I was hoping to piggy-back on the comment
for the previous overload :-)  I'll add a new one though.

>> +tree
>> +gimple_build (gimple_seq *seq, location_t loc, code_helper code,
>> +             tree type, tree op0, tree op1)
>> +{
>> +  if (code.is_tree_code ())
>> +    return gimple_build (seq, loc, tree_code (code), type, op0, op1);
>> +  return gimple_build (seq, loc, combined_fn (code), type, op0, op1);
>> +}
>> +
>>  /* Build the conversion (TYPE) OP with a result of type TYPE
>>     with location LOC if such conversion is neccesary in GIMPLE,
>>     simplifying it first.
>> diff --git a/gcc/gimple-match-head.c b/gcc/gimple-match-head.c
>> index d4d7d767075..4558a3db5fc 100644
>> --- a/gcc/gimple-match-head.c
>> +++ b/gcc/gimple-match-head.c
>> @@ -1304,3 +1304,73 @@ optimize_successive_divisions_p (tree divisor, tree inner_div)
>>      }
>>    return true;
>>  }
>> +
>> +/* If CODE, operating on TYPE, represents a built-in function that has an
>> +   associated internal function, return the associated internal function,
>> +   otherwise return CODE.  This function does not check whether the
>> +   internal function is supported, only that it exists.  */
>
> Hmm, why not name the function associated_internal_fn then, or
> have it contain internal_fn?

I didn't want to call it associated_internal_fn because the existing
forms of that function return an internal_fn.  Here the idea is to
avoid having multiple representations of the same operation, so that
code_helpers can be compared for equality.  I guess the fact that that
currently means mapping built-in functions to internal functions
(and nothing else) is more of an implementation detail.

So I guess the emphasis in the comment is wrong.  How about if I change
it to:

/* Return a canonical form for CODE when operating on TYPE.  The idea
   is to remove redundant ways of representing the same operation so
   that code_helpers can be hashed and compared for equality.

   The only current canonicalization is to replace built-in functions
   with internal functions, in cases where internal-fn.def defines
   such an internal function.

   Note that the new code_helper cannot necessarily be used in place of
   the original code_helper.  For example, the new code_helper might be
   an internal function that the target does not support.  */

> I also wonder why all the functions below are not member functions
> of code_helper?

TBH I don't really like that style very much. :-)  E.g. it's not obvious
that directly_supported_p should be a member of code_helper when it's
really querying information in the optab array.  If code_helper was
available more widely then the code would probably be in optab*.c
instead.

The queries are also about (code_helper, type) pairs rather than about
code_helpers in isolation.

IMO the current code_helper member functions seem like the right set,
in that they're providing the abstraction “tree_code or combined_fn”.
That abstraction can then be used in all sorts of places.

>> +code_helper
>> +canonicalize_code (code_helper code, tree type)
>> +{
>> +  if (code.is_fn_code ())
>> +    return associated_internal_fn (combined_fn (code), type);
>> +  return code;
>> +}
>> +
>> +/* Return true if CODE is a binary operation that is commutative when
>> +   operating on type TYPE.  */
>> +
>> +bool
>> +commutative_binary_op_p (code_helper code, tree type)
>> +{
>> +  if (code.is_tree_code ())
>> +    return commutative_tree_code (tree_code (code));
>> +  auto cfn = combined_fn (code);
>> +  return commutative_binary_fn_p (associated_internal_fn (cfn, type));
>> +}
>
> Do we need commutative_ternary_op_p?  Can we do a more generic
> commutative_p instead?

How about using the first_commutative_argument interface from internal-fn.c,
which returns the first argument in a commutative pair or -1 if none?

>> +
>> +/* Return true if CODE is a binary operation that is associative when
>> +   operating on type TYPE.  */
>> +
>> +bool
>> +associative_binary_op_p (code_helper code, tree type)
>
> We only have associative_tree_code, is _binary relevant here?

But we do have commutative_ternary_tree_code, like you say :-)
I guess it didn't seem worth going back and renaming all the
uses of commutative_tree_code to commutative_binary_tree_code
to account for that.

So this was partly future-proofing.  It was also partly to emphasise
that the caller doesn't need to check that the operator is a binary
operator first (although I'm not sure the name actually achieves
that, oh well).

>> +{
>> +  if (code.is_tree_code ())
>> +    return associative_tree_code (tree_code (code));
>> +  auto cfn = combined_fn (code);
>> +  return associative_binary_fn_p (associated_internal_fn (cfn, type));
>> +}
>> +
>> +/* Return true if the target directly supports operation CODE on type TYPE.
>> +   QUERY_TYPE acts as for optab_for_tree_code.  */
>> +
>> +bool
>> +directly_supported_p (code_helper code, tree type, optab_subtype query_type)
>> +{
>> +  if (code.is_tree_code ())
>> +    {
>> +      direct_optab optab = optab_for_tree_code (tree_code (code), type,
>> +                                               query_type);
>> +      return (optab != unknown_optab
>> +             && optab_handler (optab, TYPE_MODE (type)) != CODE_FOR_nothing);
>> +    }
>> +  gcc_assert (query_type == optab_default
>> +             || (query_type == optab_vector && VECTOR_TYPE_P (type))
>> +             || (query_type == optab_scalar && !VECTOR_TYPE_P (type)));
>> +  internal_fn ifn = associated_internal_fn (combined_fn (code), type);
>> +  return (direct_internal_fn_p (ifn)
>> +         && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
>> +}
>> +
>> +/* A wrapper around the internal-fn.c versions of get_conditional_internal_fn
>> +   for a code_helper CODE operating on type TYPE.  */
>> +
>> +internal_fn
>> +get_conditional_internal_fn (code_helper code, tree type)
>> +{
>> +  if (code.is_tree_code ())
>> +    return get_conditional_internal_fn (tree_code (code));
>> +  auto cfn = combined_fn (code);
>> +  return get_conditional_internal_fn (associated_internal_fn (cfn, type));
>> +}
>> diff --git a/gcc/gimple-match.h b/gcc/gimple-match.h
>> index 1b9dc3851c2..6d24a8a2378 100644
>> --- a/gcc/gimple-match.h
>> +++ b/gcc/gimple-match.h
>> @@ -31,6 +31,7 @@ public:
>>    code_helper () {}
>>    code_helper (tree_code code) : rep ((int) code) {}
>>    code_helper (combined_fn fn) : rep (-(int) fn) {}
>> +  code_helper (internal_fn fn) : rep (-(int) as_combined_fn (fn)) {}
>>    explicit operator tree_code () const { return (tree_code) rep; }
>>    explicit operator combined_fn () const { return (combined_fn) -rep; }
>
> Do we want a
>
>      explicit operator internal_fn () const { ... }
>
> for completeness?

Yeah, guess that would simplify some things.  Maybe a built_in_function
one too.

>
>>    bool is_tree_code () const { return rep > 0; }
>> @@ -346,4 +347,23 @@ tree maybe_push_res_to_seq (gimple_match_op *, gimple_seq *,
>>  void maybe_build_generic_op (gimple_match_op *);
>>
>>
>> +bool commutative_binary_op_p (code_helper, tree);
>> +bool associative_binary_op_p (code_helper, tree);
>> +code_helper canonicalize_code (code_helper, tree);
>> +
>> +#ifdef GCC_OPTABS_TREE_H
>> +bool directly_supported_p (code_helper, tree, optab_subtype = optab_default);
>> +#endif
>> +
>> +internal_fn get_conditional_internal_fn (code_helper, tree);
>> +
>> +extern tree gimple_build (gimple_seq *, location_t,
>> +                         code_helper, tree, tree, tree);
>> +inline tree
>> +gimple_build (gimple_seq *seq, code_helper code, tree type, tree op0,
>> +             tree op1)
>> +{
>> +  return gimple_build (seq, UNKNOWN_LOCATION, code, type, op0, op1);
>> +}
>
> That looks a bit misplaced and should be in gimple-fold.h, no?

Files don't need to include gimple-match.h before gimple-fold.h.
So I saw this as being a bit like the optab stuff: it's generalising
interfaces provided elsewhere for the "tree_code or combined_fn" union.

One alternative would be to put the functions in gimple-fold.h but
protect them with #ifdef GCC_GIMPLE_MATCH_H.  Another would be to
move code_helper somewhere else, such as gimple.h.

Thanks,
Richard

>> +
>>  #endif  /* GCC_GIMPLE_MATCH_H */
>> diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
>> index da7d8355214..7b13db6dfe3 100644
>> --- a/gcc/internal-fn.c
>> +++ b/gcc/internal-fn.c
>> @@ -3815,6 +3815,43 @@ direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
>>    return direct_internal_fn_supported_p (fn, types, opt_type);
>>  }
>>
>> +/* Return true if FN is a commutative binary operation.  */
>> +
>> +bool
>> +commutative_binary_fn_p (internal_fn fn)
>> +{
>> +  switch (fn)
>> +    {
>> +    case IFN_AVG_FLOOR:
>> +    case IFN_AVG_CEIL:
>> +    case IFN_MULH:
>> +    case IFN_MULHS:
>> +    case IFN_MULHRS:
>> +    case IFN_FMIN:
>> +    case IFN_FMAX:
>> +      return true;
>> +
>> +    default:
>> +      return false;
>> +    }
>> +}
>> +
>> +/* Return true if FN is an associative binary operation.  */
>> +
>> +bool
>> +associative_binary_fn_p (internal_fn fn)
>
> See above - without _binary?
>
>> +{
>> +  switch (fn)
>> +    {
>> +    case IFN_FMIN:
>> +    case IFN_FMAX:
>> +      return true;
>> +
>> +    default:
>> +      return false;
>> +    }
>> +}
>> +
>>  /* If FN is commutative in two consecutive arguments, return the
>>     index of the first, otherwise return -1.  */
>>
>> @@ -3827,13 +3864,6 @@ first_commutative_argument (internal_fn fn)
>>      case IFN_FMS:
>>      case IFN_FNMA:
>>      case IFN_FNMS:
>> -    case IFN_AVG_FLOOR:
>> -    case IFN_AVG_CEIL:
>> -    case IFN_MULH:
>> -    case IFN_MULHS:
>> -    case IFN_MULHRS:
>> -    case IFN_FMIN:
>> -    case IFN_FMAX:
>>        return 0;
>>
>>      case IFN_COND_ADD:
>> @@ -3852,7 +3882,7 @@ first_commutative_argument (internal_fn fn)
>>        return 1;
>>
>>      default:
>> -      return -1;
>> +      return commutative_binary_fn_p (fn) ? 0 : -1;
>>      }
>>  }
>>
>> diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
>> index 19d0f849a5a..82ef4b0d792 100644
>> --- a/gcc/internal-fn.h
>> +++ b/gcc/internal-fn.h
>> @@ -206,6 +206,8 @@ direct_internal_fn_supported_p (internal_fn fn, tree type0, tree type1,
>>                                          opt_type);
>>  }
>>
>> +extern bool commutative_binary_fn_p (internal_fn);
>
> I'm somewhat missing commutative_ternary_fn_p which would work
> on FMAs?
>
> So that was all API comments, the real changes below look good to me.
>
> Thanks,
> Richard.
>
>> +extern bool associative_binary_fn_p (internal_fn);
>>  extern int first_commutative_argument (internal_fn);
>>
>>  extern bool set_edom_supported_p (void);
>> diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
>> index 1cd5dbcb6f7..cae895a88f2 100644
>> --- a/gcc/tree-vect-loop.c
>> +++ b/gcc/tree-vect-loop.c
>> @@ -54,6 +54,7 @@ along with GCC; see the file COPYING3.  If not see
>>  #include "tree-vector-builder.h"
>>  #include "vec-perm-indices.h"
>>  #include "tree-eh.h"
>> +#include "case-cfn-macros.h"
>>
>>  /* Loop Vectorization Pass.
>>
>> @@ -3125,17 +3126,14 @@ vect_analyze_loop (class loop *loop, vec_info_shared *shared)
>>     it in *REDUC_FN if so.  */
>>
>>  static bool
>> -fold_left_reduction_fn (tree_code code, internal_fn *reduc_fn)
>> +fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
>>  {
>> -  switch (code)
>> +  if (code == PLUS_EXPR)
>>      {
>> -    case PLUS_EXPR:
>>        *reduc_fn = IFN_FOLD_LEFT_PLUS;
>>        return true;
>> -
>> -    default:
>> -      return false;
>>      }
>> +  return false;
>>  }
>>
>>  /* Function reduction_fn_for_scalar_code
>> @@ -3152,21 +3150,22 @@ fold_left_reduction_fn (tree_code code, internal_fn *reduc_fn)
>>     Return FALSE if CODE currently cannot be vectorized as reduction.  */
>>
>>  bool
>> -reduction_fn_for_scalar_code (enum tree_code code, internal_fn *reduc_fn)
>> +reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
>>  {
>> -  switch (code)
>> -    {
>> +  if (code.is_tree_code ())
>> +    switch (tree_code (code))
>> +      {
>>        case MAX_EXPR:
>> -        *reduc_fn = IFN_REDUC_MAX;
>> -        return true;
>> +       *reduc_fn = IFN_REDUC_MAX;
>> +       return true;
>>
>>        case MIN_EXPR:
>> -        *reduc_fn = IFN_REDUC_MIN;
>> -        return true;
>> +       *reduc_fn = IFN_REDUC_MIN;
>> +       return true;
>>
>>        case PLUS_EXPR:
>> -        *reduc_fn = IFN_REDUC_PLUS;
>> -        return true;
>> +       *reduc_fn = IFN_REDUC_PLUS;
>> +       return true;
>>
>>        case BIT_AND_EXPR:
>>         *reduc_fn = IFN_REDUC_AND;
>> @@ -3182,12 +3181,13 @@ reduction_fn_for_scalar_code (enum tree_code code, internal_fn *reduc_fn)
>>
>>        case MULT_EXPR:
>>        case MINUS_EXPR:
>> -        *reduc_fn = IFN_LAST;
>> -        return true;
>> +       *reduc_fn = IFN_LAST;
>> +       return true;
>>
>>        default:
>> -       return false;
>> +       break;
>>      }
>> +  return false;
>>  }
>>
>>  /* If there is a neutral value X such that a reduction would not be affected
>> @@ -3197,32 +3197,35 @@ reduction_fn_for_scalar_code (enum tree_code code, internal_fn *reduc_fn)
>>     then INITIAL_VALUE is that value, otherwise it is null.  */
>>
>>  tree
>> -neutral_op_for_reduction (tree scalar_type, tree_code code, tree initial_value)
>> +neutral_op_for_reduction (tree scalar_type, code_helper code,
>> +                         tree initial_value)
>>  {
>> -  switch (code)
>> -    {
>> -    case WIDEN_SUM_EXPR:
>> -    case DOT_PROD_EXPR:
>> -    case SAD_EXPR:
>> -    case PLUS_EXPR:
>> -    case MINUS_EXPR:
>> -    case BIT_IOR_EXPR:
>> -    case BIT_XOR_EXPR:
>> -      return build_zero_cst (scalar_type);
>> +  if (code.is_tree_code ())
>> +    switch (tree_code (code))
>> +      {
>> +      case WIDEN_SUM_EXPR:
>> +      case DOT_PROD_EXPR:
>> +      case SAD_EXPR:
>> +      case PLUS_EXPR:
>> +      case MINUS_EXPR:
>> +      case BIT_IOR_EXPR:
>> +      case BIT_XOR_EXPR:
>> +       return build_zero_cst (scalar_type);
>>
>> -    case MULT_EXPR:
>> -      return build_one_cst (scalar_type);
>> +      case MULT_EXPR:
>> +       return build_one_cst (scalar_type);
>>
>> -    case BIT_AND_EXPR:
>> -      return build_all_ones_cst (scalar_type);
>> +      case BIT_AND_EXPR:
>> +       return build_all_ones_cst (scalar_type);
>>
>> -    case MAX_EXPR:
>> -    case MIN_EXPR:
>> -      return initial_value;
>> +      case MAX_EXPR:
>> +      case MIN_EXPR:
>> +       return initial_value;
>>
>> -    default:
>> -      return NULL_TREE;
>> -    }
>> +      default:
>> +       break;
>> +      }
>> +  return NULL_TREE;
>>  }
>>
>>  /* Error reporting helper for vect_is_simple_reduction below.  GIMPLE statement
>> @@ -3239,26 +3242,27 @@ report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
>>     overflow must wrap.  */
>>
>>  bool
>> -needs_fold_left_reduction_p (tree type, tree_code code)
>> +needs_fold_left_reduction_p (tree type, code_helper code)
>>  {
>>    /* CHECKME: check for !flag_finite_math_only too?  */
>>    if (SCALAR_FLOAT_TYPE_P (type))
>> -    switch (code)
>> -      {
>> -      case MIN_EXPR:
>> -      case MAX_EXPR:
>> -       return false;
>> +    {
>> +      if (code.is_tree_code ())
>> +       switch (tree_code (code))
>> +         {
>> +         case MIN_EXPR:
>> +         case MAX_EXPR:
>> +           return false;
>>
>> -      default:
>> -       return !flag_associative_math;
>> -      }
>> +         default:
>> +           break;
>> +         }
>> +      return !flag_associative_math;
>> +    }
>>
>>    if (INTEGRAL_TYPE_P (type))
>> -    {
>> -      if (!operation_no_trapping_overflow (type, code))
>> -       return true;
>> -      return false;
>> -    }
>> +    return (!code.is_tree_code ()
>> +           || !operation_no_trapping_overflow (type, tree_code (code)));
>>
>>    if (SAT_FIXED_POINT_TYPE_P (type))
>>      return true;
>> @@ -3272,7 +3276,7 @@ needs_fold_left_reduction_p (tree type, tree_code code)
>>
>>  static bool
>>  check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
>> -                     tree loop_arg, enum tree_code *code,
>> +                     tree loop_arg, code_helper *code,
>>                       vec<std::pair<ssa_op_iter, use_operand_p> > &path)
>>  {
>>    auto_bitmap visited;
>> @@ -3347,45 +3351,57 @@ pop:
>>    for (unsigned i = 1; i < path.length (); ++i)
>>      {
>>        gimple *use_stmt = USE_STMT (path[i].second);
>> -      tree op = USE_FROM_PTR (path[i].second);
>> -      if (! is_gimple_assign (use_stmt)
>> +      gimple_match_op op;
>> +      if (!gimple_extract_op (use_stmt, &op))
>> +       {
>> +         fail = true;
>> +         break;
>> +       }
>> +      unsigned int opi = op.num_ops;
>> +      if (gassign *assign = dyn_cast<gassign *> (use_stmt))
>> +       {
>>           /* The following make sure we can compute the operand index
>>              easily plus it mostly disallows chaining via COND_EXPR condition
>>              operands.  */
>> -         || (gimple_assign_rhs1_ptr (use_stmt) != path[i].second->use
>> -             && (gimple_num_ops (use_stmt) <= 2
>> -                 || gimple_assign_rhs2_ptr (use_stmt) != path[i].second->use)
>> -             && (gimple_num_ops (use_stmt) <= 3
>> -                 || gimple_assign_rhs3_ptr (use_stmt) != path[i].second->use)))
>> +         for (opi = 0; opi < op.num_ops; ++opi)
>> +           if (gimple_assign_rhs1_ptr (assign) + opi == path[i].second->use)
>> +             break;
>> +       }
>> +      else if (gcall *call = dyn_cast<gcall *> (use_stmt))
>> +       {
>> +         for (opi = 0; opi < op.num_ops; ++opi)
>> +           if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
>> +             break;
>> +       }
>> +      if (opi == op.num_ops)
>>         {
>>           fail = true;
>>           break;
>>         }
>> -      tree_code use_code = gimple_assign_rhs_code (use_stmt);
>> -      if (use_code == MINUS_EXPR)
>> +      op.code = canonicalize_code (op.code, op.type);
>> +      if (op.code == MINUS_EXPR)
>>         {
>> -         use_code = PLUS_EXPR;
>> +         op.code = PLUS_EXPR;
>>           /* Track whether we negate the reduction value each iteration.  */
>> -         if (gimple_assign_rhs2 (use_stmt) == op)
>> +         if (op.ops[1] == op.ops[opi])
>>             neg = ! neg;
>>         }
>> -      if (CONVERT_EXPR_CODE_P (use_code)
>> -         && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (use_stmt)),
>> -                                   TREE_TYPE (gimple_assign_rhs1 (use_stmt))))
>> +      if (CONVERT_EXPR_CODE_P (op.code)
>> +         && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
>>         ;
>>        else if (*code == ERROR_MARK)
>>         {
>> -         *code = use_code;
>> -         sign = TYPE_SIGN (TREE_TYPE (gimple_assign_lhs (use_stmt)));
>> +         *code = op.code;
>> +         sign = TYPE_SIGN (op.type);
>>         }
>> -      else if (use_code != *code)
>> +      else if (op.code != *code)
>>         {
>>           fail = true;
>>           break;
>>         }
>> -      else if ((use_code == MIN_EXPR
>> -               || use_code == MAX_EXPR)
>> -              && sign != TYPE_SIGN (TREE_TYPE (gimple_assign_lhs (use_stmt))))
>> +      else if ((op.code == MIN_EXPR
>> +               || op.code == MAX_EXPR)
>> +              && sign != TYPE_SIGN (op.type))
>>         {
>>           fail = true;
>>           break;
>> @@ -3397,7 +3413,7 @@ pop:
>>        imm_use_iterator imm_iter;
>>        gimple *op_use_stmt;
>>        unsigned cnt = 0;
>> -      FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op)
>> +      FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
>>         if (!is_gimple_debug (op_use_stmt)
>>             && (*code != ERROR_MARK
>>                 || flow_bb_inside_loop_p (loop, gimple_bb (op_use_stmt))))
>> @@ -3427,7 +3443,7 @@ check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
>>                       tree loop_arg, enum tree_code code)
>>  {
>>    auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
>> -  enum tree_code code_;
>> +  code_helper code_;
>>    return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path)
>>           && code_ == code);
>>  }
>> @@ -3596,9 +3612,9 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
>>        gimple *def1 = SSA_NAME_DEF_STMT (op1);
>>        if (gimple_bb (def1)
>>           && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
>> -          && loop->inner
>> -          && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
>> -          && is_gimple_assign (def1)
>> +         && loop->inner
>> +         && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
>> +         && (is_gimple_assign (def1) || is_gimple_call (def1))
>>           && is_a <gphi *> (phi_use_stmt)
>>           && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt)))
>>          {
>> @@ -3615,7 +3631,7 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
>>
>>    /* Look for the expression computing latch_def from then loop PHI result.  */
>>    auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
>> -  enum tree_code code;
>> +  code_helper code;
>>    if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
>>                             path))
>>      {
>> @@ -3633,15 +3649,24 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
>>         {
>>           gimple *stmt = USE_STMT (path[i].second);
>>           stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
>> -         STMT_VINFO_REDUC_IDX (stmt_info)
>> -           = path[i].second->use - gimple_assign_rhs1_ptr (stmt);
>> -         enum tree_code stmt_code = gimple_assign_rhs_code (stmt);
>> -         bool leading_conversion = (CONVERT_EXPR_CODE_P (stmt_code)
>> +         gimple_match_op op;
>> +         if (!gimple_extract_op (stmt, &op))
>> +           gcc_unreachable ();
>> +         if (gassign *assign = dyn_cast<gassign *> (stmt))
>> +           STMT_VINFO_REDUC_IDX (stmt_info)
>> +             = path[i].second->use - gimple_assign_rhs1_ptr (assign);
>> +         else
>> +           {
>> +             gcall *call = as_a<gcall *> (stmt);
>> +             STMT_VINFO_REDUC_IDX (stmt_info)
>> +               = path[i].second->use - gimple_call_arg_ptr (call, 0);
>> +           }
>> +         bool leading_conversion = (CONVERT_EXPR_CODE_P (op.code)
>>                                      && (i == 1 || i == path.length () - 1));
>> -         if ((stmt_code != code && !leading_conversion)
>> +         if ((op.code != code && !leading_conversion)
>>               /* We can only handle the final value in epilogue
>>                  generation for reduction chains.  */
>> -             || (i != 1 && !has_single_use (gimple_assign_lhs (stmt))))
>> +             || (i != 1 && !has_single_use (gimple_get_lhs (stmt))))
>>             is_slp_reduc = false;
>>           /* For reduction chains we support a trailing/leading
>>              conversions.  We do not store those in the actual chain.  */
>> @@ -4390,8 +4415,6 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
>>                            int ncopies, stmt_vector_for_cost *cost_vec)
>>  {
>>    int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
>> -  enum tree_code code;
>> -  optab optab;
>>    tree vectype;
>>    machine_mode mode;
>>    class loop *loop = NULL;
>> @@ -4407,7 +4430,9 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
>>    mode = TYPE_MODE (vectype);
>>    stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
>>
>> -  code = gimple_assign_rhs_code (orig_stmt_info->stmt);
>> +  gimple_match_op op;
>> +  if (!gimple_extract_op (orig_stmt_info->stmt, &op))
>> +    gcc_unreachable ();
>>
>>    if (reduction_type == EXTRACT_LAST_REDUCTION)
>>      /* No extra instructions are needed in the prologue.  The loop body
>> @@ -4501,20 +4526,16 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
>>        else
>>         {
>>           int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
>> -         tree bitsize =
>> -           TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt_info->stmt)));
>> +         tree bitsize = TYPE_SIZE (op.type);
>>           int element_bitsize = tree_to_uhwi (bitsize);
>>           int nelements = vec_size_in_bits / element_bitsize;
>>
>> -         if (code == COND_EXPR)
>> -           code = MAX_EXPR;
>> -
>> -         optab = optab_for_tree_code (code, vectype, optab_default);
>> +         if (op.code == COND_EXPR)
>> +           op.code = MAX_EXPR;
>>
>>           /* We have a whole vector shift available.  */
>> -         if (optab != unknown_optab
>> -             && VECTOR_MODE_P (mode)
>> -             && optab_handler (optab, mode) != CODE_FOR_nothing
>> +         if (VECTOR_MODE_P (mode)
>> +             && directly_supported_p (op.code, vectype)
>>               && have_whole_vector_shift (mode))
>>             {
>>               /* Final reduction via vector shifts and the reduction operator.
>> @@ -4855,7 +4876,7 @@ vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
>>          initialize the accumulator with a neutral value instead.  */
>>        if (!operand_equal_p (initial_value, main_adjustment))
>>         return false;
>> -      tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
>> +      code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
>>        initial_values[0] = neutral_op_for_reduction (TREE_TYPE (initial_value),
>>                                                     code, initial_value);
>>      }
>> @@ -4870,7 +4891,7 @@ vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
>>     CODE emitting stmts before GSI.  Returns a vector def of VECTYPE.  */
>>
>>  static tree
>> -vect_create_partial_epilog (tree vec_def, tree vectype, enum tree_code code,
>> +vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
>>                             gimple_seq *seq)
>>  {
>>    unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def)).to_constant ();
>> @@ -4953,9 +4974,7 @@ vect_create_partial_epilog (tree vec_def, tree vectype, enum tree_code code,
>>           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
>>         }
>>
>> -      new_temp = make_ssa_name (vectype1);
>> -      epilog_stmt = gimple_build_assign (new_temp, code, dst1, dst2);
>> -      gimple_seq_add_stmt_without_update (seq, epilog_stmt);
>> +      new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
>>      }
>>
>>    return new_temp;
>> @@ -5032,7 +5051,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
>>      }
>>    gphi *reduc_def_stmt
>>      = as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
>> -  enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
>> +  code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
>>    internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
>>    tree vectype;
>>    machine_mode mode;
>> @@ -5699,14 +5718,9 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
>>        tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
>>                                                            stype, nunits1);
>>        reduce_with_shift = have_whole_vector_shift (mode1);
>> -      if (!VECTOR_MODE_P (mode1))
>> +      if (!VECTOR_MODE_P (mode1)
>> +         || !directly_supported_p (code, vectype1))
>>         reduce_with_shift = false;
>> -      else
>> -       {
>> -         optab optab = optab_for_tree_code (code, vectype1, optab_default);
>> -         if (optab_handler (optab, mode1) == CODE_FOR_nothing)
>> -           reduce_with_shift = false;
>> -       }
>>
>>        /* First reduce the vector to the desired vector size we should
>>          do shift reduction on by combining upper and lower halves.  */
>> @@ -5944,7 +5958,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
>>    for (k = 0; k < live_out_stmts.size (); k++)
>>      {
>>        stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
>> -      scalar_dest = gimple_assign_lhs (scalar_stmt_info->stmt);
>> +      scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
>>
>>        phis.create (3);
>>        /* Find the loop-closed-use at the loop exit of the original scalar
>> @@ -6277,7 +6291,7 @@ is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, class loop *loop)
>>     CODE is the code for the operation.  COND_FN is the conditional internal
>>     function, if it exists.  VECTYPE_IN is the type of the vector input.  */
>>  static bool
>> -use_mask_by_cond_expr_p (enum tree_code code, internal_fn cond_fn,
>> +use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
>>                          tree vectype_in)
>>  {
>>    if (cond_fn != IFN_LAST
>> @@ -6285,15 +6299,17 @@ use_mask_by_cond_expr_p (enum tree_code code, internal_fn cond_fn,
>>                                          OPTIMIZE_FOR_SPEED))
>>      return false;
>>
>> -  switch (code)
>> -    {
>> -    case DOT_PROD_EXPR:
>> -    case SAD_EXPR:
>> -      return true;
>> +  if (code.is_tree_code ())
>> +    switch (tree_code (code))
>> +      {
>> +      case DOT_PROD_EXPR:
>> +      case SAD_EXPR:
>> +       return true;
>>
>> -    default:
>> -      return false;
>> -    }
>> +      default:
>> +       break;
>> +      }
>> +  return false;
>>  }
>>
>>  /* Insert a conditional expression to enable masked vectorization.  CODE is the
>> @@ -6301,10 +6317,10 @@ use_mask_by_cond_expr_p (enum tree_code code, internal_fn cond_fn,
>>     mask.  GSI is a statement iterator used to place the new conditional
>>     expression.  */
>>  static void
>> -build_vect_cond_expr (enum tree_code code, tree vop[3], tree mask,
>> +build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
>>                       gimple_stmt_iterator *gsi)
>>  {
>> -  switch (code)
>> +  switch (tree_code (code))
>>      {
>>      case DOT_PROD_EXPR:
>>        {
>> @@ -6390,12 +6406,10 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>                         slp_instance slp_node_instance,
>>                         stmt_vector_for_cost *cost_vec)
>>  {
>> -  tree scalar_dest;
>>    tree vectype_in = NULL_TREE;
>>    class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
>>    enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
>>    stmt_vec_info cond_stmt_vinfo = NULL;
>> -  tree scalar_type;
>>    int i;
>>    int ncopies;
>>    bool single_defuse_cycle = false;
>> @@ -6508,18 +6522,18 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>          info_for_reduction to work.  */
>>        if (STMT_VINFO_LIVE_P (vdef))
>>         STMT_VINFO_REDUC_DEF (def) = phi_info;
>> -      gassign *assign = dyn_cast <gassign *> (vdef->stmt);
>> -      if (!assign)
>> +      gimple_match_op op;
>> +      if (!gimple_extract_op (vdef->stmt, &op))
>>         {
>>           if (dump_enabled_p ())
>>             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> -                            "reduction chain includes calls.\n");
>> +                            "reduction chain includes unsupported"
>> +                            " statement type.\n");
>>           return false;
>>         }
>> -      if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
>> +      if (CONVERT_EXPR_CODE_P (op.code))
>>         {
>> -         if (!tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (assign)),
>> -                                     TREE_TYPE (gimple_assign_rhs1 (assign))))
>> +         if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
>>             {
>>               if (dump_enabled_p ())
>>                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> @@ -6530,7 +6544,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>        else if (!stmt_info)
>>         /* First non-conversion stmt.  */
>>         stmt_info = vdef;
>> -      reduc_def = gimple_op (vdef->stmt, 1 + STMT_VINFO_REDUC_IDX (vdef));
>> +      reduc_def = op.ops[STMT_VINFO_REDUC_IDX (vdef)];
>>        reduc_chain_length++;
>>        if (!stmt_info && slp_node)
>>         slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
>> @@ -6588,26 +6602,24 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>
>>    tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
>>    STMT_VINFO_REDUC_VECTYPE (reduc_info) = vectype_out;
>> -  gassign *stmt = as_a <gassign *> (stmt_info->stmt);
>> -  enum tree_code code = gimple_assign_rhs_code (stmt);
>> -  bool lane_reduc_code_p
>> -    = (code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR);
>> -  int op_type = TREE_CODE_LENGTH (code);
>> +  gimple_match_op op;
>> +  if (!gimple_extract_op (stmt_info->stmt, &op))
>> +    gcc_unreachable ();
>> +  bool lane_reduc_code_p = (op.code == DOT_PROD_EXPR
>> +                           || op.code == WIDEN_SUM_EXPR
>> +                           || op.code == SAD_EXPR);
>>    enum optab_subtype optab_query_kind = optab_vector;
>> -  if (code == DOT_PROD_EXPR
>> -      && TYPE_SIGN (TREE_TYPE (gimple_assign_rhs1 (stmt)))
>> -          != TYPE_SIGN (TREE_TYPE (gimple_assign_rhs2 (stmt))))
>> +  if (op.code == DOT_PROD_EXPR
>> +      && (TYPE_SIGN (TREE_TYPE (op.ops[0]))
>> +         != TYPE_SIGN (TREE_TYPE (op.ops[1]))))
>>      optab_query_kind = optab_vector_mixed_sign;
>>
>> -
>> -  scalar_dest = gimple_assign_lhs (stmt);
>> -  scalar_type = TREE_TYPE (scalar_dest);
>> -  if (!POINTER_TYPE_P (scalar_type) && !INTEGRAL_TYPE_P (scalar_type)
>> -      && !SCALAR_FLOAT_TYPE_P (scalar_type))
>> +  if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
>> +      && !SCALAR_FLOAT_TYPE_P (op.type))
>>      return false;
>>
>>    /* Do not try to vectorize bit-precision reductions.  */
>> -  if (!type_has_mode_precision_p (scalar_type))
>> +  if (!type_has_mode_precision_p (op.type))
>>      return false;
>>
>>    /* For lane-reducing ops we're reducing the number of reduction PHIs
>> @@ -6626,25 +6638,23 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>       The last use is the reduction variable.  In case of nested cycle this
>>       assumption is not true: we use reduc_index to record the index of the
>>       reduction variable.  */
>> -  slp_tree *slp_op = XALLOCAVEC (slp_tree, op_type);
>> +  slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
>>    /* We need to skip an extra operand for COND_EXPRs with embedded
>>       comparison.  */
>>    unsigned opno_adjust = 0;
>> -  if (code == COND_EXPR
>> -      && COMPARISON_CLASS_P (gimple_assign_rhs1 (stmt)))
>> +  if (op.code == COND_EXPR && COMPARISON_CLASS_P (op.ops[0]))
>>      opno_adjust = 1;
>> -  for (i = 0; i < op_type; i++)
>> +  for (i = 0; i < (int) op.num_ops; i++)
>>      {
>>        /* The condition of COND_EXPR is checked in vectorizable_condition().  */
>> -      if (i == 0 && code == COND_EXPR)
>> +      if (i == 0 && op.code == COND_EXPR)
>>          continue;
>>
>>        stmt_vec_info def_stmt_info;
>>        enum vect_def_type dt;
>> -      tree op;
>>        if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_for_stmt_info,
>> -                              i + opno_adjust, &op, &slp_op[i], &dt, &tem,
>> -                              &def_stmt_info))
>> +                              i + opno_adjust, &op.ops[i], &slp_op[i], &dt,
>> +                              &tem, &def_stmt_info))
>>         {
>>           if (dump_enabled_p ())
>>             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> @@ -6669,13 +6679,13 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>                   < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (tem))))))
>>         vectype_in = tem;
>>
>> -      if (code == COND_EXPR)
>> +      if (op.code == COND_EXPR)
>>         {
>>           /* Record how the non-reduction-def value of COND_EXPR is defined.  */
>>           if (dt == vect_constant_def)
>>             {
>>               cond_reduc_dt = dt;
>> -             cond_reduc_val = op;
>> +             cond_reduc_val = op.ops[i];
>>             }
>>           if (dt == vect_induction_def
>>               && def_stmt_info
>> @@ -6845,7 +6855,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>            (and also the same tree-code) when generating the epilog code and
>>            when generating the code inside the loop.  */
>>
>> -  enum tree_code orig_code = STMT_VINFO_REDUC_CODE (phi_info);
>> +  code_helper orig_code = STMT_VINFO_REDUC_CODE (phi_info);
>>    STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
>>
>>    vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
>> @@ -6864,7 +6874,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>           && !REDUC_GROUP_FIRST_ELEMENT (stmt_info)
>>           && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
>>         ;
>> -      else if (needs_fold_left_reduction_p (scalar_type, orig_code))
>> +      else if (needs_fold_left_reduction_p (op.type, orig_code))
>>         {
>>           /* When vectorizing a reduction chain w/o SLP the reduction PHI
>>              is not directy used in stmt.  */
>> @@ -6879,8 +6889,8 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>           STMT_VINFO_REDUC_TYPE (reduc_info)
>>             = reduction_type = FOLD_LEFT_REDUCTION;
>>         }
>> -      else if (!commutative_tree_code (orig_code)
>> -              || !associative_tree_code (orig_code))
>> +      else if (!commutative_binary_op_p (orig_code, op.type)
>> +              || !associative_binary_op_p (orig_code, op.type))
>>         {
>>           if (dump_enabled_p ())
>>             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> @@ -6935,7 +6945,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>    else if (reduction_type == COND_REDUCTION)
>>      {
>>        int scalar_precision
>> -       = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
>> +       = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
>>        cr_index_scalar_type = make_unsigned_type (scalar_precision);
>>        cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
>>                                                 vectype_out);
>> @@ -7121,28 +7131,19 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>
>>    if (single_defuse_cycle || lane_reduc_code_p)
>>      {
>> -      gcc_assert (code != COND_EXPR);
>> +      gcc_assert (op.code != COND_EXPR);
>>
>>        /* 4. Supportable by target?  */
>>        bool ok = true;
>>
>>        /* 4.1. check support for the operation in the loop  */
>> -      optab optab = optab_for_tree_code (code, vectype_in, optab_query_kind);
>> -      if (!optab)
>> -       {
>> -         if (dump_enabled_p ())
>> -           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> -                            "no optab.\n");
>> -         ok = false;
>> -        }
>> -
>>        machine_mode vec_mode = TYPE_MODE (vectype_in);
>> -      if (ok && optab_handler (optab, vec_mode) == CODE_FOR_nothing)
>> +      if (!directly_supported_p (op.code, vectype_in, optab_query_kind))
>>          {
>>            if (dump_enabled_p ())
>>              dump_printf (MSG_NOTE, "op not supported by target.\n");
>>           if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
>> -             || !vect_can_vectorize_without_simd_p (code))
>> +             || !vect_can_vectorize_without_simd_p (op.code))
>>             ok = false;
>>           else
>>             if (dump_enabled_p ())
>> @@ -7150,7 +7151,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>          }
>>
>>        if (vect_emulated_vector_p (vectype_in)
>> -         && !vect_can_vectorize_without_simd_p (code))
>> +         && !vect_can_vectorize_without_simd_p (op.code))
>>         {
>>           if (dump_enabled_p ())
>>             dump_printf (MSG_NOTE, "using word mode not possible.\n");
>> @@ -7183,11 +7184,9 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>
>>    if (slp_node
>>        && !(!single_defuse_cycle
>> -          && code != DOT_PROD_EXPR
>> -          && code != WIDEN_SUM_EXPR
>> -          && code != SAD_EXPR
>> +          && !lane_reduc_code_p
>>            && reduction_type != FOLD_LEFT_REDUCTION))
>> -    for (i = 0; i < op_type; i++)
>> +    for (i = 0; i < (int) op.num_ops; i++)
>>        if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_in))
>>         {
>>           if (dump_enabled_p ())
>> @@ -7206,10 +7205,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>    /* Cost the reduction op inside the loop if transformed via
>>       vect_transform_reduction.  Otherwise this is costed by the
>>       separate vectorizable_* routines.  */
>> -  if (single_defuse_cycle
>> -      || code == DOT_PROD_EXPR
>> -      || code == WIDEN_SUM_EXPR
>> -      || code == SAD_EXPR)
>> +  if (single_defuse_cycle || lane_reduc_code_p)
>>      record_stmt_cost (cost_vec, ncopies, vector_stmt, stmt_info, 0, vect_body);
>>
>>    if (dump_enabled_p ()
>> @@ -7220,9 +7216,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>    /* All but single defuse-cycle optimized, lane-reducing and fold-left
>>       reductions go through their own vectorizable_* routines.  */
>>    if (!single_defuse_cycle
>> -      && code != DOT_PROD_EXPR
>> -      && code != WIDEN_SUM_EXPR
>> -      && code != SAD_EXPR
>> +      && !lane_reduc_code_p
>>        && reduction_type != FOLD_LEFT_REDUCTION)
>>      {
>>        stmt_vec_info tem
>> @@ -7238,10 +7232,10 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>>    else if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
>>      {
>>        vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
>> -      internal_fn cond_fn = get_conditional_internal_fn (code);
>> +      internal_fn cond_fn = get_conditional_internal_fn (op.code, op.type);
>>
>>        if (reduction_type != FOLD_LEFT_REDUCTION
>> -         && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
>> +         && !use_mask_by_cond_expr_p (op.code, cond_fn, vectype_in)
>>           && (cond_fn == IFN_LAST
>>               || !direct_internal_fn_supported_p (cond_fn, vectype_in,
>>                                                   OPTIMIZE_FOR_SPEED)))
>> @@ -7294,24 +7288,11 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
>>        gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info) == vect_double_reduction_def);
>>      }
>>
>> -  gassign *stmt = as_a <gassign *> (stmt_info->stmt);
>> -  enum tree_code code = gimple_assign_rhs_code (stmt);
>> -  int op_type = TREE_CODE_LENGTH (code);
>> -
>> -  /* Flatten RHS.  */
>> -  tree ops[3];
>> -  switch (get_gimple_rhs_class (code))
>> -    {
>> -    case GIMPLE_TERNARY_RHS:
>> -      ops[2] = gimple_assign_rhs3 (stmt);
>> -      /* Fall thru.  */
>> -    case GIMPLE_BINARY_RHS:
>> -      ops[0] = gimple_assign_rhs1 (stmt);
>> -      ops[1] = gimple_assign_rhs2 (stmt);
>> -      break;
>> -    default:
>> -      gcc_unreachable ();
>> -    }
>> +  gimple_match_op op;
>> +  if (!gimple_extract_op (stmt_info->stmt, &op))
>> +    gcc_unreachable ();
>> +  gcc_assert (op.code.is_tree_code ());
>> +  auto code = tree_code (op.code);
>>
>>    /* All uses but the last are expected to be defined in the loop.
>>       The last use is the reduction variable.  In case of nested cycle this
>> @@ -7359,7 +7340,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
>>        internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
>>        return vectorize_fold_left_reduction
>>           (loop_vinfo, stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
>> -          reduc_fn, ops, vectype_in, reduc_index, masks);
>> +          reduc_fn, op.ops, vectype_in, reduc_index, masks);
>>      }
>>
>>    bool single_defuse_cycle = STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info);
>> @@ -7369,22 +7350,22 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
>>               || code == SAD_EXPR);
>>
>>    /* Create the destination vector  */
>> -  tree scalar_dest = gimple_assign_lhs (stmt);
>> +  tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
>>    tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
>>
>>    vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, ncopies,
>>                      single_defuse_cycle && reduc_index == 0
>> -                    ? NULL_TREE : ops[0], &vec_oprnds0,
>> +                    ? NULL_TREE : op.ops[0], &vec_oprnds0,
>>                      single_defuse_cycle && reduc_index == 1
>> -                    ? NULL_TREE : ops[1], &vec_oprnds1,
>> -                    op_type == ternary_op
>> +                    ? NULL_TREE : op.ops[1], &vec_oprnds1,
>> +                    op.num_ops == 3
>>                      && !(single_defuse_cycle && reduc_index == 2)
>> -                    ? ops[2] : NULL_TREE, &vec_oprnds2);
>> +                    ? op.ops[2] : NULL_TREE, &vec_oprnds2);
>>    if (single_defuse_cycle)
>>      {
>>        gcc_assert (!slp_node);
>>        vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, 1,
>> -                                    ops[reduc_index],
>> +                                    op.ops[reduc_index],
>>                                      reduc_index == 0 ? &vec_oprnds0
>>                                      : (reduc_index == 1 ? &vec_oprnds1
>>                                         : &vec_oprnds2));
>> @@ -7414,7 +7395,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
>>         }
>>        else
>>         {
>> -         if (op_type == ternary_op)
>> +         if (op.num_ops == 3)
>>             vop[2] = vec_oprnds2[i];
>>
>>           if (masked_loop_p && mask_by_cond_expr)
>> @@ -7546,7 +7527,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
>>             {
>>               tree initial_value
>>                 = (num_phis == 1 ? initial_values[0] : NULL_TREE);
>> -             tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
>> +             code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
>>               tree neutral_op
>>                 = neutral_op_for_reduction (TREE_TYPE (vectype_out),
>>                                             code, initial_value);
>> @@ -7603,7 +7584,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
>>           if (!reduc_info->reduc_initial_values.is_empty ())
>>             {
>>               initial_def = reduc_info->reduc_initial_values[0];
>> -             enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
>> +             code_helper code = STMT_VINFO_REDUC_CODE (reduc_info);
>>               tree neutral_op
>>                 = neutral_op_for_reduction (TREE_TYPE (initial_def),
>>                                             code, initial_def);
>> @@ -7901,6 +7882,15 @@ vect_can_vectorize_without_simd_p (tree_code code)
>>      }
>>  }
>>
>> +/* Likewise, but taking a code_helper.  */
>> +
>> +bool
>> +vect_can_vectorize_without_simd_p (code_helper code)
>> +{
>> +  return (code.is_tree_code ()
>> +         && vect_can_vectorize_without_simd_p (tree_code (code)));
>> +}
>> +
>>  /* Function vectorizable_induction
>>
>>     Check if STMT_INFO performs an induction computation that can be vectorized.
>> diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
>> index 854cbcff390..26421ee5511 100644
>> --- a/gcc/tree-vect-patterns.c
>> +++ b/gcc/tree-vect-patterns.c
>> @@ -5594,8 +5594,10 @@ vect_mark_pattern_stmts (vec_info *vinfo,
>>    /* Transfer reduction path info to the pattern.  */
>>    if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
>>      {
>> -      tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
>> -                               1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
>> +      gimple_match_op op;
>> +      if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
>> +       gcc_unreachable ();
>> +      tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
>>        /* Search the pattern def sequence and the main pattern stmt.  Note
>>           we may have inserted all into a containing pattern def sequence
>>          so the following is a bit awkward.  */
>> @@ -5615,14 +5617,15 @@ vect_mark_pattern_stmts (vec_info *vinfo,
>>        do
>>         {
>>           bool found = false;
>> -         for (unsigned i = 1; i < gimple_num_ops (s); ++i)
>> -           if (gimple_op (s, i) == lookfor)
>> -             {
>> -               STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i - 1;
>> -               lookfor = gimple_get_lhs (s);
>> -               found = true;
>> -               break;
>> -             }
>> +         if (gimple_extract_op (s, &op))
>> +           for (unsigned i = 0; i < op.num_ops; ++i)
>> +             if (op.ops[i] == lookfor)
>> +               {
>> +                 STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
>> +                 lookfor = gimple_get_lhs (s);
>> +                 found = true;
>> +                 break;
>> +               }
>>           if (s == pattern_stmt)
>>             {
>>               if (!found && dump_enabled_p ())
>> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
>> index 03cc7267cf8..1e197023b98 100644
>> --- a/gcc/tree-vect-stmts.c
>> +++ b/gcc/tree-vect-stmts.c
>> @@ -3202,7 +3202,6 @@ vectorizable_call (vec_info *vinfo,
>>    int ndts = ARRAY_SIZE (dt);
>>    int ncopies, j;
>>    auto_vec<tree, 8> vargs;
>> -  auto_vec<tree, 8> orig_vargs;
>>    enum { NARROW, NONE, WIDEN } modifier;
>>    size_t i, nargs;
>>    tree lhs;
>> @@ -3426,6 +3425,8 @@ vectorizable_call (vec_info *vinfo,
>>       needs to be generated.  */
>>    gcc_assert (ncopies >= 1);
>>
>> +  int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
>> +  internal_fn cond_fn = get_conditional_internal_fn (ifn);
>>    vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
>>    if (!vec_stmt) /* transformation not required.  */
>>      {
>> @@ -3446,14 +3447,33 @@ vectorizable_call (vec_info *vinfo,
>>         record_stmt_cost (cost_vec, ncopies / 2,
>>                           vec_promote_demote, stmt_info, 0, vect_body);
>>
>> -      if (loop_vinfo && mask_opno >= 0)
>> +      if (loop_vinfo
>> +         && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
>> +         && (reduc_idx >= 0 || mask_opno >= 0))
>>         {
>> -         unsigned int nvectors = (slp_node
>> -                                  ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)
>> -                                  : ncopies);
>> -         tree scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
>> -         vect_record_loop_mask (loop_vinfo, masks, nvectors,
>> -                                vectype_out, scalar_mask);
>> +         if (reduc_idx >= 0
>> +             && (cond_fn == IFN_LAST
>> +                 || !direct_internal_fn_supported_p (cond_fn, vectype_out,
>> +                                                     OPTIMIZE_FOR_SPEED)))
>> +           {
>> +             if (dump_enabled_p ())
>> +               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>> +                                "can't use a fully-masked loop because no"
>> +                                " conditional operation is available.\n");
>> +             LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
>> +           }
>> +         else
>> +           {
>> +             unsigned int nvectors
>> +               = (slp_node
>> +                  ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)
>> +                  : ncopies);
>> +             tree scalar_mask = NULL_TREE;
>> +             if (mask_opno >= 0)
>> +               scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
>> +             vect_record_loop_mask (loop_vinfo, masks, nvectors,
>> +                                    vectype_out, scalar_mask);
>> +           }
>>         }
>>        return true;
>>      }
>> @@ -3468,12 +3488,17 @@ vectorizable_call (vec_info *vinfo,
>>    vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
>>
>>    bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
>> +  unsigned int vect_nargs = nargs;
>> +  if (masked_loop_p && reduc_idx >= 0)
>> +    {
>> +      ifn = cond_fn;
>> +      vect_nargs += 2;
>> +    }
>>
>>    if (modifier == NONE || ifn != IFN_LAST)
>>      {
>>        tree prev_res = NULL_TREE;
>> -      vargs.safe_grow (nargs, true);
>> -      orig_vargs.safe_grow (nargs, true);
>> +      vargs.safe_grow (vect_nargs, true);
>>        auto_vec<vec<tree> > vec_defs (nargs);
>>        for (j = 0; j < ncopies; ++j)
>>         {
>> @@ -3488,12 +3513,23 @@ vectorizable_call (vec_info *vinfo,
>>               /* Arguments are ready.  Create the new vector stmt.  */
>>               FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
>>                 {
>> +                 int varg = 0;
>> +                 if (masked_loop_p && reduc_idx >= 0)
>> +                   {
>> +                     unsigned int vec_num = vec_oprnds0.length ();
>> +                     /* Always true for SLP.  */
>> +                     gcc_assert (ncopies == 1);
>> +                     vargs[varg++] = vect_get_loop_mask (gsi, masks, vec_num,
>> +                                                         vectype_out, i);
>> +                   }
>>                   size_t k;
>>                   for (k = 0; k < nargs; k++)
>>                     {
>>                       vec<tree> vec_oprndsk = vec_defs[k];
>> -                     vargs[k] = vec_oprndsk[i];
>> +                     vargs[varg++] = vec_oprndsk[i];
>>                     }
>> +                 if (masked_loop_p && reduc_idx >= 0)
>> +                   vargs[varg++] = vargs[reduc_idx + 1];
>>                   gimple *new_stmt;
>>                   if (modifier == NARROW)
>>                     {
>> @@ -3546,6 +3582,10 @@ vectorizable_call (vec_info *vinfo,
>>               continue;
>>             }
>>
>> +         int varg = 0;
>> +         if (masked_loop_p && reduc_idx >= 0)
>> +           vargs[varg++] = vect_get_loop_mask (gsi, masks, ncopies,
>> +                                               vectype_out, j);
>>           for (i = 0; i < nargs; i++)
>>             {
>>               op = gimple_call_arg (stmt, i);
>> @@ -3556,8 +3596,10 @@ vectorizable_call (vec_info *vinfo,
>>                                                  op, &vec_defs[i],
>>                                                  vectypes[i]);
>>                 }
>> -             orig_vargs[i] = vargs[i] = vec_defs[i][j];
>> +             vargs[varg++] = vec_defs[i][j];
>>             }
>> +         if (masked_loop_p && reduc_idx >= 0)
>> +           vargs[varg++] = vargs[reduc_idx + 1];
>>
>>           if (mask_opno >= 0 && masked_loop_p)
>>             {
>> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
>> index f8f30641512..8330cd897b8 100644
>> --- a/gcc/tree-vectorizer.h
>> +++ b/gcc/tree-vectorizer.h
>> @@ -28,6 +28,7 @@ typedef class _stmt_vec_info *stmt_vec_info;
>>  #include "target.h"
>>  #include "internal-fn.h"
>>  #include "tree-ssa-operands.h"
>> +#include "gimple-match.h"
>>
>>  /* Used for naming of new temporaries.  */
>>  enum vect_var_kind {
>> @@ -1192,7 +1193,7 @@ public:
>>    enum vect_reduction_type reduc_type;
>>
>>    /* The original reduction code, to be used in the epilogue.  */
>> -  enum tree_code reduc_code;
>> +  code_helper reduc_code;
>>    /* An internal function we should use in the epilogue.  */
>>    internal_fn reduc_fn;
>>
>> @@ -2151,7 +2152,7 @@ extern tree vect_create_addr_base_for_vector_ref (vec_info *,
>>                                                   tree);
>>
>>  /* In tree-vect-loop.c.  */
>> -extern tree neutral_op_for_reduction (tree, tree_code, tree);
>> +extern tree neutral_op_for_reduction (tree, code_helper, tree);
>>  extern widest_int vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo);
>>  bool vect_rgroup_iv_might_wrap_p (loop_vec_info, rgroup_controls *);
>>  /* Used in tree-vect-loop-manip.c */
>> @@ -2160,7 +2161,7 @@ extern opt_result vect_determine_partial_vectors_and_peeling (loop_vec_info,
>>  /* Used in gimple-loop-interchange.c and tree-parloops.c.  */
>>  extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
>>                                   enum tree_code);
>> -extern bool needs_fold_left_reduction_p (tree, tree_code);
>> +extern bool needs_fold_left_reduction_p (tree, code_helper);
>>  /* Drive for loop analysis stage.  */
>>  extern opt_loop_vec_info vect_analyze_loop (class loop *, vec_info_shared *);
>>  extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
>> @@ -2178,7 +2179,7 @@ extern tree vect_get_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
>>                                unsigned int);
>>  extern gimple_seq vect_gen_len (tree, tree, tree, tree);
>>  extern stmt_vec_info info_for_reduction (vec_info *, stmt_vec_info);
>> -extern bool reduction_fn_for_scalar_code (enum tree_code, internal_fn *);
>> +extern bool reduction_fn_for_scalar_code (code_helper, internal_fn *);
>>
>>  /* Drive for loop transformation stage.  */
>>  extern class loop *vect_transform_loop (loop_vec_info, gimple *);
>> @@ -2216,6 +2217,7 @@ extern bool vectorizable_phi (vec_info *, stmt_vec_info, gimple **, slp_tree,
>>                               stmt_vector_for_cost *);
>>  extern bool vect_emulated_vector_p (tree);
>>  extern bool vect_can_vectorize_without_simd_p (tree_code);
>> +extern bool vect_can_vectorize_without_simd_p (code_helper);
>>  extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
>>                                         stmt_vector_for_cost *,
>>                                         stmt_vector_for_cost *,
>> --
>> 2.25.1
>>


More information about the Gcc-patches mailing list