Re: [mask-vec_cond, patch 1/2] Support vectorization of VEC_COND_EXPR with no embedded comparison

[Richard Biener <richard dot guenther at gmail dot com>]

On Thu, Oct 8, 2015 at 5:50 PM, Ilya Enkovich <enkovich.gnu@gmail.com> wrote:
> Hi,
>
> This patch allows COND_EXPR with no embedded comparison to be vectorized.  It's applied on top of vectorized comparison support series.  New optab vcond_mask_optab is introduced for such statements.  Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target.

Ok.

Thanks,
Richard.

> Thanks,
> Ilya
> --
> gcc/
>
> 2015-10-08  Ilya Enkovich  <enkovich.gnu@gmail.com>
>
>         * optabs-query.h (get_vcond_mask_icode): New.
>         * optabs-tree.c (expand_vec_cond_expr_p): Use
>         get_vcond_mask_icode for VEC_COND_EXPR with mask.
>         * optabs.c (expand_vec_cond_mask_expr): New.
>         (expand_vec_cond_expr): Use get_vcond_mask_icode
>         when possible.
>         * optabs.def (vcond_mask_optab): New.
>         * tree-vect-patterns.c (vect_recog_bool_pattern): Don't
>         generate redundant comparison for COND_EXPR.
>         * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME
>         as a condition.
>         (vectorizable_condition): Likewise.
>
>
> diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h
> index 162d2e9..48bcf7c 100644
> --- a/gcc/optabs-query.h
> +++ b/gcc/optabs-query.h
> @@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns)
>    return icode;
>  }
>
> +/* Return insn code for a conditional operator with a mask mode
> +   MMODE resulting in a value of mode VMODE.  */
> +
> +static inline enum insn_code
> +get_vcond_mask_icode (machine_mode vmode, machine_mode mmode)
> +{
> +  return convert_optab_handler (vcond_mask_optab, vmode, mmode);
> +}
> +
>  /* Enumerates the possible extraction_insn operations.  */
>  enum extraction_pattern { EP_insv, EP_extv, EP_extzv };
>
> diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c
> index aa863cf..d887619 100644
> --- a/gcc/optabs-tree.c
> +++ b/gcc/optabs-tree.c
> @@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
>  {
>    machine_mode value_mode = TYPE_MODE (value_type);
>    machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
> +  if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type))
> +    return get_vcond_mask_icode (TYPE_MODE (value_type),
> +                                TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing;
>    if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
>        || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
>        || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
> diff --git a/gcc/optabs.c b/gcc/optabs.c
> index ca1a6e7..d26b8f8 100644
> --- a/gcc/optabs.c
> +++ b/gcc/optabs.c
> @@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
>    return tmp;
>  }
>
> +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
> +   three operands.  */
> +
> +rtx
> +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
> +                          rtx target)
> +{
> +  struct expand_operand ops[4];
> +  machine_mode mode = TYPE_MODE (vec_cond_type);
> +  machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
> +  enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
> +  rtx mask, rtx_op1, rtx_op2;
> +
> +  if (icode == CODE_FOR_nothing)
> +    return 0;
> +
> +  mask = expand_normal (op0);
> +  rtx_op1 = expand_normal (op1);
> +  rtx_op2 = expand_normal (op2);
> +
> +  mask = force_reg (GET_MODE (mask), mask);
> +  rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
> +
> +  create_output_operand (&ops[0], target, mode);
> +  create_input_operand (&ops[1], rtx_op1, mode);
> +  create_input_operand (&ops[2], rtx_op2, mode);
> +  create_input_operand (&ops[3], mask, mask_mode);
> +  expand_insn (icode, 4, ops);
> +
> +  return ops[0].value;
> +}
> +
>  /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
>     three operands.  */
>
> @@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
>      }
>    else
>      {
> -      /* Fake op0 < 0.  */
>        gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
> -      op0a = op0;
> -      op0b = build_zero_cst (TREE_TYPE (op0));
> -      tcode = LT_EXPR;
> -      unsignedp = false;
> +      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
> +         != CODE_FOR_nothing)
> +       return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
> +                                         op2, target);
> +      /* Fake op0 < 0.  */
> +      else
> +       {
> +         gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
> +                     == MODE_VECTOR_INT);
> +         op0a = op0;
> +         op0b = build_zero_cst (TREE_TYPE (op0));
> +         tcode = LT_EXPR;
> +         unsignedp = false;
> +       }
>      }
>    cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
>
> diff --git a/gcc/optabs.def b/gcc/optabs.def
> index 9804378..70530a6 100644
> --- a/gcc/optabs.def
> +++ b/gcc/optabs.def
> @@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b")
>  OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b")
>  OPTAB_CD(vcond_optab, "vcond$a$b")
>  OPTAB_CD(vcondu_optab, "vcondu$a$b")
> +OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
>  OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
>  OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
>  OPTAB_CD(maskload_optab, "maskload$a$b")
> diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
> index e3be3d1..6269b69 100644
> --- a/gcc/tree-vect-patterns.c
> +++ b/gcc/tree-vect-patterns.c
> @@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
>        else
>         {
>           tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
> -         tree cst0, cst1, cmp, tmp;
> +         tree cst0, cst1, tmp;
>
>           if (!type)
>             return NULL;
> @@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
>           cst0 = build_int_cst (type, 0);
>           cst1 = build_int_cst (type, 1);
>           tmp = vect_recog_temp_ssa_var (type, NULL);
> -         cmp = build2 (NE_EXPR, boolean_type_node,
> -                       var, build_int_cst (TREE_TYPE (var), 0));
> -         pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0);
> +         pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
>
>           if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
>             {
> @@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
>        if (get_vectype_for_scalar_type (type) == NULL_TREE)
>         return NULL;
>
> -      if (check_bool_pattern (var, loop_vinfo, bb_vinfo))
> -       {
> -         rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
> -         rhs = build2 (NE_EXPR, boolean_type_node,
> -                       rhs, build_int_cst (type, 0));
> -       }
> -      else
> -       rhs = build2 (NE_EXPR, boolean_type_node,
> -                     var, build_int_cst (TREE_TYPE (var), 0)),
> +      if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
> +       return NULL;
> +
> +      rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
>
>        lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
>        pattern_stmt
> -         = gimple_build_assign (lhs, COND_EXPR, rhs,
> +         = gimple_build_assign (lhs, COND_EXPR,
> +                                build2 (NE_EXPR, boolean_type_node,
> +                                        rhs, build_int_cst (type, 0)),
>                                  gimple_assign_rhs2 (last_stmt),
>                                  gimple_assign_rhs3 (last_stmt));
>        *type_out = vectype;
> @@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
>        else
>         {
>           tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
> -         tree cst0, cst1, cmp, new_vectype;
> +         tree cst0, cst1, new_vectype;
>
>           if (!type)
>             return NULL;
> @@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
>           new_vectype = get_vectype_for_scalar_type (type);
>
>           rhs = vect_recog_temp_ssa_var (type, NULL);
> -         cmp = build2 (NE_EXPR, boolean_type_node,
> -                       var, build_int_cst (TREE_TYPE (var), 0));
> -         pattern_stmt = gimple_build_assign (rhs, COND_EXPR,
> -                                             cmp, cst1, cst0);
> +         pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
>
>           pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
>                                                  bb_vinfo);
> diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
> index d6819ec..b139601 100644
> --- a/gcc/tree-vect-stmts.c
> +++ b/gcc/tree-vect-stmts.c
> @@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo,
>    enum vect_def_type dt;
>    tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
>
> +  /* Mask case.  */
> +  if (TREE_CODE (cond) == SSA_NAME
> +      && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE)
> +    {
> +      gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
> +      if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo,
> +                                &lhs_def_stmt, &def, &dt, comp_vectype)
> +         || !*comp_vectype
> +         || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
> +       return false;
> +      return true;
> +    }
> +
>    if (!COMPARISON_CLASS_P (cond))
>      return false;
>
> @@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>    vec<tree> vec_oprnds2 = vNULL;
>    vec<tree> vec_oprnds3 = vNULL;
>    tree vec_cmp_type;
> +  bool masked = false;
>
>    if (reduc_index && STMT_SLP_TYPE (stmt_info))
>      return false;
> @@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>            && TREE_CODE (else_clause) != FIXED_CST)
>      return false;
>
> -  vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
> +  if (VECTOR_BOOLEAN_TYPE_P (comp_vectype))
> +    {
> +      vec_cmp_type = comp_vectype;
> +      masked = true;
> +    }
> +  else
> +    vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
>    if (vec_cmp_type == NULL_TREE)
>      return false;
>
> @@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>                auto_vec<tree, 4> ops;
>               auto_vec<vec<tree>, 4> vec_defs;
>
> -              ops.safe_push (TREE_OPERAND (cond_expr, 0));
> -              ops.safe_push (TREE_OPERAND (cond_expr, 1));
> +             if (masked)
> +                 ops.safe_push (cond_expr);
> +             else
> +               {
> +                 ops.safe_push (TREE_OPERAND (cond_expr, 0));
> +                 ops.safe_push (TREE_OPERAND (cond_expr, 1));
> +               }
>                ops.safe_push (then_clause);
>                ops.safe_push (else_clause);
>                vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
>               vec_oprnds3 = vec_defs.pop ();
>               vec_oprnds2 = vec_defs.pop ();
> -             vec_oprnds1 = vec_defs.pop ();
> +             if (!masked)
> +               vec_oprnds1 = vec_defs.pop ();
>               vec_oprnds0 = vec_defs.pop ();
>
>                ops.release ();
> @@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>            else
>              {
>               gimple *gtemp;
> -             vec_cond_lhs =
> -             vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
> -                                           stmt, NULL, comp_vectype);
> -             vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
> -                                 loop_vinfo, NULL, &gtemp, &def, &dts[0]);
> -
> -             vec_cond_rhs =
> -               vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
> -                                             stmt, NULL, comp_vectype);
> -             vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
> -                                 loop_vinfo, NULL, &gtemp, &def, &dts[1]);
> +             if (masked)
> +               {
> +                 vec_cond_lhs
> +                   = vect_get_vec_def_for_operand (cond_expr, stmt,
> +                                                   NULL, comp_vectype);
> +                 vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL,
> +                                     &gtemp, &def, &dts[0]);
> +               }
> +             else
> +               {
> +                 vec_cond_lhs =
> +                   vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
> +                                                 stmt, NULL, comp_vectype);
> +                 vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
> +                                     loop_vinfo, NULL, &gtemp, &def, &dts[0]);
> +
> +                 vec_cond_rhs =
> +                   vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
> +                                                 stmt, NULL, comp_vectype);
> +                 vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
> +                                     loop_vinfo, NULL, &gtemp, &def, &dts[1]);
> +               }
>               if (reduc_index == 1)
>                 vec_then_clause = reduc_def;
>               else
> @@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>         }
>        else
>         {
> -         vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0],
> -                                                        vec_oprnds0.pop ());
> -         vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1],
> -                                                        vec_oprnds1.pop ());
> +         vec_cond_lhs
> +           = vect_get_vec_def_for_stmt_copy (dts[0],
> +                                             vec_oprnds0.pop ());
> +         if (!masked)
> +           vec_cond_rhs
> +             = vect_get_vec_def_for_stmt_copy (dts[1],
> +                                               vec_oprnds1.pop ());
> +
>           vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
>                                                             vec_oprnds2.pop ());
>           vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
> @@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>        if (!slp_node)
>          {
>           vec_oprnds0.quick_push (vec_cond_lhs);
> -         vec_oprnds1.quick_push (vec_cond_rhs);
> +         if (!masked)
> +           vec_oprnds1.quick_push (vec_cond_rhs);
>           vec_oprnds2.quick_push (vec_then_clause);
>           vec_oprnds3.quick_push (vec_else_clause);
>         }
> @@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
>        /* Arguments are ready.  Create the new vector stmt.  */
>        FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
>          {
> -          vec_cond_rhs = vec_oprnds1[i];
>            vec_then_clause = vec_oprnds2[i];
>            vec_else_clause = vec_oprnds3[i];
>
> -         vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
> -                               vec_cond_lhs, vec_cond_rhs);
> +         if (masked)
> +           vec_compare = vec_cond_lhs;
> +         else
> +           {
> +             vec_cond_rhs = vec_oprnds1[i];
> +             vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
> +                                   vec_cond_lhs, vec_cond_rhs);
> +           }
>            vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
>                          vec_compare, vec_then_clause, vec_else_clause);
>