Re: [17/46] Make LOOP_VINFO_REDUCTIONS an auto_vec<stmt_vec_info>

[Richard Biener <richard dot guenther at gmail dot com>]

On Tue, Jul 24, 2018 at 11:59 AM Richard Sandiford
<richard.sandiford@arm.com> wrote:
>
> This patch changes LOOP_VINFO_REDUCTIONS from an auto_vec<gimple *>
> to an auto_vec<stmt_vec_info>.  It also changes the associated
> vect_force_simple_reduction so that it takes and returns stmt_vec_infos
> instead of gimple stmts.

OK.

Highlights that reduction detection needs refactoring to be usable outside
of the vectorizer (see tree-parloops.c).  Exposing vinfos doesn't make the
situation better here...

Richard.

>
> 2018-07-24  Richard Sandiford  <richard.sandiford@arm.com>
>
> gcc/
>         * tree-vectorizer.h (_loop_vec_info::reductions): Change from an
>         auto_vec<gimple *> to an auto_vec<stmt_vec_info>.
>         (vect_force_simple_reduction): Take and return stmt_vec_infos rather
>         than gimple stmts.
>         * tree-parloops.c (valid_reduction_p): Take a stmt_vec_info instead
>         of a gimple stmt.
>         (gather_scalar_reductions): Update after above interface changes.
>         * tree-vect-loop.c (vect_analyze_scalar_cycles_1): Likewise.
>         (vect_is_simple_reduction): Take and return stmt_vec_infos rather
>         than gimple stmts.
>         (vect_force_simple_reduction): Likewise.
>         * tree-vect-patterns.c (vect_pattern_recog_1): Update use of
>         LOOP_VINFO_REDUCTIONS.
>         * tree-vect-slp.c (vect_analyze_slp_instance): Likewise.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h       2018-07-24 10:22:53.909100298 +0100
> +++ gcc/tree-vectorizer.h       2018-07-24 10:22:57.277070390 +0100
> @@ -475,7 +475,7 @@ typedef struct _loop_vec_info : public v
>    auto_vec<gimple *> may_misalign_stmts;
>
>    /* Reduction cycles detected in the loop. Used in loop-aware SLP.  */
> -  auto_vec<gimple *> reductions;
> +  auto_vec<stmt_vec_info> reductions;
>
>    /* All reduction chains in the loop, represented by the first
>       stmt in the chain.  */
> @@ -1627,8 +1627,8 @@ extern tree vect_create_addr_base_for_ve
>
>  /* In tree-vect-loop.c.  */
>  /* FORNOW: Used in tree-parloops.c.  */
> -extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *,
> -                                           bool *, bool);
> +extern stmt_vec_info vect_force_simple_reduction (loop_vec_info, stmt_vec_info,
> +                                                 bool *, bool);
>  /* Used in gimple-loop-interchange.c.  */
>  extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
>                                   enum tree_code);
> Index: gcc/tree-parloops.c
> ===================================================================
> --- gcc/tree-parloops.c 2018-06-27 10:27:09.778650686 +0100
> +++ gcc/tree-parloops.c 2018-07-24 10:22:57.273070426 +0100
> @@ -2570,15 +2570,14 @@ set_reduc_phi_uids (reduction_info **slo
>    return 1;
>  }
>
> -/* Return true if the type of reduction performed by STMT is suitable
> +/* Return true if the type of reduction performed by STMT_INFO is suitable
>     for this pass.  */
>
>  static bool
> -valid_reduction_p (gimple *stmt)
> +valid_reduction_p (stmt_vec_info stmt_info)
>  {
>    /* Parallelization would reassociate the operation, which isn't
>       allowed for in-order reductions.  */
> -  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
>    vect_reduction_type reduc_type = STMT_VINFO_REDUC_TYPE (stmt_info);
>    return reduc_type != FOLD_LEFT_REDUCTION;
>  }
> @@ -2615,10 +2614,11 @@ gather_scalar_reductions (loop_p loop, r
>        if (simple_iv (loop, loop, res, &iv, true))
>         continue;
>
> -      gimple *reduc_stmt
> -       = vect_force_simple_reduction (simple_loop_info, phi,
> +      stmt_vec_info reduc_stmt_info
> +       = vect_force_simple_reduction (simple_loop_info,
> +                                      simple_loop_info->lookup_stmt (phi),
>                                        &double_reduc, true);
> -      if (!reduc_stmt || !valid_reduction_p (reduc_stmt))
> +      if (!reduc_stmt_info || !valid_reduction_p (reduc_stmt_info))
>         continue;
>
>        if (double_reduc)
> @@ -2627,11 +2627,11 @@ gather_scalar_reductions (loop_p loop, r
>             continue;
>
>           double_reduc_phis.safe_push (phi);
> -         double_reduc_stmts.safe_push (reduc_stmt);
> +         double_reduc_stmts.safe_push (reduc_stmt_info->stmt);
>           continue;
>         }
>
> -      build_new_reduction (reduction_list, reduc_stmt, phi);
> +      build_new_reduction (reduction_list, reduc_stmt_info->stmt, phi);
>      }
>    delete simple_loop_info;
>
> @@ -2661,12 +2661,15 @@ gather_scalar_reductions (loop_p loop, r
>                              &iv, true))
>                 continue;
>
> -             gimple *inner_reduc_stmt
> -               = vect_force_simple_reduction (simple_loop_info, inner_phi,
> +             stmt_vec_info inner_phi_info
> +               = simple_loop_info->lookup_stmt (inner_phi);
> +             stmt_vec_info inner_reduc_stmt_info
> +               = vect_force_simple_reduction (simple_loop_info,
> +                                              inner_phi_info,
>                                                &double_reduc, true);
>               gcc_assert (!double_reduc);
> -             if (inner_reduc_stmt == NULL
> -                 || !valid_reduction_p (inner_reduc_stmt))
> +             if (!inner_reduc_stmt_info
> +                 || !valid_reduction_p (inner_reduc_stmt_info))
>                 continue;
>
>               build_new_reduction (reduction_list, double_reduc_stmts[i], phi);
> Index: gcc/tree-vect-loop.c
> ===================================================================
> --- gcc/tree-vect-loop.c        2018-07-24 10:22:53.909100298 +0100
> +++ gcc/tree-vect-loop.c        2018-07-24 10:22:57.273070426 +0100
> @@ -546,7 +546,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
>        gimple *phi = worklist.pop ();
>        tree def = PHI_RESULT (phi);
>        stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
> -      gimple *reduc_stmt;
>
>        if (dump_enabled_p ())
>          {
> @@ -557,9 +556,10 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
>        gcc_assert (!virtual_operand_p (def)
>                   && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
>
> -      reduc_stmt = vect_force_simple_reduction (loop_vinfo, phi,
> -                                               &double_reduc, false);
> -      if (reduc_stmt)
> +      stmt_vec_info reduc_stmt_info
> +       = vect_force_simple_reduction (loop_vinfo, stmt_vinfo,
> +                                      &double_reduc, false);
> +      if (reduc_stmt_info)
>          {
>            if (double_reduc)
>              {
> @@ -568,8 +568,8 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
>                                  "Detected double reduction.\n");
>
>                STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
> -              STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
> -                                                    vect_double_reduction_def;
> +             STMT_VINFO_DEF_TYPE (reduc_stmt_info)
> +               = vect_double_reduction_def;
>              }
>            else
>              {
> @@ -580,8 +580,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
>                                      "Detected vectorizable nested cycle.\n");
>
>                    STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
> -                  STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
> -                                                             vect_nested_cycle;
> +                 STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_nested_cycle;
>                  }
>                else
>                  {
> @@ -590,13 +589,13 @@ vect_analyze_scalar_cycles_1 (loop_vec_i
>                                      "Detected reduction.\n");
>
>                    STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
> -                  STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
> -                                                           vect_reduction_def;
> +                 STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def;
>                    /* Store the reduction cycles for possible vectorization in
>                       loop-aware SLP if it was not detected as reduction
>                      chain.  */
> -                 if (! REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (reduc_stmt)))
> -                   LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt);
> +                 if (! REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info))
> +                   LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push
> +                     (reduc_stmt_info);
>                  }
>              }
>          }
> @@ -2530,8 +2529,8 @@ vect_is_slp_reduction (loop_vec_info loo
>    struct loop *loop = (gimple_bb (phi))->loop_father;
>    struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
>    enum tree_code code;
> -  gimple *current_stmt = NULL, *loop_use_stmt = NULL, *first, *next_stmt;
> -  stmt_vec_info use_stmt_info, current_stmt_info;
> +  gimple *loop_use_stmt = NULL, *first, *next_stmt;
> +  stmt_vec_info use_stmt_info, current_stmt_info = NULL;
>    tree lhs;
>    imm_use_iterator imm_iter;
>    use_operand_p use_p;
> @@ -2593,9 +2592,8 @@ vect_is_slp_reduction (loop_vec_info loo
>
>        /* Insert USE_STMT into reduction chain.  */
>        use_stmt_info = loop_info->lookup_stmt (loop_use_stmt);
> -      if (current_stmt)
> +      if (current_stmt_info)
>          {
> -          current_stmt_info = vinfo_for_stmt (current_stmt);
>           REDUC_GROUP_NEXT_ELEMENT (current_stmt_info) = loop_use_stmt;
>            REDUC_GROUP_FIRST_ELEMENT (use_stmt_info)
>              = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
> @@ -2604,7 +2602,7 @@ vect_is_slp_reduction (loop_vec_info loo
>         REDUC_GROUP_FIRST_ELEMENT (use_stmt_info) = loop_use_stmt;
>
>        lhs = gimple_assign_lhs (loop_use_stmt);
> -      current_stmt = loop_use_stmt;
> +      current_stmt_info = use_stmt_info;
>        size++;
>     }
>
> @@ -2614,7 +2612,7 @@ vect_is_slp_reduction (loop_vec_info loo
>    /* Swap the operands, if needed, to make the reduction operand be the second
>       operand.  */
>    lhs = PHI_RESULT (phi);
> -  next_stmt = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt));
> +  next_stmt = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
>    while (next_stmt)
>      {
>        if (gimple_assign_rhs2 (next_stmt) == lhs)
> @@ -2671,7 +2669,7 @@ vect_is_slp_reduction (loop_vec_info loo
>      }
>
>    /* Save the chain for further analysis in SLP detection.  */
> -  first = REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt));
> +  first = REDUC_GROUP_FIRST_ELEMENT (current_stmt_info);
>    LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (first);
>    REDUC_GROUP_SIZE (vinfo_for_stmt (first)) = size;
>
> @@ -2867,15 +2865,16 @@ check_reduction_path (dump_user_location
>
>  */
>
> -static gimple *
> -vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi,
> +static stmt_vec_info
> +vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
>                           bool *double_reduc,
>                           bool need_wrapping_integral_overflow,
>                           enum vect_reduction_type *v_reduc_type)
>  {
> +  gphi *phi = as_a <gphi *> (phi_info->stmt);
>    struct loop *loop = (gimple_bb (phi))->loop_father;
>    struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
> -  gimple *def_stmt, *phi_use_stmt = NULL;
> +  gimple *phi_use_stmt = NULL;
>    enum tree_code orig_code, code;
>    tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE;
>    tree type;
> @@ -2937,13 +2936,16 @@ vect_is_simple_reduction (loop_vec_info
>        return NULL;
>      }
>
> -  def_stmt = SSA_NAME_DEF_STMT (loop_arg);
> -  if (is_gimple_assign (def_stmt))
> +  stmt_vec_info def_stmt_info = loop_info->lookup_def (loop_arg);
> +  if (!def_stmt_info)
> +    return NULL;
> +
> +  if (gassign *def_stmt = dyn_cast <gassign *> (def_stmt_info->stmt))
>      {
>        name = gimple_assign_lhs (def_stmt);
>        phi_def = false;
>      }
> -  else if (gimple_code (def_stmt) == GIMPLE_PHI)
> +  else if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt))
>      {
>        name = PHI_RESULT (def_stmt);
>        phi_def = true;
> @@ -2954,14 +2956,12 @@ vect_is_simple_reduction (loop_vec_info
>         {
>           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
>                            "reduction: unhandled reduction operation: ");
> -         dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, def_stmt, 0);
> +         dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
> +                           def_stmt_info->stmt, 0);
>         }
>        return NULL;
>      }
>
> -  if (! flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
> -    return NULL;
> -
>    nloop_uses = 0;
>    auto_vec<gphi *, 3> lcphis;
>    FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
> @@ -2987,6 +2987,7 @@ vect_is_simple_reduction (loop_vec_info
>       defined in the inner loop.  */
>    if (phi_def)
>      {
> +      gphi *def_stmt = as_a <gphi *> (def_stmt_info->stmt);
>        op1 = PHI_ARG_DEF (def_stmt, 0);
>
>        if (gimple_phi_num_args (def_stmt) != 1
> @@ -3012,7 +3013,7 @@ vect_is_simple_reduction (loop_vec_info
>                             "detected double reduction: ");
>
>            *double_reduc = true;
> -          return def_stmt;
> +         return def_stmt_info;
>          }
>
>        return NULL;
> @@ -3038,6 +3039,7 @@ vect_is_simple_reduction (loop_vec_info
>           }
>      }
>
> +  gassign *def_stmt = as_a <gassign *> (def_stmt_info->stmt);
>    bool nested_in_vect_loop = flow_loop_nested_p (vect_loop, loop);
>    code = orig_code = gimple_assign_rhs_code (def_stmt);
>
> @@ -3178,7 +3180,7 @@ vect_is_simple_reduction (loop_vec_info
>      {
>        if (dump_enabled_p ())
>         report_vect_op (MSG_NOTE, def_stmt, "detected reduction: ");
> -      return def_stmt;
> +      return def_stmt_info;
>      }
>
>    if (def1_info
> @@ -3237,7 +3239,7 @@ vect_is_simple_reduction (loop_vec_info
>              report_vect_op (MSG_NOTE, def_stmt, "detected reduction: ");
>          }
>
> -      return def_stmt;
> +      return def_stmt_info;
>      }
>
>    /* Try to find SLP reduction chain.  */
> @@ -3250,7 +3252,7 @@ vect_is_simple_reduction (loop_vec_info
>          report_vect_op (MSG_NOTE, def_stmt,
>                         "reduction: detected reduction chain: ");
>
> -      return def_stmt;
> +      return def_stmt_info;
>      }
>
>    /* Dissolve group eventually half-built by vect_is_slp_reduction.  */
> @@ -3264,9 +3266,8 @@ vect_is_simple_reduction (loop_vec_info
>      }
>
>    /* Look for the expression computing loop_arg from loop PHI result.  */
> -  if (check_reduction_path (vect_location, loop, as_a <gphi *> (phi), loop_arg,
> -                           code))
> -    return def_stmt;
> +  if (check_reduction_path (vect_location, loop, phi, loop_arg, code))
> +    return def_stmt_info;
>
>    if (dump_enabled_p ())
>      {
> @@ -3281,25 +3282,24 @@ vect_is_simple_reduction (loop_vec_info
>     in-place if it enables detection of more reductions.  Arguments
>     as there.  */
>
> -gimple *
> -vect_force_simple_reduction (loop_vec_info loop_info, gimple *phi,
> +stmt_vec_info
> +vect_force_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
>                              bool *double_reduc,
>                              bool need_wrapping_integral_overflow)
>  {
>    enum vect_reduction_type v_reduc_type;
> -  gimple *def = vect_is_simple_reduction (loop_info, phi, double_reduc,
> -                                         need_wrapping_integral_overflow,
> -                                         &v_reduc_type);
> -  if (def)
> +  stmt_vec_info def_info
> +    = vect_is_simple_reduction (loop_info, phi_info, double_reduc,
> +                               need_wrapping_integral_overflow,
> +                               &v_reduc_type);
> +  if (def_info)
>      {
> -      stmt_vec_info phi_info = vinfo_for_stmt (phi);
> -      stmt_vec_info def_info = vinfo_for_stmt (def);
>        STMT_VINFO_REDUC_TYPE (phi_info) = v_reduc_type;
>        STMT_VINFO_REDUC_DEF (phi_info) = def_info;
>        STMT_VINFO_REDUC_TYPE (def_info) = v_reduc_type;
>        STMT_VINFO_REDUC_DEF (def_info) = phi_info;
>      }
> -  return def;
> +  return def_info;
>  }
>
>  /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times.  */
> Index: gcc/tree-vect-patterns.c
> ===================================================================
> --- gcc/tree-vect-patterns.c    2018-07-24 10:22:44.289185723 +0100
> +++ gcc/tree-vect-patterns.c    2018-07-24 10:22:57.277070390 +0100
> @@ -4851,9 +4851,9 @@ vect_pattern_recog_1 (vect_recog_func *r
>    if (loop_vinfo)
>      {
>        unsigned ix, ix2;
> -      gimple **elem_ptr;
> +      stmt_vec_info *elem_ptr;
>        VEC_ORDERED_REMOVE_IF (LOOP_VINFO_REDUCTIONS (loop_vinfo), ix, ix2,
> -                            elem_ptr, *elem_ptr == stmt);
> +                            elem_ptr, *elem_ptr == stmt_info);
>      }
>  }
>
> Index: gcc/tree-vect-slp.c
> ===================================================================
> --- gcc/tree-vect-slp.c 2018-07-24 10:22:50.777128110 +0100
> +++ gcc/tree-vect-slp.c 2018-07-24 10:22:57.277070390 +0100
> @@ -1931,6 +1931,7 @@ vect_analyze_slp_instance (vec_info *vin
>    unsigned int group_size;
>    tree vectype, scalar_type = NULL_TREE;
>    gimple *next;
> +  stmt_vec_info next_info;
>    unsigned int i;
>    vec<slp_tree> loads;
>    struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
> @@ -2008,9 +2009,9 @@ vect_analyze_slp_instance (vec_info *vin
>    else
>      {
>        /* Collect reduction statements.  */
> -      vec<gimple *> reductions = as_a <loop_vec_info> (vinfo)->reductions;
> -      for (i = 0; reductions.iterate (i, &next); i++)
> -       scalar_stmts.safe_push (next);
> +      vec<stmt_vec_info> reductions = as_a <loop_vec_info> (vinfo)->reductions;
> +      for (i = 0; reductions.iterate (i, &next_info); i++)
> +       scalar_stmts.safe_push (next_info);
>      }
>
>    loads.create (group_size);