This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Re: [6/7] Explicitly classify vector loads and stores


On Wed, Jun 15, 2016 at 10:52 AM, Richard Sandiford
<richard.sandiford@arm.com> wrote:
> This is the main patch in the series.  It adds a new enum and routines
> for classifying a vector load or store implementation.
>
> Tested on aarch64-linux-gnu and x86_64-linux-gnu.  OK to install?

Why's the setting and checking of the memory access type conditional on !slp?
I'd rather avoid doing this :/

Otherwise it looks like a step in the right direction of splitting the
vectorizable_*
functions into a analysis part that records all decisions made and a transform
part that just applies it.

Thanks,
Richard.

> Thanks,
> Richard
>
>
> gcc/
>         * tree-vectorizer.h (vect_memory_access_type): New enum.
>         (_stmt_vec_info): Add a memory_access_type field.
>         (STMT_VINFO_MEMORY_ACCESS_TYPE): New macro.
>         (vect_model_store_cost): Take an access type instead of a boolean.
>         (vect_model_load_cost): Likewise.
>         * tree-vect-slp.c (vect_analyze_slp_cost_1): Update calls to
>         vect_model_store_cost and vect_model_load_cost.
>         * tree-vect-stmts.c (vec_load_store_type): New enum.
>         (vect_model_store_cost): Take an access type instead of a
>         store_lanes_p boolean.  Simplify tests.
>         (vect_model_load_cost): Likewise, but for load_lanes_p.
>         (get_group_load_store_type, get_load_store_type): New functions.
>         (vectorizable_store): Use get_load_store_type.  Record the access
>         type in STMT_VINFO_MEMORY_ACCESS_TYPE.
>         (vectorizable_load): Likewise.
>         (vectorizable_mask_load_store): Likewise.  Replace is_store
>         variable with vls_type.
>
> Index: gcc/tree-vectorizer.h
> ===================================================================
> --- gcc/tree-vectorizer.h
> +++ gcc/tree-vectorizer.h
> @@ -485,6 +485,33 @@ enum slp_vect_type {
>    hybrid
>  };
>
> +/* Describes how we're going to vectorize an individual load or store,
> +   or a group of loads or stores.  */
> +enum vect_memory_access_type {
> +  /* A simple contiguous access.  */
> +  VMAT_CONTIGUOUS,
> +
> +  /* A simple contiguous access in which the elements need to be permuted
> +     after loading or before storing.  Only used for loop vectorization;
> +     SLP uses separate permutes.  */
> +  VMAT_CONTIGUOUS_PERMUTE,
> +
> +  /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES.  */
> +  VMAT_LOAD_STORE_LANES,
> +
> +  /* An access in which each scalar element is loaded or stored
> +     individually.  */
> +  VMAT_ELEMENTWISE,
> +
> +  /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
> +     SLP accesses.  Each unrolled iteration uses a contiguous load
> +     or store for the whole group, but the groups from separate iterations
> +     are combined in the same way as for VMAT_ELEMENTWISE.  */
> +  VMAT_STRIDED_SLP,
> +
> +  /* The access uses gather loads or scatter stores.  */
> +  VMAT_GATHER_SCATTER
> +};
>
>  typedef struct data_reference *dr_p;
>
> @@ -602,6 +629,10 @@ typedef struct _stmt_vec_info {
>    /* True if this is an access with loop-invariant stride.  */
>    bool strided_p;
>
> +  /* Classifies how the load or store is going to be implemented
> +     for loop vectorization.  */
> +  vect_memory_access_type memory_access_type;
> +
>    /* For both loads and stores.  */
>    bool simd_lane_access_p;
>
> @@ -659,6 +690,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
>  #define STMT_VINFO_DATA_REF(S)             (S)->data_ref_info
>  #define STMT_VINFO_GATHER_SCATTER_P(S)    (S)->gather_scatter_p
>  #define STMT_VINFO_STRIDED_P(S)                   (S)->strided_p
> +#define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
>  #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
>  #define STMT_VINFO_VEC_REDUCTION_TYPE(S)   (S)->v_reduc_type
>
> @@ -1006,12 +1038,12 @@ extern void free_stmt_vec_info (gimple *stmt);
>  extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
>                                      stmt_vector_for_cost *,
>                                     stmt_vector_for_cost *);
> -extern void vect_model_store_cost (stmt_vec_info, int, bool,
> +extern void vect_model_store_cost (stmt_vec_info, int, vect_memory_access_type,
>                                    enum vect_def_type, slp_tree,
>                                    stmt_vector_for_cost *,
>                                    stmt_vector_for_cost *);
> -extern void vect_model_load_cost (stmt_vec_info, int, bool, slp_tree,
> -                                 stmt_vector_for_cost *,
> +extern void vect_model_load_cost (stmt_vec_info, int, vect_memory_access_type,
> +                                 slp_tree, stmt_vector_for_cost *,
>                                   stmt_vector_for_cost *);
>  extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
>                                   enum vect_cost_for_stmt, stmt_vec_info,
> Index: gcc/tree-vect-slp.c
> ===================================================================
> --- gcc/tree-vect-slp.c
> +++ gcc/tree-vect-slp.c
> @@ -1490,9 +1490,13 @@ vect_analyze_slp_cost_1 (slp_instance instance, slp_tree node,
>    stmt_info = vinfo_for_stmt (stmt);
>    if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
>      {
> +      vect_memory_access_type memory_access_type
> +       = (STMT_VINFO_STRIDED_P (stmt_info)
> +          ? VMAT_STRIDED_SLP
> +          : VMAT_CONTIGUOUS);
>        if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
> -       vect_model_store_cost (stmt_info, ncopies_for_cost, false,
> -                              vect_uninitialized_def,
> +       vect_model_store_cost (stmt_info, ncopies_for_cost,
> +                              memory_access_type, vect_uninitialized_def,
>                                node, prologue_cost_vec, body_cost_vec);
>        else
>         {
> @@ -1515,8 +1519,9 @@ vect_analyze_slp_cost_1 (slp_instance instance, slp_tree node,
>               ncopies_for_cost *= SLP_INSTANCE_UNROLLING_FACTOR (instance);
>             }
>           /* Record the cost for the vector loads.  */
> -         vect_model_load_cost (stmt_info, ncopies_for_cost, false,
> -                               node, prologue_cost_vec, body_cost_vec);
> +         vect_model_load_cost (stmt_info, ncopies_for_cost,
> +                               memory_access_type, node, prologue_cost_vec,
> +                               body_cost_vec);
>           return;
>         }
>      }
> Index: gcc/tree-vect-stmts.c
> ===================================================================
> --- gcc/tree-vect-stmts.c
> +++ gcc/tree-vect-stmts.c
> @@ -52,6 +52,14 @@ along with GCC; see the file COPYING3.  If not see
>  /* For lang_hooks.types.type_for_mode.  */
>  #include "langhooks.h"
>
> +/* Says whether a statement is a load, a store of a vectorized statement
> +   result, or a store of an invariant value.  */
> +enum vec_load_store_type {
> +  VLS_LOAD,
> +  VLS_STORE,
> +  VLS_STORE_INVARIANT
> +};
> +
>  /* Return the vectorized type for the given statement.  */
>
>  tree
> @@ -873,8 +881,8 @@ vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
>
>  void
>  vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
> -                      bool store_lanes_p, enum vect_def_type dt,
> -                      slp_tree slp_node,
> +                      vect_memory_access_type memory_access_type,
> +                      enum vect_def_type dt, slp_tree slp_node,
>                        stmt_vector_for_cost *prologue_cost_vec,
>                        stmt_vector_for_cost *body_cost_vec)
>  {
> @@ -903,14 +911,9 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
>    /* We assume that the cost of a single store-lanes instruction is
>       equivalent to the cost of GROUP_SIZE separate stores.  If a grouped
>       access is instead being provided by a permute-and-store operation,
> -     include the cost of the permutes.
> -
> -     For SLP, the caller has already counted the permutation, if any.  */
> -  if (grouped_access_p
> -      && first_stmt_p
> -      && !store_lanes_p
> -      && !STMT_VINFO_STRIDED_P (stmt_info)
> -      && !slp_node)
> +     include the cost of the permutes.  */
> +  if (first_stmt_p
> +      && memory_access_type == VMAT_CONTIGUOUS_PERMUTE)
>      {
>        /* Uses a high and low interleave or shuffle operations for each
>          needed permute.  */
> @@ -927,17 +930,16 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
>
>    tree vectype = STMT_VINFO_VECTYPE (stmt_info);
>    /* Costs of the stores.  */
> -  if (STMT_VINFO_STRIDED_P (stmt_info) && !(slp_node && grouped_access_p))
> -    {
> -      /* N scalar stores plus extracting the elements.  */
> -      inside_cost += record_stmt_cost (body_cost_vec,
> -                                      ncopies * TYPE_VECTOR_SUBPARTS (vectype),
> -                                      scalar_store, stmt_info, 0, vect_body);
> -    }
> +  if (memory_access_type == VMAT_ELEMENTWISE)
> +    /* N scalar stores plus extracting the elements.  */
> +    inside_cost += record_stmt_cost (body_cost_vec,
> +                                    ncopies * TYPE_VECTOR_SUBPARTS (vectype),
> +                                    scalar_store, stmt_info, 0, vect_body);
>    else
>      vect_get_store_cost (dr, ncopies, &inside_cost, body_cost_vec);
>
> -  if (STMT_VINFO_STRIDED_P (stmt_info))
> +  if (memory_access_type == VMAT_ELEMENTWISE
> +      || memory_access_type == VMAT_STRIDED_SLP)
>      inside_cost += record_stmt_cost (body_cost_vec,
>                                      ncopies * TYPE_VECTOR_SUBPARTS (vectype),
>                                      vec_to_scalar, stmt_info, 0, vect_body);
> @@ -1011,7 +1013,8 @@ vect_get_store_cost (struct data_reference *dr, int ncopies,
>
>  void
>  vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
> -                     bool load_lanes_p, slp_tree slp_node,
> +                     vect_memory_access_type memory_access_type,
> +                     slp_tree slp_node,
>                       stmt_vector_for_cost *prologue_cost_vec,
>                       stmt_vector_for_cost *body_cost_vec)
>  {
> @@ -1036,14 +1039,9 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
>    /* We assume that the cost of a single load-lanes instruction is
>       equivalent to the cost of GROUP_SIZE separate loads.  If a grouped
>       access is instead being provided by a load-and-permute operation,
> -     include the cost of the permutes.
> -
> -     For SLP, the caller has already counted the permutation, if any.  */
> -  if (grouped_access_p
> -      && first_stmt_p
> -      && !load_lanes_p
> -      && !STMT_VINFO_STRIDED_P (stmt_info)
> -      && !slp_node)
> +     include the cost of the permutes.  */
> +  if (first_stmt_p
> +      && memory_access_type == VMAT_CONTIGUOUS_PERMUTE)
>      {
>        /* Uses an even and odd extract operations or shuffle operations
>          for each needed permute.  */
> @@ -1059,7 +1057,7 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
>      }
>
>    /* The loads themselves.  */
> -  if (STMT_VINFO_STRIDED_P (stmt_info) && !(slp_node && grouped_access_p))
> +  if (memory_access_type == VMAT_ELEMENTWISE)
>      {
>        /* N scalar loads plus gathering them into a vector.  */
>        tree vectype = STMT_VINFO_VECTYPE (stmt_info);
> @@ -1071,7 +1069,8 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
>      vect_get_load_cost (dr, ncopies, first_stmt_p,
>                         &inside_cost, &prologue_cost,
>                         prologue_cost_vec, body_cost_vec, true);
> -  if (STMT_VINFO_STRIDED_P (stmt_info))
> +  if (memory_access_type == VMAT_ELEMENTWISE
> +      || memory_access_type == VMAT_STRIDED_SLP)
>      inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_construct,
>                                      stmt_info, 0, vect_body);
>
> @@ -1674,6 +1673,209 @@ static tree permute_vec_elements (tree, tree, tree, gimple *,
>                                   gimple_stmt_iterator *);
>
>
> +/* A subroutine of get_load_store_type, with a subset of the same
> +   arguments.  Handle the case where STMT is part of a grouped load
> +   or store.
> +
> +   For stores, the statements in the group are all consecutive
> +   and there is no gap at the end.  For loads, the statements in the
> +   group might not be consecutive; there can be gaps between statements
> +   as well as at the end.  */
> +
> +static bool
> +get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
> +                          vec_load_store_type vls_type,
> +                          vect_memory_access_type *memory_access_type)
> +{
> +  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> +  vec_info *vinfo = stmt_info->vinfo;
> +  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
> +  struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
> +  gimple *first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
> +  unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
> +  bool single_element_p = (stmt == first_stmt
> +                          && !GROUP_NEXT_ELEMENT (stmt_info));
> +  unsigned HOST_WIDE_INT gap = GROUP_GAP (vinfo_for_stmt (first_stmt));
> +  int nunits = TYPE_VECTOR_SUBPARTS (vectype);
> +
> +  /* True if the vectorized statements would access beyond the last
> +     statement in the group.  */
> +  bool overrun_p = false;
> +
> +  /* True if we can cope with such overrun by peeling for gaps, so that
> +     there is at least one final scalar iteration after the vector loop.  */
> +  bool can_overrun_p = (vls_type == VLS_LOAD && loop_vinfo && !loop->inner);
> +
> +  /* There can only be a gap at the end of the group if the stride is
> +     known at compile time.  */
> +  gcc_assert (!STMT_VINFO_STRIDED_P (stmt_info) || gap == 0);
> +
> +  /* Stores can't yet have gaps.  */
> +  gcc_assert (slp || vls_type == VLS_LOAD || gap == 0);
> +
> +  if (slp)
> +    {
> +      if (STMT_VINFO_STRIDED_P (stmt_info))
> +       {
> +         /* Try to use consecutive accesses of GROUP_SIZE elements,
> +            separated by the stride, until we have a complete vector.
> +            Fall back to scalar accesses if that isn't possible.  */
> +         if (nunits % group_size == 0)
> +           *memory_access_type = VMAT_STRIDED_SLP;
> +         else
> +           *memory_access_type = VMAT_ELEMENTWISE;
> +       }
> +      else
> +       {
> +         overrun_p = loop_vinfo && gap != 0;
> +         if (overrun_p && vls_type != VLS_LOAD)
> +           {
> +             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                              "Grouped store with gaps requires"
> +                              " non-consecutive accesses\n");
> +             return false;
> +           }
> +         if (overrun_p && !can_overrun_p)
> +           {
> +             if (dump_enabled_p ())
> +               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                                "Peeling for outer loop is not supported\n");
> +             return false;
> +           }
> +         *memory_access_type = VMAT_CONTIGUOUS;
> +       }
> +    }
> +  else
> +    {
> +      /* We can always handle this case using elementwise accesses,
> +        but see if something more efficient is available.  */
> +      *memory_access_type = VMAT_ELEMENTWISE;
> +
> +      /* If there is a gap at the end of the group then these optimizations
> +        would access excess elements in the last iteration.  */
> +      bool would_overrun_p = (gap != 0);
> +      if (!STMT_VINFO_STRIDED_P (stmt_info)
> +         && (can_overrun_p || !would_overrun_p))
> +       {
> +         /* First try using LOAD/STORE_LANES.  */
> +         if (vls_type == VLS_LOAD
> +             ? vect_load_lanes_supported (vectype, group_size)
> +             : vect_store_lanes_supported (vectype, group_size))
> +           {
> +             *memory_access_type = VMAT_LOAD_STORE_LANES;
> +             overrun_p = would_overrun_p;
> +           }
> +
> +         /* If that fails, try using permuting loads.  */
> +         if (*memory_access_type == VMAT_ELEMENTWISE
> +             && (vls_type == VLS_LOAD
> +                 ? vect_grouped_load_supported (vectype, single_element_p,
> +                                                group_size)
> +                 : vect_grouped_store_supported (vectype, group_size)))
> +           {
> +             *memory_access_type = VMAT_CONTIGUOUS_PERMUTE;
> +             overrun_p = would_overrun_p;
> +           }
> +       }
> +    }
> +
> +  if (vls_type != VLS_LOAD && first_stmt == stmt)
> +    {
> +      /* STMT is the leader of the group. Check the operands of all the
> +        stmts of the group.  */
> +      gimple *next_stmt = GROUP_NEXT_ELEMENT (stmt_info);
> +      while (next_stmt)
> +       {
> +         gcc_assert (gimple_assign_single_p (next_stmt));
> +         tree op = gimple_assign_rhs1 (next_stmt);
> +         gimple *def_stmt;
> +         enum vect_def_type dt;
> +         if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
> +           {
> +             if (dump_enabled_p ())
> +               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                                "use not simple.\n");
> +             return false;
> +           }
> +         next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
> +       }
> +    }
> +
> +  if (overrun_p)
> +    {
> +      gcc_assert (can_overrun_p);
> +      if (dump_enabled_p ())
> +       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                        "Data access with gaps requires scalar "
> +                        "epilogue loop\n");
> +      LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
> +    }
> +
> +  return true;
> +}
> +
> +/* Analyze load or store statement STMT of type VLS_TYPE.  Return true
> +   if there is a memory access type that the vectorized form can use,
> +   storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
> +   or scatters, fill in GS_INFO accordingly.
> +
> +   SLP says whether we're performing SLP rather than loop vectorization.
> +   VECTYPE is the vector type that the vectorized statements will use.  */
> +
> +static bool
> +get_load_store_type (gimple *stmt, tree vectype, bool slp,
> +                    vec_load_store_type vls_type,
> +                    vect_memory_access_type *memory_access_type,
> +                    gather_scatter_info *gs_info)
> +{
> +  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
> +  vec_info *vinfo = stmt_info->vinfo;
> +  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
> +  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +    {
> +      *memory_access_type = VMAT_GATHER_SCATTER;
> +      gimple *def_stmt;
> +      if (!vect_check_gather_scatter (stmt, loop_vinfo, gs_info))
> +       gcc_unreachable ();
> +      else if (!vect_is_simple_use (gs_info->offset, vinfo, &def_stmt,
> +                                   &gs_info->offset_dt,
> +                                   &gs_info->offset_vectype))
> +       {
> +         if (dump_enabled_p ())
> +           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                            "%s index use not simple.\n",
> +                            vls_type == VLS_LOAD ? "gather" : "scatter");
> +         return false;
> +       }
> +    }
> +  else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> +    {
> +      if (!get_group_load_store_type (stmt, vectype, slp, vls_type,
> +                                     memory_access_type))
> +       return false;
> +    }
> +  else if (STMT_VINFO_STRIDED_P (stmt_info))
> +    {
> +      gcc_assert (!slp);
> +      *memory_access_type = VMAT_ELEMENTWISE;
> +    }
> +  else
> +    *memory_access_type = VMAT_CONTIGUOUS;
> +
> +  /* FIXME: At the moment the cost model seems to underestimate the
> +     cost of using elementwise accesses.  This check preserves the
> +     traditional behavior until that can be fixed.  */
> +  if (*memory_access_type == VMAT_ELEMENTWISE
> +      && !STMT_VINFO_STRIDED_P (stmt_info))
> +    {
> +      if (dump_enabled_p ())
> +       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                        "not falling back to elementwise accesses\n");
> +      return false;
> +    }
> +  return true;
> +}
> +
>  /* Function vectorizable_mask_load_store.
>
>     Check if STMT performs a conditional load or store that can be vectorized.
> @@ -1705,7 +1907,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>    int i, j;
>    bool inv_p;
>    gather_scatter_info gs_info;
> -  bool is_store;
> +  vec_load_store_type vls_type;
>    tree mask;
>    gimple *def_stmt;
>    enum vect_def_type dt;
> @@ -1716,7 +1918,6 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>    ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
>    gcc_assert (ncopies >= 1);
>
> -  is_store = gimple_call_internal_fn (stmt) == IFN_MASK_STORE;
>    mask = gimple_call_arg (stmt, 2);
>
>    if (TREE_CODE (TREE_TYPE (mask)) != BOOLEAN_TYPE)
> @@ -1743,12 +1944,6 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>
>    elem_type = TREE_TYPE (vectype);
>
> -  if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> -    return false;
> -
> -  if (STMT_VINFO_STRIDED_P (stmt_info))
> -    return false;
> -
>    if (TREE_CODE (mask) != SSA_NAME)
>      return false;
>
> @@ -1762,27 +1957,26 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>        || TYPE_VECTOR_SUBPARTS (mask_vectype) != TYPE_VECTOR_SUBPARTS (vectype))
>      return false;
>
> -  if (is_store)
> +  if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE)
>      {
>        tree rhs = gimple_call_arg (stmt, 3);
>        if (!vect_is_simple_use (rhs, loop_vinfo, &def_stmt, &dt, &rhs_vectype))
>         return false;
> +      if (dt == vect_constant_def || dt == vect_external_def)
> +       vls_type = VLS_STORE_INVARIANT;
> +      else
> +       vls_type = VLS_STORE;
>      }
> +  else
> +    vls_type = VLS_LOAD;
>
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -    {
> -      gimple *def_stmt;
> -      if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> -       gcc_unreachable ();
> -      if (!vect_is_simple_use (gs_info.offset, loop_vinfo, &def_stmt,
> -                              &gs_info.offset_dt, &gs_info.offset_vectype))
> -       {
> -         if (dump_enabled_p ())
> -           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                            "gather index use not simple.");
> -         return false;
> -       }
> +  vect_memory_access_type memory_access_type;
> +  if (!get_load_store_type (stmt, vectype, false, vls_type,
> +                           &memory_access_type, &gs_info))
> +    return false;
>
> +  if (memory_access_type == VMAT_GATHER_SCATTER)
> +    {
>        tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
>        tree masktype
>         = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
> @@ -1794,6 +1988,14 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>           return false;
>         }
>      }
> +  else if (memory_access_type != VMAT_CONTIGUOUS)
> +    {
> +      if (dump_enabled_p ())
> +       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +                        "unsupported access type for masked %s\n",
> +                        vls_type == VLS_LOAD ? "load" : "store");
> +      return false;
> +    }
>    else if (tree_int_cst_compare (nested_in_vect_loop
>                                  ? STMT_VINFO_DR_STEP (stmt_info)
>                                  : DR_STEP (dr), size_zero_node) <= 0)
> @@ -1801,25 +2003,28 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>    else if (!VECTOR_MODE_P (TYPE_MODE (vectype))
>            || !can_vec_mask_load_store_p (TYPE_MODE (vectype),
>                                           TYPE_MODE (mask_vectype),
> -                                         !is_store)
> +                                         vls_type == VLS_LOAD)
>            || (rhs_vectype
>                && !useless_type_conversion_p (vectype, rhs_vectype)))
>      return false;
>
>    if (!vec_stmt) /* transformation not required.  */
>      {
> +      STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>        STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
> -      if (is_store)
> -       vect_model_store_cost (stmt_info, ncopies, false, dt,
> -                              NULL, NULL, NULL);
> +      if (vls_type == VLS_LOAD)
> +       vect_model_load_cost (stmt_info, ncopies, memory_access_type,
> +                             NULL, NULL, NULL);
>        else
> -       vect_model_load_cost (stmt_info, ncopies, false, NULL, NULL, NULL);
> +       vect_model_store_cost (stmt_info, ncopies, memory_access_type,
> +                              dt, NULL, NULL, NULL);
>        return true;
>      }
> +  gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
>
>    /** Transform.  **/
>
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +  if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        tree vec_oprnd0 = NULL_TREE, op;
>        tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> @@ -1993,7 +2198,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>        gsi_replace (gsi, new_stmt, true);
>        return true;
>      }
> -  else if (is_store)
> +  else if (vls_type != VLS_LOAD)
>      {
>        tree vec_rhs = NULL_TREE, vec_mask = NULL_TREE;
>        prev_stmt_info = NULL;
> @@ -2102,7 +2307,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
>         }
>      }
>
> -  if (!is_store)
> +  if (vls_type == VLS_LOAD)
>      {
>        /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
>          from the IL.  */
> @@ -5188,9 +5393,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    gimple *ptr_incr = NULL;
>    int ncopies;
>    int j;
> -  gimple *next_stmt, *first_stmt = NULL;
> -  bool grouped_store = false;
> -  bool store_lanes_p = false;
> +  gimple *next_stmt, *first_stmt;
> +  bool grouped_store;
>    unsigned int group_size, i;
>    vec<tree> dr_chain = vNULL;
>    vec<tree> oprnds = vNULL;
> @@ -5207,6 +5411,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    gather_scatter_info gs_info;
>    enum vect_def_type scatter_src_dt = vect_unknown_def_type;
>    gimple *new_stmt;
> +  vec_load_store_type vls_type;
>
>    if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
>      return false;
> @@ -5274,6 +5479,11 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>        return false;
>      }
>
> +  if (dt == vect_constant_def || dt == vect_external_def)
> +    vls_type = VLS_STORE_INVARIANT;
> +  else
> +    vls_type = VLS_STORE;
> +
>    if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
>      return false;
>
> @@ -5303,7 +5513,6 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>         }
>        if (negative)
>         {
> -         gcc_assert (!grouped_store);
>           alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
>           if (alignment_support_scheme != dr_aligned
>               && alignment_support_scheme != dr_unaligned_supported)
> @@ -5325,80 +5534,31 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>         }
>      }
>
> -  if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
> -    {
> -      grouped_store = true;
> -      first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
> -      group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
> -      if (!slp && !STMT_VINFO_STRIDED_P (stmt_info))
> -       {
> -         if (vect_store_lanes_supported (vectype, group_size))
> -           store_lanes_p = true;
> -         else if (!vect_grouped_store_supported (vectype, group_size))
> -           return false;
> -       }
> -
> -      if (STMT_VINFO_STRIDED_P (stmt_info)
> -         && slp
> -         && (group_size > nunits
> -             || nunits % group_size != 0))
> -       {
> -         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                          "unhandled strided group store\n");
> -         return false;
> -       }
> -
> -      if (first_stmt == stmt)
> -       {
> -          /* STMT is the leader of the group. Check the operands of all the
> -             stmts of the group.  */
> -          next_stmt = GROUP_NEXT_ELEMENT (stmt_info);
> -          while (next_stmt)
> -            {
> -             gcc_assert (gimple_assign_single_p (next_stmt));
> -             op = gimple_assign_rhs1 (next_stmt);
> -              if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
> -                {
> -                  if (dump_enabled_p ())
> -                    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                                     "use not simple.\n");
> -                  return false;
> -                }
> -              next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
> -            }
> -        }
> -    }
> -
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -    {
> -      gimple *def_stmt;
> -      if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> -       gcc_unreachable ();
> -      if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
> -                              &gs_info.offset_dt, &gs_info.offset_vectype))
> -       {
> -         if (dump_enabled_p ())
> -           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                             "scatter index use not simple.");
> -         return false;
> -       }
> -    }
> +  vect_memory_access_type memory_access_type;
> +  if (!get_load_store_type (stmt, vectype, slp, vls_type,
> +                           &memory_access_type, &gs_info))
> +    return false;
>
>    if (!vec_stmt) /* transformation not required.  */
>      {
> +      if (!slp)
> +       STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>        STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
>        /* The SLP costs are calculated during SLP analysis.  */
>        if (!PURE_SLP_STMT (stmt_info))
> -       vect_model_store_cost (stmt_info, ncopies, store_lanes_p, dt,
> +       vect_model_store_cost (stmt_info, ncopies, memory_access_type, dt,
>                                NULL, NULL, NULL);
>        return true;
>      }
> +  if (!slp)
> +    gcc_assert (memory_access_type
> +               == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
>
>    /** Transform.  **/
>
>    ensure_base_align (stmt_info, dr);
>
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +  if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src;
>        tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> @@ -5538,8 +5698,10 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>        return true;
>      }
>
> +  grouped_store = STMT_VINFO_GROUPED_ACCESS (stmt_info);
>    if (grouped_store)
>      {
> +      first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
>        first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
>        group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
>
> @@ -5585,7 +5747,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>      dump_printf_loc (MSG_NOTE, vect_location,
>                       "transform store. ncopies = %d\n", ncopies);
>
> -  if (STMT_VINFO_STRIDED_P (stmt_info))
> +  if (memory_access_type == VMAT_ELEMENTWISE
> +      || memory_access_type == VMAT_STRIDED_SLP)
>      {
>        gimple_stmt_iterator incr_gsi;
>        bool insert_after;
> @@ -5756,14 +5919,14 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    gcc_assert (alignment_support_scheme);
>    /* Targets with store-lane instructions must not require explicit
>       realignment.  */
> -  gcc_assert (!store_lanes_p
> +  gcc_assert (memory_access_type != VMAT_LOAD_STORE_LANES
>               || alignment_support_scheme == dr_aligned
>               || alignment_support_scheme == dr_unaligned_supported);
>
>    if (negative)
>      offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
>
> -  if (store_lanes_p)
> +  if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
>    else
>      aggr_type = vectype;
> @@ -5901,7 +6064,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>                                            TYPE_SIZE_UNIT (aggr_type));
>         }
>
> -      if (store_lanes_p)
> +      if (memory_access_type == VMAT_LOAD_STORE_LANES)
>         {
>           tree vec_array;
>
> @@ -6185,7 +6348,6 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    gphi *phi = NULL;
>    vec<tree> dr_chain = vNULL;
>    bool grouped_load = false;
> -  bool load_lanes_p = false;
>    gimple *first_stmt;
>    gimple *first_stmt_for_drptr = NULL;
>    bool inv_p;
> @@ -6294,48 +6456,11 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>      {
>        grouped_load = true;
>        /* FORNOW */
> -      gcc_assert (!nested_in_vect_loop && !STMT_VINFO_GATHER_SCATTER_P (stmt_info));
> +      gcc_assert (!nested_in_vect_loop);
> +      gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
>
>        first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
>        group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
> -      bool single_element_p = (first_stmt == stmt
> -                              && !GROUP_NEXT_ELEMENT (stmt_info));
> -
> -      if (!slp && !STMT_VINFO_STRIDED_P (stmt_info))
> -       {
> -         if (vect_load_lanes_supported (vectype, group_size))
> -           load_lanes_p = true;
> -         else if (!vect_grouped_load_supported (vectype, single_element_p,
> -                                                group_size))
> -           return false;
> -       }
> -
> -      if (single_element_p)
> -       {
> -         /* Single-element interleaving requires peeling for gaps.  */
> -         gcc_assert (GROUP_GAP (stmt_info));
> -       }
> -
> -      /* If there is a gap in the end of the group then we access excess
> -        elements in the last iteration and thus need to peel that off.  */
> -      if (loop_vinfo
> -         && ! STMT_VINFO_STRIDED_P (stmt_info)
> -         && GROUP_GAP (vinfo_for_stmt (first_stmt)) != 0)
> -       {
> -         if (dump_enabled_p ())
> -           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                            "Data access with gaps requires scalar "
> -                            "epilogue loop\n");
> -         if (loop->inner)
> -           {
> -             if (dump_enabled_p ())
> -               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                                "Peeling for outer loop is not supported\n");
> -             return false;
> -           }
> -
> -         LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
> -       }
>
>        if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
>         slp_perm = true;
> @@ -6381,24 +6506,13 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>         }
>      }
>
> +  vect_memory_access_type memory_access_type;
> +  if (!get_load_store_type (stmt, vectype, slp, VLS_LOAD,
> +                           &memory_access_type, &gs_info))
> +    return false;
>
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -    {
> -      gimple *def_stmt;
> -      if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
> -       gcc_unreachable ();
> -      if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
> -                              &gs_info.offset_dt, &gs_info.offset_vectype))
> -       {
> -         if (dump_enabled_p ())
> -           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                             "gather index use not simple.\n");
> -         return false;
> -       }
> -    }
> -  else if (STMT_VINFO_STRIDED_P (stmt_info))
> -    ;
> -  else
> +  if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info)
> +      && !STMT_VINFO_STRIDED_P (stmt_info))
>      {
>        negative = tree_int_cst_compare (nested_in_vect_loop
>                                        ? STMT_VINFO_DR_STEP (stmt_info)
> @@ -6444,14 +6558,20 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>
>    if (!vec_stmt) /* transformation not required.  */
>      {
> +      if (!slp)
> +       STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>        STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
>        /* The SLP costs are calculated during SLP analysis.  */
>        if (!PURE_SLP_STMT (stmt_info))
> -       vect_model_load_cost (stmt_info, ncopies, load_lanes_p,
> +       vect_model_load_cost (stmt_info, ncopies, memory_access_type,
>                               NULL, NULL, NULL);
>        return true;
>      }
>
> +  if (!slp)
> +    gcc_assert (memory_access_type
> +               == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
> +
>    if (dump_enabled_p ())
>      dump_printf_loc (MSG_NOTE, vect_location,
>                       "transform load. ncopies = %d\n", ncopies);
> @@ -6460,7 +6580,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>
>    ensure_base_align (stmt_info, dr);
>
> -  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +  if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        tree vec_oprnd0 = NULL_TREE, op;
>        tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
> @@ -6627,7 +6747,9 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>         }
>        return true;
>      }
> -  else if (STMT_VINFO_STRIDED_P (stmt_info))
> +
> +  if (memory_access_type == VMAT_ELEMENTWISE
> +      || memory_access_type == VMAT_STRIDED_SLP)
>      {
>        gimple_stmt_iterator incr_gsi;
>        bool insert_after;
> @@ -6694,26 +6816,23 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>        int lnel = 1;
>        tree ltype = TREE_TYPE (vectype);
>        auto_vec<tree> dr_chain;
> -      if (slp)
> +      if (memory_access_type == VMAT_STRIDED_SLP)
>         {
> -         if (group_size < nunits
> -             && nunits % group_size == 0)
> +         nloads = nunits / group_size;
> +         if (group_size < nunits)
>             {
> -             nloads = nunits / group_size;
>               lnel = group_size;
>               ltype = build_vector_type (TREE_TYPE (vectype), group_size);
> -             ltype = build_aligned_type (ltype,
> -                                         TYPE_ALIGN (TREE_TYPE (vectype)));
>             }
> -         else if (group_size >= nunits
> -                  && group_size % nunits == 0)
> +         else
>             {
> -             nloads = 1;
>               lnel = nunits;
>               ltype = vectype;
> -             ltype = build_aligned_type (ltype,
> -                                         TYPE_ALIGN (TREE_TYPE (vectype)));
>             }
> +         ltype = build_aligned_type (ltype, TYPE_ALIGN (TREE_TYPE (vectype)));
> +       }
> +      if (slp)
> +       {
>           /* For SLP permutation support we need to load the whole group,
>              not only the number of vector stmts the permutation result
>              fits in.  */
> @@ -6845,7 +6964,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    gcc_assert (alignment_support_scheme);
>    /* Targets with load-lane instructions must not require explicit
>       realignment.  */
> -  gcc_assert (!load_lanes_p
> +  gcc_assert (memory_access_type != VMAT_LOAD_STORE_LANES
>               || alignment_support_scheme == dr_aligned
>               || alignment_support_scheme == dr_unaligned_supported);
>
> @@ -6980,7 +7099,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>    if (negative)
>      offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
>
> -  if (load_lanes_p)
> +  if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
>    else
>      aggr_type = vectype;
> @@ -7043,7 +7162,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>        if (grouped_load || slp_perm)
>         dr_chain.create (vec_num);
>
> -      if (load_lanes_p)
> +      if (memory_access_type == VMAT_LOAD_STORE_LANES)
>         {
>           tree vec_array;
>
> @@ -7313,7 +7432,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
>          {
>            if (grouped_load)
>             {
> -             if (!load_lanes_p)
> +             if (memory_access_type != VMAT_LOAD_STORE_LANES)
>                 vect_transform_grouped_load (stmt, dr_chain, group_size, gsi);
>               *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
>             }


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]