VECT: Apply LEN_FOLD_EXTRACT_LAST into loop vectorizer

Hi.

This patch is apply LEN_FOLD_EXTRACT_LAST into loop vectorizer.

Consider this following case:

/* Simple condition reduction.  */

int __attribute__ ((noinline, noclone))
condition_reduction (int *a, int min_v)
{
  int last = 66; /* High start value.  */

  for (int i = 0; i < N; i++)
    if (a[i] < min_v)
      last = i;

  return last;
}

With this patch, we can generate this following IR:

  _44 = .SELECT_VL (ivtmp_42, POLY_INT_CST [4, 4]);
  _34 = vect_vec_iv_.5_33 + { POLY_INT_CST [4, 4], ... };
  ivtmp_36 = _44 * 4;
  vect__4.8_39 = .MASK_LEN_LOAD (vectp_a.6_37, 32B, { -1, ... }, _44, 0);

  mask__11.9_41 = vect__4.8_39 < vect_cst__40;
  last_5 = .LEN_FOLD_EXTRACT_LAST (last_14, mask__11.9_41, vect_vec_iv_.5_33, _44, 0);
  ...

gcc/ChangeLog:

	* tree-vect-loop.cc (vectorizable_reduction): Apply
	LEN_FOLD_EXTRACT_LAST.
	* tree-vect-stmts.cc (vectorizable_condition): Ditto.

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 1cd6c291377729b28f01168e7745a1665073836e..ebee8037e02787ded8bbb75b6e7006cb7698a5c7 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -7494,8 +7494,11 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
        }
 
       if (reduc_chain_length == 1
-         && direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST,
-                                            vectype_in, OPTIMIZE_FOR_SPEED))
+         && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in,
+                                             OPTIMIZE_FOR_SPEED)
+             || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
+                                                vectype_in,
+                                                OPTIMIZE_FOR_SPEED)))
        {
          if (dump_enabled_p ())
            dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 413a88750d629dac6203350b8d270212f028caba..be9f3a280bd6614ddfc8466e93060c5951c161a3 100644 (file)
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -11740,8 +11740,17 @@ vectorizable_condition (vec_info *vinfo,
          && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
        {
          if (reduction_type == EXTRACT_LAST_REDUCTION)
-           vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo),
-                                  ncopies * vec_num, vectype, NULL);
+           {
+             if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
+                                                 vectype, OPTIMIZE_FOR_SPEED))
+               vect_record_loop_len (loop_vinfo,
+                                     &LOOP_VINFO_LENS (loop_vinfo),
+                                     ncopies * vec_num, vectype, 1);
+             else
+               vect_record_loop_mask (loop_vinfo,
+                                      &LOOP_VINFO_MASKS (loop_vinfo),
+                                      ncopies * vec_num, vectype, NULL);
+           }
          /* Extra inactive lanes should be safe for vect_nested_cycle.  */
          else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle)
            {
@@ -11772,7 +11781,13 @@ vectorizable_condition (vec_info *vinfo,
      mask to the condition, or to its inverse.  */
 
   vec_loop_masks *masks = NULL;
-  if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+  vec_loop_lens *lens = NULL;
+  if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+    {
+      if (reduction_type == EXTRACT_LAST_REDUCTION)
+       lens = &LOOP_VINFO_LENS (loop_vinfo);
+    }
+  else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
     {
       if (reduction_type == EXTRACT_LAST_REDUCTION)
        masks = &LOOP_VINFO_MASKS (loop_vinfo);
@@ -11910,7 +11925,8 @@ vectorizable_condition (vec_info *vinfo,
       /* Force vec_compare to be an SSA_NAME rather than a comparison,
         in cases where that's necessary.  */
 
-      if (masks || reduction_type == EXTRACT_LAST_REDUCTION)
+      tree len = NULL_TREE, bias = NULL_TREE;
+      if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
        {
          if (!is_gimple_val (vec_compare))
            {
@@ -11931,6 +11947,23 @@ vectorizable_condition (vec_info *vinfo,
              vec_compare = vec_compare_name;
            }
 
+         if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
+                                             vectype, OPTIMIZE_FOR_SPEED))
+           {
+             if (lens)
+               {
+                 len = vect_get_loop_len (loop_vinfo, gsi, lens,
+                                          vec_num * ncopies, vectype, i, 1);
+                 signed char biasval
+                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+                 bias = build_int_cst (intQI_type_node, biasval);
+               }
+             else
+               {
+                 len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
+                 bias = build_int_cst (intQI_type_node, 0);
+               }
+           }
          if (masks)
            {
              tree loop_mask
@@ -11950,9 +11983,14 @@ vectorizable_condition (vec_info *vinfo,
        {
          gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
          tree lhs = gimple_get_lhs (old_stmt);
-         new_stmt = gimple_build_call_internal
-             (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
-              vec_then_clause);
+         if (len)
+           new_stmt = gimple_build_call_internal
+               (IFN_LEN_FOLD_EXTRACT_LAST, 5, else_clause, vec_compare,
+                vec_then_clause, len, bias);
+         else
+           new_stmt = gimple_build_call_internal
+               (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
+                vec_then_clause);
          gimple_call_set_lhs (new_stmt, lhs);
          SSA_NAME_DEF_STMT (lhs) = new_stmt;
          if (old_stmt == gsi_stmt (*gsi))