[patch] Skip unvectorizable statements in basic block SLP

[Ira Rosen <IRAR at il dot ibm dot com>]

Hi,

This patch adds an ability to skip unvectorizable statements in basic block
vectorization, and continue to look for other SLP opportunities in the same
basic block.

Bootstrapped and tested on x86_64-suse-linux. Committed.

Ira


ChangeLog:

      * tree-vectorizer.h (struct _stmt_vec_info): Add new field to
      determine if the statement is vectorizable, and a macro to
      access it.
      * tree-vect-data-refs.c (vect_analyze_data_ref_dependence):
      Skip statements that can't be vectorized. If the analysis
      fails, mark the statement as unvectorizable if vectorizing
      basic block.
      (vect_compute_data_refs_alignment): Likewise.
      (vect_verify_datarefs_alignment): Skip statements marked as
      unvectorizable. Add print.
      (vect_analyze_group_access): Skip statements that can't be
      vectorized. If the analysis fails, mark the statement as
      unvectorizable if vectorizing basic block.
      (vect_analyze_data_ref_accesses, vect_analyze_data_refs):
      Likewise.
      * tree-vect-stmts.c (vectorizable_store): Fix the number of
      generated stmts for SLP.
      (new_stmt_vec_info): Initialize the new field.
      * tree-vect-slp.c (vect_build_slp_tree): Fail to vectorize
      statements marked as unvectorizable.

testsuite/ChangeLog:

      * gcc.dg/vect/bb-slp-23.c: New test.

Index: tree-vectorizer.h
===================================================================
--- tree-vectorizer.h   (revision 158697)
+++ tree-vectorizer.h   (working copy)
@@ -489,6 +489,10 @@ typedef struct _stmt_vec_info {

   /* The bb_vec_info with respect to which STMT is vectorized.  */
   bb_vec_info bb_vinfo;
+
+  /* Is this statement vectorizable or should it be skipped in (partial)
+     vectorization.  */
+  bool vectorizable;
 } *stmt_vec_info;

 /* Access Functions.  */
@@ -500,6 +504,7 @@ typedef struct _stmt_vec_info {
 #define STMT_VINFO_LIVE_P(S)               (S)->live
 #define STMT_VINFO_VECTYPE(S)              (S)->vectype
 #define STMT_VINFO_VEC_STMT(S)             (S)->vectorized_stmt
+#define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable
 #define STMT_VINFO_DATA_REF(S)             (S)->data_ref_info

 #define STMT_VINFO_DR_BASE_ADDRESS(S)      (S)->dr_base_address
Index: tree-vect-data-refs.c
===================================================================
--- tree-vect-data-refs.c       (revision 158697)
+++ tree-vect-data-refs.c       (working copy)
@@ -503,6 +503,11 @@ vect_analyze_data_ref_dependence (struct
   lambda_vector dist_v;
   unsigned int loop_depth;

+  /* Don't bother to analyze statements marked as unvectorizable.  */
+  if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
+      || !STMT_VINFO_VECTORIZABLE (stmtinfo_b))
+    return false;
+
   if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
     {
       /* Independent data accesses.  */
@@ -546,7 +551,11 @@ vect_analyze_data_ref_dependence (struct
           print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
         }

-      return true;
+      /* Mark the statements as unvectorizable.  */
+      STMT_VINFO_VECTORIZABLE (stmtinfo_a) = false;
+      STMT_VINFO_VECTORIZABLE (stmtinfo_b) = false;
+
+      return false;
     }

   /* Versioning for alias is not yet supported for basic block SLP, and
@@ -851,8 +860,18 @@ vect_compute_data_refs_alignment (loop_v
     datarefs = BB_VINFO_DATAREFS (bb_vinfo);
   for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
-    if (!vect_compute_data_ref_alignment (dr))
-      return false;
+    if (STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr)))
+        && !vect_compute_data_ref_alignment (dr))
+      {
+        if (bb_vinfo)
+          {
+            /* Mark unsupported statement as unvectorizable.  */
+            STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr))) =
false;
+            continue;
+          }
+        else
+          return false;
+      }

   return true;
 }
@@ -939,9 +958,11 @@ vect_verify_datarefs_alignment (loop_vec
       gimple stmt = DR_STMT (dr);
       stmt_vec_info stmt_info = vinfo_for_stmt (stmt);

-      /* For interleaving, only the alignment of the first access matters.
*/
-      if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
-          && DR_GROUP_FIRST_DR (stmt_info) != stmt)
+      /* For interleaving, only the alignment of the first access matters.
+         Skip statements marked as not vectorizable.  */
+      if ((STMT_VINFO_STRIDED_ACCESS (stmt_info)
+           && DR_GROUP_FIRST_DR (stmt_info) != stmt)
+          || !STMT_VINFO_VECTORIZABLE (stmt_info))
         continue;

       supportable_dr_alignment = vect_supportable_dr_alignment (dr);
@@ -955,6 +976,8 @@ vect_verify_datarefs_alignment (loop_vec
               else
                 fprintf (vect_dump,
                          "not vectorized: unsupported unaligned store.");
+
+              print_generic_expr (vect_dump, DR_REF (dr), TDF_SLIM);
             }
           return false;
         }
@@ -1564,8 +1587,20 @@ vect_analyze_group_access (struct data_r
            }
          return true;
        }
+
       if (vect_print_dump_info (REPORT_DETAILS))
-       fprintf (vect_dump, "not consecutive access");
+        {
+         fprintf (vect_dump, "not consecutive access ");
+          print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+        }
+
+      if (bb_vinfo)
+        {
+          /* Mark the statement as unvectorizable.  */
+          STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr))) = false;
+          return true;
+        }
+
       return false;
     }

@@ -1836,11 +1871,20 @@ vect_analyze_data_ref_accesses (loop_vec
     datarefs = BB_VINFO_DATAREFS (bb_vinfo);

   for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
-    if (!vect_analyze_data_ref_access (dr))
+    if (STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr)))
+        && !vect_analyze_data_ref_access (dr))
       {
        if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
          fprintf (vect_dump, "not vectorized: complicated access
pattern.");
-       return false;
+
+        if (bb_vinfo)
+          {
+            /* Mark the statement as not vectorizable.  */
+            STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr))) =
false;
+            continue;
+          }
+        else
+          return false;
       }

   return true;
@@ -2013,7 +2057,15 @@ vect_analyze_data_refs (loop_vec_info lo
               fprintf (vect_dump, "not vectorized: data ref analysis
failed ");
               print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
             }
-          return false;
+
+          if (bb_vinfo)
+            {
+              /* Mark the statement as not vectorizable.  */
+              STMT_VINFO_VECTORIZABLE (stmt_info) = false;
+              continue;
+            }
+          else
+            return false;
         }

       if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
@@ -2021,7 +2073,14 @@ vect_analyze_data_refs (loop_vec_info lo
           if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
             fprintf (vect_dump, "not vectorized: base addr of dr is a "
                      "constant");
-          return false;
+          if (bb_vinfo)
+            {
+              /* Mark the statement as not vectorizable.  */
+              STMT_VINFO_VECTORIZABLE (stmt_info) = false;
+              continue;
+            }
+          else
+            return false;
         }

       base = unshare_expr (DR_BASE_ADDRESS (dr));
@@ -2163,7 +2222,15 @@ vect_analyze_data_refs (loop_vec_info lo
               fprintf (vect_dump, " scalar_type: ");
               print_generic_expr (vect_dump, scalar_type, TDF_DETAILS);
             }
-          return false;
+
+          if (bb_vinfo)
+            {
+              /* Mark the statement as not vectorizable.  */
+              STMT_VINFO_VECTORIZABLE (stmt_info) = false;
+              continue;
+            }
+          else
+            return false;
         }

       /* Adjust the minimal vectorization factor according to the
Index: tree-vect-stmts.c
===================================================================
--- tree-vect-stmts.c   (revision 158697)
+++ tree-vect-stmts.c   (working copy)
@@ -3031,12 +3031,15 @@ vectorizable_store (gimple stmt, gimple_
        }

       if (slp)
-       strided_store = false;
-
-      /* VEC_NUM is the number of vect stmts to be created for this group.
*/
-      if (slp)
-       vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+        {
+          strided_store = false;
+          /* VEC_NUM is the number of vect stmts to be created for this
group.  */
+          vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+          first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS
(slp_node), 0);
+          first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
+        }
       else
+        /* VEC_NUM is the number of vect stmts to be created for this
group.  */
        vec_num = group_size;
     }
   else
@@ -4327,6 +4330,7 @@ new_stmt_vec_info (gimple stmt, loop_vec
   STMT_VINFO_LIVE_P (res) = false;
   STMT_VINFO_VECTYPE (res) = NULL;
   STMT_VINFO_VEC_STMT (res) = NULL;
+  STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_IN_PATTERN_P (res) = false;
   STMT_VINFO_RELATED_STMT (res) = NULL;
   STMT_VINFO_DATA_REF (res) = NULL;
Index: tree-vect-slp.c
===================================================================
--- tree-vect-slp.c     (revision 158697)
+++ tree-vect-slp.c     (working copy)
@@ -344,6 +344,19 @@ vect_build_slp_tree (loop_vec_info loop_
          print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
        }

+      /* Fail to vectorize statements marked as unvectorizable.  */
+      if (!STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
+        {
+          if (vect_print_dump_info (REPORT_SLP))
+            {
+              fprintf (vect_dump,
+                       "Build SLP failed: unvectorizable statement ");
+              print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+            }
+
+          return false;
+        }
+
       lhs = gimple_get_lhs (stmt);
       if (lhs == NULL_TREE)
        {
Index: testsuite/gcc.dg/vect/bb-slp-23.c
===================================================================
--- testsuite/gcc.dg/vect/bb-slp-23.c   (revision 0)
+++ testsuite/gcc.dg/vect/bb-slp-23.c   (revision 0)
@@ -0,0 +1,54 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include "tree-vect.h"
+
+#define N 16
+
+unsigned int out[N];
+unsigned int in[N] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+int a[N], b[N];
+
+__attribute__ ((noinline)) int
+main1 (unsigned int x, unsigned int y)
+{
+  int i;
+  unsigned int a0, a1, a2, a3;
+
+  /* This statement is ignored in vectorization of this basic block.  */
+  a[x] = b [y];
+
+  a0 = in[0] + 23;
+  a1 = in[1] + 142;
+  a2 = in[2] + 2;
+  a3 = in[3] + 31;
+
+  out[0] = a0 * x;
+  out[1] = a1 * y;
+  out[2] = a2 * x;
+  out[3] = a3 * y;
+
+  /* Check results.  */
+  if (out[0] != (in[0] + 23) * x
+      || out[1] != (in[1] + 142) * y
+      || out[2] != (in[2] + 2) * x
+      || out[3] != (in[3] + 31) * y)
+    abort();
+
+  return 0;
+}
+
+int main (void)
+{
+  check_vect ();
+
+  main1 (2, 3);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "basic block vectorized using SLP" 1
"slp" { target vect_int_mult } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
+