This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[5/6] Insert pattern statements into vec_basic_blocks


The point of this patch is to put pattern statements in the same
vec_basic_block as the statements they replace, with the pattern
statements for S coming between S and S's original predecessor.
This removes the need to handle them specially in various places.


2018-08-28  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (vec_basic_block): Expand comment.
	(_stmt_vec_info::pattern_def_seq): Delete.
	(STMT_VINFO_PATTERN_DEF_SEQ): Likewise.
	(is_main_pattern_stmt_p): New function.
	* tree-vect-loop.c (vect_determine_vf_for_stmt_1): Rename to...
	(vect_determine_vf_for_stmt): ...this, deleting the original
	function with this name.  Remove vectype_maybe_set_p argument
	and test is_pattern_stmt_p instead.  Retain the "examining..."
	message from the previous vect_determine_vf_for_stmt.
	(vect_compute_single_scalar_iteration_cost, vect_update_vf_for_slp)
	(vect_analyze_loop_2): Don't treat pattern statements specially.
	(vect_transform_loop): Likewise.  Use vect_orig_stmt to find the
	insertion point.
	* tree-vect-slp.c (vect_detect_hybrid_slp): Expect pattern statements
	to be in the statement list, without needing to follow
	STMT_VINFO_RELATED_STMT.  Remove PATTERN_DEF_SEQ handling.
	* tree-vect-stmts.c (vect_analyze_stmt): Don't handle pattern
	statements specially.
	(vect_remove_dead_scalar_stmts): Ignore pattern statements.
	* tree-vect-patterns.c (vect_set_pattern_stmt): Insert the pattern
	statement into the vec_basic_block immediately before the statement
	it replaces.
	(append_pattern_def_seq): Likewise.  If the original statement is
	itself a pattern statement, associate the new one with the original
	statement.
	(vect_split_statement): Use append_pattern_def_seq to insert the
	first pattern statement.
	(vect_recog_vector_vector_shift_pattern): Remove mention of
	STMT_VINFO_PATTERN_DEF_SEQ.
	(adjust_bool_stmts): Get the last pattern statement from the
	stmt_vec_info chain.
	(vect_mark_pattern_stmts): Rename to...
	(vect_replace_stmt_with_pattern): ...this.  Remove the
	PATTERN_DEF_SEQ handling and process only the pattern statement given.
	Use append_pattern_def_seq when replacing a pattern statement with
	another pattern statement, and use vec_basic_block::remove instead
	of gsi_remove to remove the old one.
	(vect_pattern_recog_1): Update accordingly.  Remove PATTERN_DEF_SEQ
	handling.  On failure, remove any half-formed pattern sequence from
	the vec_basic_block.  Install the vector type in pattern statements
	that don't yet have one.
	(vect_pattern_recog): Iterate over statements that are added
	by previous recognizers, but skipping those that have already
	been replaced, or the main pattern statement in such a replacement.

Index: gcc/tree-vectorizer.h
===================================================================
--- gcc/tree-vectorizer.h	2018-08-28 12:05:14.014961439 +0100
+++ gcc/tree-vectorizer.h	2018-08-28 12:05:16.522940287 +0100
@@ -172,7 +172,13 @@ #define SLP_TREE_TWO_OPERATORS(S)		 (S)-
 #define SLP_TREE_DEF_TYPE(S)			 (S)->def_type
 
 /* Information about the phis and statements in a block that we're trying
-   to vectorize, in their original order.  */
+   to vectorize.  This includes the phis and statements that were in the
+   original scalar code, in their original order.  It also includes any
+   pattern statements that the vectorizer has created to replace some
+   of the scalar ones.  Such pattern statements come immediately before
+   the statement that they replace; that is, all pattern statements P for
+   which vect_orig_stmt (P) == S form a sequence that comes immediately
+   before S.  */
 class vec_basic_block
 {
 public:
@@ -870,11 +876,6 @@ struct _stmt_vec_info {
         pattern).  */
   stmt_vec_info related_stmt;
 
-  /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
-     The sequence is attached to the original statement rather than the
-     pattern statement.  */
-  gimple_seq pattern_def_seq;
-
   /* List of datarefs that are known to have the same alignment as the dataref
      of this stmt.  */
   vec<dr_p> same_align_refs;
@@ -1048,7 +1049,6 @@ #define STMT_VINFO_DR_INFO(S) \
 
 #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
 #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
-#define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq
 #define STMT_VINFO_SAME_ALIGN_REFS(S)      (S)->same_align_refs
 #define STMT_VINFO_SIMD_CLONE_INFO(S)	   (S)->simd_clone_info
 #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
@@ -1176,6 +1176,17 @@ is_pattern_stmt_p (stmt_vec_info stmt_in
   return stmt_info->pattern_stmt_p;
 }
 
+/* Return TRUE if a statement represented by STMT_INFO is the final
+   statement in a pattern.  */
+
+static inline bool
+is_main_pattern_stmt_p (stmt_vec_info stmt_info)
+{
+  stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
+  return (is_pattern_stmt_p (stmt_info)
+	  && STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt_info);
+}
+
 /* If STMT_INFO is a pattern statement, return the statement that it
    replaces, otherwise return STMT_INFO itself.  */
 
Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c	2018-08-28 12:05:14.010961472 +0100
+++ gcc/tree-vect-loop.c	2018-08-28 12:05:16.518940320 +0100
@@ -155,18 +155,24 @@ Software Foundation; either version 3, o
 
 static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *);
 
-/* Subroutine of vect_determine_vf_for_stmt that handles only one
-   statement.  VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
-   may already be set for general statements (not just data refs).  */
+/* Subroutine of vect_determine_vectorization_factor.  Set the vector
+   type of STMT_INFO and update the vectorization factor VF accordingly.
+   If the statement produces a mask result whose vector type can only be
+   calculated later, add it to MASK_PRODUCERS.  Return true on success
+   or false if something prevented vectorization.  */
 
 static bool
-vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
-			      bool vectype_maybe_set_p,
-			      poly_uint64 *vf,
-			      vec<stmt_vec_info > *mask_producers)
+vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
+			    vec<stmt_vec_info > *mask_producers)
 {
   gimple *stmt = stmt_info->stmt;
 
+  if (dump_enabled_p ())
+    {
+      dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
+      dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
+    }
+
   if ((!STMT_VINFO_RELEVANT_P (stmt_info)
        && !STMT_VINFO_LIVE_P (stmt_info))
       || gimple_clobber_p (stmt))
@@ -188,7 +194,7 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
 	   that contain a data ref, or for "pattern-stmts" (stmts generated
 	   by the vectorizer to represent/replace a certain idiom).  */
 	gcc_assert ((STMT_VINFO_DATA_REF (stmt_info)
-		     || vectype_maybe_set_p)
+		     || is_pattern_stmt_p (stmt_info))
 		    && STMT_VINFO_VECTYPE (stmt_info) == stmt_vectype);
       else if (stmt_vectype == boolean_type_node)
 	mask_producers->safe_push (stmt_info);
@@ -202,62 +208,6 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
   return true;
 }
 
-/* Subroutine of vect_determine_vectorization_factor.  Set the vector
-   types of STMT_INFO and all attached pattern statements and update
-   the vectorization factor VF accordingly.  If some of the statements
-   produce a mask result whose vector type can only be calculated later,
-   add them to MASK_PRODUCERS.  Return true on success or false if
-   something prevented vectorization.  */
-
-static bool
-vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
-			    vec<stmt_vec_info > *mask_producers)
-{
-  vec_info *vinfo = stmt_info->vinfo;
-  if (dump_enabled_p ())
-    {
-      dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
-      dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
-    }
-  if (!vect_determine_vf_for_stmt_1 (stmt_info, false, vf, mask_producers))
-    return false;
-
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-      && STMT_VINFO_RELATED_STMT (stmt_info))
-    {
-      gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
-      stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
-
-      /* If a pattern statement has def stmts, analyze them too.  */
-      for (gimple_stmt_iterator si = gsi_start (pattern_def_seq);
-	   !gsi_end_p (si); gsi_next (&si))
-	{
-	  stmt_vec_info def_stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
-	  if (dump_enabled_p ())
-	    {
-	      dump_printf_loc (MSG_NOTE, vect_location,
-			       "==> examining pattern def stmt: ");
-	      dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
-				def_stmt_info->stmt, 0);
-	    }
-	  if (!vect_determine_vf_for_stmt_1 (def_stmt_info, true,
-					     vf, mask_producers))
-	    return false;
-	}
-
-      if (dump_enabled_p ())
-	{
-	  dump_printf_loc (MSG_NOTE, vect_location,
-			   "==> examining pattern statement: ");
-	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
-	}
-      if (!vect_determine_vf_for_stmt_1 (stmt_info, true, vf, mask_producers))
-	return false;
-    }
-
-  return true;
-}
-
 /* Function vect_determine_vectorization_factor
 
    Determine the vectorization factor (VF).  VF is the number of data elements
@@ -1113,9 +1063,8 @@ vect_compute_single_scalar_iteration_cos
           /* Skip stmts that are not vectorized inside the loop.  */
           if (stmt_info
               && !STMT_VINFO_RELEVANT_P (stmt_info)
-              && (!STMT_VINFO_LIVE_P (stmt_info)
-                  || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
-	      && !STMT_VINFO_IN_PATTERN_P (stmt_info))
+	      && (!VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
+		  || !STMT_VINFO_LIVE_P (stmt_info)))
             continue;
 
 	  vect_cost_for_stmt kind;
@@ -1429,14 +1378,11 @@ vect_update_vf_for_slp (loop_vec_info lo
   vec_basic_block *vec_bb;
   FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
     FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
-      {
-	stmt_vec_info final_info = vect_stmt_to_vectorize (stmt_info);
-	if ((STMT_VINFO_RELEVANT_P (final_info)
-	     || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (final_info)))
-	    && !PURE_SLP_STMT (final_info))
-	  /* STMT needs both SLP and loop-based vectorization.  */
-	  only_slp_in_loop = false;
-      }
+      if ((STMT_VINFO_RELEVANT_P (stmt_info)
+	   || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
+	  && !PURE_SLP_STMT (stmt_info))
+	/* STMT needs both SLP and loop-based vectorization.  */
+	only_slp_in_loop = false;
 
   if (only_slp_in_loop)
     {
@@ -2199,18 +2145,7 @@ vect_analyze_loop_2 (loop_vec_info loop_
   vec_basic_block *vec_bb;
   FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
     FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
-      {
-	STMT_SLP_TYPE (stmt_info) = loop_vect;
-	if (STMT_VINFO_IN_PATTERN_P (stmt_info))
-	  {
-	    gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
-	    STMT_SLP_TYPE (STMT_VINFO_RELATED_STMT (stmt_info)) = loop_vect;
-	    for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
-		 !gsi_end_p (pi); gsi_next (&pi))
-	      STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
-		= loop_vect;
-	  }
-      }
+      STMT_SLP_TYPE (stmt_info) = loop_vect;
 
   /* Free optimized alias test DDRS.  */
   LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
@@ -8409,22 +8344,8 @@ vect_transform_loop (loop_vec_info loop_
 	    loop_vinfo->remove_stmt (stmt_info);
 	  else
 	    {
-	      gimple_stmt_iterator si = gsi_for_stmt (stmt_info->stmt);
-	      if (STMT_VINFO_IN_PATTERN_P (stmt_info))
-		{
-		  gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
-		  for (gimple_stmt_iterator subsi = gsi_start (def_seq);
-		       !gsi_end_p (subsi); gsi_next (&subsi))
-		    {
-		      stmt_vec_info pat_stmt_info
-			= loop_vinfo->lookup_stmt (gsi_stmt (subsi));
-		      vect_transform_loop_stmt (loop_vinfo, pat_stmt_info,
-						&si);
-		    }
-		  stmt_vec_info pat_stmt_info
-		    = STMT_VINFO_RELATED_STMT (stmt_info);
-		  vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, &si);
-		}
+	      stmt_vec_info place = vect_orig_stmt (stmt_info);
+	      gimple_stmt_iterator si = gsi_for_stmt (place->stmt);
 	      vect_transform_loop_stmt (loop_vinfo, stmt_info, &si);
 	    }
 	}
Index: gcc/tree-vect-slp.c
===================================================================
--- gcc/tree-vect-slp.c	2018-08-28 12:05:14.010961472 +0100
+++ gcc/tree-vect-slp.c	2018-08-28 12:05:16.522940287 +0100
@@ -2362,18 +2362,14 @@ vect_detect_hybrid_slp (loop_vec_info lo
   vec_basic_block *vec_bb;
   FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
     FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
-      if (STMT_VINFO_IN_PATTERN_P (stmt_info))
+      if (is_pattern_stmt_p (stmt_info))
 	{
 	  walk_stmt_info wi;
 	  memset (&wi, 0, sizeof (wi));
 	  wi.info = loop_vinfo;
-	  gimple_stmt_iterator gsi2
-	    = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt);
-	  walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
+	  gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt);
+	  walk_gimple_stmt (&gsi, vect_detect_hybrid_slp_2,
 			    vect_detect_hybrid_slp_1, &wi);
-	  walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
-			   vect_detect_hybrid_slp_2,
-			   vect_detect_hybrid_slp_1, &wi);
 	}
 
   /* Then walk the SLP instance trees marking stmts with uses in
Index: gcc/tree-vect-stmts.c
===================================================================
--- gcc/tree-vect-stmts.c	2018-08-28 12:05:14.014961439 +0100
+++ gcc/tree-vect-stmts.c	2018-08-28 12:05:16.522940287 +0100
@@ -9391,11 +9391,9 @@ vect_analyze_stmt (stmt_vec_info stmt_in
 		   slp_tree node, slp_instance node_instance,
 		   stmt_vector_for_cost *cost_vec)
 {
-  vec_info *vinfo = stmt_info->vinfo;
   bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
   enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
   bool ok;
-  gimple_seq pattern_def_seq;
 
   if (dump_enabled_p ())
     {
@@ -9412,94 +9410,21 @@ vect_analyze_stmt (stmt_vec_info stmt_in
       return false;
     }
 
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-      && node == NULL
-      && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
-    {
-      gimple_stmt_iterator si;
-
-      for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
-	{
-	  stmt_vec_info pattern_def_stmt_info
-	    = vinfo->lookup_stmt (gsi_stmt (si));
-	  if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
-	    {
-	      /* Analyze def stmt of STMT if it's a pattern stmt.  */
-	      if (dump_enabled_p ())
-		{
-		  dump_printf_loc (MSG_NOTE, vect_location,
-				   "==> examining pattern def statement: ");
-		  dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
-				    pattern_def_stmt_info->stmt, 0);
-		}
-
-	      if (!vect_analyze_stmt (pattern_def_stmt_info,
-				      need_to_vectorize, node, node_instance,
-				      cost_vec))
-		return false;
-	    }
-	}
-    }
-
   /* Skip stmts that do not need to be vectorized. In loops this is expected
      to include:
      - the COND_EXPR which is the loop exit condition
      - any LABEL_EXPRs in the loop
      - computations that are used only for array indexing or loop control.
      In basic blocks we only analyze statements that are a part of some SLP
-     instance, therefore, all the statements are relevant.
-
-     Pattern statement needs to be analyzed instead of the original statement
-     if the original statement is not relevant.  Otherwise, we analyze both
-     statements.  In basic blocks we are called from some SLP instance
-     traversal, don't analyze pattern stmts instead, the pattern stmts
-     already will be part of SLP instance.  */
+     instance, therefore, all the statements are relevant.  */
 
-  stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
   if (!STMT_VINFO_RELEVANT_P (stmt_info)
       && !STMT_VINFO_LIVE_P (stmt_info))
     {
-      if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	  && pattern_stmt_info
-	  && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-        {
-          /* Analyze PATTERN_STMT instead of the original stmt.  */
-	  stmt_info = pattern_stmt_info;
-          if (dump_enabled_p ())
-            {
-              dump_printf_loc (MSG_NOTE, vect_location,
-                               "==> examining pattern statement: ");
-	      dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
-            }
-        }
-      else
-        {
-          if (dump_enabled_p ())
-            dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
-
-          return true;
-        }
-    }
-  else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	   && node == NULL
-	   && pattern_stmt_info
-	   && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	       || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-    {
-      /* Analyze PATTERN_STMT too.  */
       if (dump_enabled_p ())
-        {
-          dump_printf_loc (MSG_NOTE, vect_location,
-                           "==> examining pattern statement: ");
-	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt_info->stmt, 0);
-        }
-
-      if (!vect_analyze_stmt (pattern_stmt_info, need_to_vectorize, node,
-			      node_instance, cost_vec))
-        return false;
-   }
+	dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
+      return true;
+    }
 
   switch (STMT_VINFO_DEF_TYPE (stmt_info))
     {
@@ -10924,7 +10849,8 @@ vect_remove_dead_scalar_stmts (vec_info
 	   stmt_info = prev_stmt_info)
 	{
 	  prev_stmt_info = stmt_info->prev;
-	  vect_maybe_remove_scalar_stmt (stmt_info);
+	  if (!is_pattern_stmt_p (stmt_info))
+	    vect_maybe_remove_scalar_stmt (stmt_info);
 	}
     }
 }
Index: gcc/tree-vect-patterns.c
===================================================================
--- gcc/tree-vect-patterns.c	2018-08-28 12:05:11.462982962 +0100
+++ gcc/tree-vect-patterns.c	2018-08-28 12:05:16.518940320 +0100
@@ -125,27 +125,26 @@ vect_init_pattern_stmt (gimple *pattern_
 vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
 		       tree vectype)
 {
-  STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
-  STMT_VINFO_RELATED_STMT (orig_stmt_info)
+  stmt_vec_info pattern_stmt_info
     = vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype);
+  orig_stmt_info->block->add_before (pattern_stmt_info, orig_stmt_info);
+  STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
+  STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt_info;
 }
 
-/* Add NEW_STMT to STMT_INFO's pattern definition statements.  If VECTYPE
-   is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
-   be different from the vector type of the final pattern statement.  */
+/* Add NEW_STMT to the pattern statements that replace STMT_INFO.
+   If VECTYPE is nonnull, record that NEW_STMT's vector type is VECTYPE,
+   which might be different from the vector type of the final pattern
+   statement.  */
 
 static inline void
 append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
 			tree vectype = NULL_TREE)
 {
-  vec_info *vinfo = stmt_info->vinfo;
-  if (vectype)
-    {
-      stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
-      STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
-    }
-  gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
-				      new_stmt);
+  stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
+  stmt_vec_info new_stmt_info
+    = vect_init_pattern_stmt (new_stmt, orig_stmt_info, vectype);
+  stmt_info->block->add_before (new_stmt_info, stmt_info);
 }
 
 /* The caller wants to perform new operations on vect_external variable
@@ -633,11 +632,6 @@ vect_split_statement (stmt_vec_info stmt
 {
   if (is_pattern_stmt_p (stmt2_info))
     {
-      /* STMT2_INFO is part of a pattern.  Get the statement to which
-	 the pattern is attached.  */
-      stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
-      vect_init_pattern_stmt (stmt1, orig_stmt2_info, vectype);
-
       if (dump_enabled_p ())
 	{
 	  dump_printf_loc (MSG_NOTE, vect_location,
@@ -645,6 +639,9 @@ vect_split_statement (stmt_vec_info stmt
 	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
 	}
 
+      /* Insert STMT1_INFO before STMT2_INFO.  */
+      append_pattern_def_seq (stmt2_info, stmt1, vectype);
+
       /* Since STMT2_INFO is a pattern statement, we can change it
 	 in-situ without worrying about changing the code for the
 	 containing block.  */
@@ -658,18 +655,6 @@ vect_split_statement (stmt_vec_info stmt
 	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
 	}
 
-      gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
-      if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
-	/* STMT2_INFO is the actual pattern statement.  Add STMT1
-	   to the end of the definition sequence.  */
-	gimple_seq_add_stmt_without_update (def_seq, stmt1);
-      else
-	{
-	  /* STMT2_INFO belongs to the definition sequence.  Insert STMT1
-	     before it.  */
-	  gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
-	  gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
-	}
       return true;
     }
   else
@@ -689,10 +674,8 @@ vect_split_statement (stmt_vec_info stmt
 	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt2_info->stmt, 0);
 	}
 
-      /* Add STMT1 as a singleton pattern definition sequence.  */
-      gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
-      vect_init_pattern_stmt (stmt1, stmt2_info, vectype);
-      gimple_seq_add_stmt_without_update (def_seq, stmt1);
+      /* Insert STMT1_INFO before STMT2_INFO.  */
+      append_pattern_def_seq (stmt2_info, stmt1, vectype);
 
       /* Build the second of the two pattern statements.  */
       tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
@@ -2164,7 +2147,7 @@ vect_recog_rotate_pattern (stmt_vec_info
     i.e. the shift/rotate stmt.  The original stmt (S3) is replaced
     with a shift/rotate which has same type on both operands, in the
     second case just b_T op c_T, in the first case with added cast
-    from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
+    from a_t to c_T beforehand.
 
   Output:
 
@@ -3518,9 +3501,8 @@ adjust_bool_stmts (hash_set <gimple *> &
     adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]),
 			 out_type, stmt_info, defs);
 
-  /* Pop the last pattern seq stmt and install it as pattern root for STMT.  */
-  gimple *pattern_stmt
-    = gimple_seq_last_stmt (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+  /* Return the result of the last statement we emitted.  */
+  gimple *pattern_stmt = stmt_info->prev->stmt;
   return gimple_assign_lhs (pattern_stmt);
 }
 
@@ -4684,14 +4666,14 @@ static vect_recog_func vect_vect_recog_f
 
 const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
 
-/* Mark statements that are involved in a pattern.  */
+/* Replace ORIG_STMT_INFO with PATTERN_STMT, using PATTERN_VECTYPE as
+   the vector type for PATTERN_STMT.  */
 
 static inline void
-vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
-                         tree pattern_vectype)
+vect_replace_stmt_with_pattern (stmt_vec_info orig_stmt_info,
+				gimple *pattern_stmt,
+				tree pattern_vectype)
 {
-  gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
-
   gimple *orig_pattern_stmt = NULL;
   if (is_pattern_stmt_p (orig_stmt_info))
     {
@@ -4718,32 +4700,14 @@ vect_mark_pattern_stmts (stmt_vec_info o
 	  dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
 	}
 
-      /* Switch to the statement that ORIG replaces.  */
-      orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
-
       /* We shouldn't be replacing the main pattern statement.  */
-      gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
-		  != orig_pattern_stmt);
-    }
+      gcc_assert (!is_main_pattern_stmt_p (orig_stmt_info));
 
-  if (def_seq)
-    for (gimple_stmt_iterator si = gsi_start (def_seq);
-	 !gsi_end_p (si); gsi_next (&si))
-      vect_init_pattern_stmt (gsi_stmt (si), orig_stmt_info, pattern_vectype);
-
-  if (orig_pattern_stmt)
-    {
-      vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
-
-      /* Insert all the new pattern statements before the original one.  */
-      gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
-      gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
-					       orig_def_seq);
-      gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
-      gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
+      /* Insert the new pattern statement before the original one.  */
+      append_pattern_def_seq (orig_stmt_info, pattern_stmt, pattern_vectype);
 
       /* Remove the pattern statement that this new pattern replaces.  */
-      gsi_remove (&gsi, false);
+      orig_stmt_info->block->remove (orig_stmt_info);
     }
   else
     vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
@@ -4770,30 +4734,17 @@ vect_mark_pattern_stmts (stmt_vec_info o
 static void
 vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
 {
-  vec_info *vinfo = stmt_info->vinfo;
   gimple *pattern_stmt;
   loop_vec_info loop_vinfo;
   tree pattern_vectype;
 
-  /* If this statement has already been replaced with pattern statements,
-     leave the original statement alone, since the first match wins.
-     Instead try to match against the definition statements that feed
-     the main pattern statement.  */
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info))
-    {
-      gimple_stmt_iterator gsi;
-      for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
-	   !gsi_end_p (gsi); gsi_next (&gsi))
-	vect_pattern_recog_1 (recog_func, vinfo->lookup_stmt (gsi_stmt (gsi)));
-      return;
-    }
-
-  gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+  stmt_vec_info prev_stmt_info = stmt_info->prev;
   pattern_stmt = recog_func->fn (stmt_info, &pattern_vectype);
   if (!pattern_stmt)
     {
-      /* Clear any half-formed pattern definition sequence.  */
-      STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
+      /* Delete any half-formed pattern sequence.  */
+      while (stmt_info->prev != prev_stmt_info)
+	stmt_info->block->remove (stmt_info->prev);
       return;
     }
 
@@ -4808,8 +4759,15 @@ vect_pattern_recog_1 (vect_recog_func *r
       dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_stmt, 0);
     }
 
+  /* Install the vector type in pattern definition statements that
+     don't yet have one.  */
+  for (stmt_vec_info pat_stmt_info = stmt_info->prev;
+       pat_stmt_info != prev_stmt_info; pat_stmt_info = pat_stmt_info->prev)
+    if (!STMT_VINFO_VECTYPE (pat_stmt_info))
+      STMT_VINFO_VECTYPE (pat_stmt_info) = pattern_vectype;
+
   /* Mark the stmts that are involved in the pattern. */
-  vect_mark_pattern_stmts (stmt_info, pattern_stmt, pattern_vectype);
+  vect_replace_stmt_with_pattern (stmt_info, pattern_stmt, pattern_vectype);
 
   /* Patterns cannot be vectorized using SLP, because they change the order of
      computation.  */
@@ -4911,8 +4869,27 @@ vect_pattern_recog (vec_info *vinfo)
   vec_basic_block *vec_bb;
   FOR_EACH_VEC_ELT (vinfo->blocks, i, vec_bb)
     FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
-      if (STMT_VINFO_VECTORIZABLE (stmt_info))
-	/* Scan over all generic vect_recog_xxx_pattern functions.  */
-	for (unsigned int j = 0; j < NUM_PATTERNS; j++)
-	  vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
+      {
+	stmt_vec_info begin_prev = stmt_info->prev;
+	if (STMT_VINFO_VECTORIZABLE (stmt_info))
+	  /* Scan over all generic vect_recog_xxx_pattern functions.  */
+	  for (unsigned int j = 0; j < NUM_PATTERNS; j++)
+	    {
+	      stmt_vec_info curr_prev;
+	      /* Scan over STMT_INFO and any pattern definition statements
+		 that were introduced by previous recognizers.  */
+	      for (stmt_vec_info curr_info = stmt_info;
+		   curr_info != begin_prev; curr_info = curr_prev)
+		{
+		  curr_prev = curr_info->prev;
+		  /* The first match wins, so skip statements that have
+		     already been replaced, and the final statement with
+		     which they were replaced.  */
+		  if (!STMT_VINFO_IN_PATTERN_P (curr_info)
+		      && !is_main_pattern_stmt_p (curr_info))
+		    vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j],
+					  curr_info);
+		}
+	    }
+      }
 }


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]