This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[PATCH] More vectorizer reduction TLC


Upcoming refactoring likes the cond reduction special IV more in the
epilogue generation code, thus moved there.  Some more TLC as well.

Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.

Richard.

2017-06-28  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vectorizable_reduction): Move special
	cond reduction IV var creation ...
	(vect_create_epilog_for_reduction): ... here.  Remove induction_index
	parameter.  Use STMT_VINFO_VECTYPE.
	* tree-vect-slp.c (vect_get_constant_vectors): Properly reset
	constant_p.

Index: gcc/tree-vect-loop.c
===================================================================
--- gcc/tree-vect-loop.c	(revision 249735)
+++ gcc/tree-vect-loop.c	(working copy)
@@ -4263,8 +4263,6 @@ get_initial_defs_for_reduction (slp_tree
    DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
    SLP_NODE is an SLP node containing a group of reduction statements. The 
      first one in this group is STMT.
-   INDUCTION_INDEX is the index of the loop for condition reductions.
-     Otherwise it is undefined.
 
    This function:
    1. Creates the reduction def-use cycles: sets the arguments for 
@@ -4310,7 +4308,7 @@ vect_create_epilog_for_reduction (vec<tr
 				  int ncopies, enum tree_code reduc_code,
 				  vec<gimple *> reduction_phis,
                                   int reduc_index, bool double_reduc, 
-				  slp_tree slp_node, tree induction_index)
+				  slp_tree slp_node)
 {
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
   stmt_vec_info prev_phi_info;
@@ -4331,7 +4329,7 @@ vect_create_epilog_for_reduction (vec<tr
   tree bitsize;
   tree adjustment_def = NULL;
   tree vec_initial_def = NULL;
-  tree reduction_op, expr, def, initial_def = NULL;
+  tree expr, def, initial_def = NULL;
   tree orig_name, scalar_result;
   imm_use_iterator imm_iter, phi_imm_iter;
   use_operand_p use_p, phi_use_p;
@@ -4348,6 +4346,7 @@ vect_create_epilog_for_reduction (vec<tr
   bool slp_reduc = false;
   tree new_phi_result;
   gimple *inner_phi = NULL;
+  tree induction_index = NULL_TREE;
 
   if (slp_node)
     group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); 
@@ -4360,9 +4359,7 @@ vect_create_epilog_for_reduction (vec<tr
       gcc_assert (!slp_node);
     }
 
-  reduction_op = get_reduction_op (stmt, reduc_index);
-
-  vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
+  vectype = STMT_VINFO_VECTYPE (stmt_info);
   gcc_assert (vectype);
   mode = TYPE_MODE (vectype);
 
@@ -4396,6 +4393,7 @@ vect_create_epilog_for_reduction (vec<tr
     {
       /* Get at the scalar def before the loop, that defines the initial value
 	 of the reduction variable.  */
+      tree reduction_op = get_reduction_op (stmt, reduc_index);
       gimple *def_stmt = SSA_NAME_DEF_STMT (reduction_op);
       initial_def = PHI_ARG_DEF_FROM_EDGE (def_stmt,
 					   loop_preheader_edge (loop));
@@ -4465,6 +4463,95 @@ vect_create_epilog_for_reduction (vec<tr
         }
     }
 
+  /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
+     which is updated with the current index of the loop for every match of
+     the original loop's cond_expr (VEC_STMT).  This results in a vector
+     containing the last time the condition passed for that vector lane.
+     The first match will be a 1 to allow 0 to be used for non-matching
+     indexes.  If there are no matches at all then the vector will be all
+     zeroes.  */
+  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
+    {
+      tree indx_before_incr, indx_after_incr;
+      int nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
+      int k;
+
+      gimple *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+      gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
+
+      int scalar_precision
+	= GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (vectype)));
+      tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
+      tree cr_index_vector_type = build_vector_type
+	(cr_index_scalar_type, TYPE_VECTOR_SUBPARTS (vectype));
+
+      /* First we create a simple vector induction variable which starts
+	 with the values {1,2,3,...} (SERIES_VECT) and increments by the
+	 vector size (STEP).  */
+
+      /* Create a {1,2,3,...} vector.  */
+      tree *vtemp = XALLOCAVEC (tree, nunits_out);
+      for (k = 0; k < nunits_out; ++k)
+	vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
+      tree series_vect = build_vector (cr_index_vector_type, vtemp);
+
+      /* Create a vector of the step value.  */
+      tree step = build_int_cst (cr_index_scalar_type, nunits_out);
+      tree vec_step = build_vector_from_val (cr_index_vector_type, step);
+
+      /* Create an induction variable.  */
+      gimple_stmt_iterator incr_gsi;
+      bool insert_after;
+      standard_iv_increment_position (loop, &incr_gsi, &insert_after);
+      create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
+		 insert_after, &indx_before_incr, &indx_after_incr);
+
+      /* Next create a new phi node vector (NEW_PHI_TREE) which starts
+	 filled with zeros (VEC_ZERO).  */
+
+      /* Create a vector of 0s.  */
+      tree zero = build_zero_cst (cr_index_scalar_type);
+      tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
+
+      /* Create a vector phi node.  */
+      tree new_phi_tree = make_ssa_name (cr_index_vector_type);
+      new_phi = create_phi_node (new_phi_tree, loop->header);
+      set_vinfo_for_stmt (new_phi,
+			  new_stmt_vec_info (new_phi, loop_vinfo));
+      add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
+		   loop_preheader_edge (loop), UNKNOWN_LOCATION);
+
+      /* Now take the condition from the loops original cond_expr
+	 (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
+	 every match uses values from the induction variable
+	 (INDEX_BEFORE_INCR) otherwise uses values from the phi node
+	 (NEW_PHI_TREE).
+	 Finally, we update the phi (NEW_PHI_TREE) to take the value of
+	 the new cond_expr (INDEX_COND_EXPR).  */
+
+      /* Duplicate the condition from vec_stmt.  */
+      tree ccompare = unshare_expr (gimple_assign_rhs1 (vec_stmt));
+
+      /* Create a conditional, where the condition is taken from vec_stmt
+	 (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
+	 else is the phi (NEW_PHI_TREE).  */
+      tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
+				     ccompare, indx_before_incr,
+				     new_phi_tree);
+      induction_index = make_ssa_name (cr_index_vector_type);
+      gimple *index_condition = gimple_build_assign (induction_index,
+						     index_cond_expr);
+      gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
+      stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
+							loop_vinfo);
+      STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
+      set_vinfo_for_stmt (index_condition, index_vec_info);
+
+      /* Update the phi with the vec cond.  */
+      add_phi_arg (as_a <gphi *> (new_phi), induction_index,
+		   loop_latch_edge (loop), UNKNOWN_LOCATION);
+    }
+
   /* 2. Create epilog code.
         The reduction epilog code operates across the elements of the vector
         of partial results computed by the vectorized loop.
@@ -6248,100 +6335,14 @@ vectorizable_reduction (gimple *stmt, gi
       prev_phi_info = vinfo_for_stmt (new_phi);
     }
 
-  tree indx_before_incr, indx_after_incr, cond_name = NULL;
-
   /* Finalize the reduction-phi (set its arguments) and create the
      epilog reduction code.  */
   if ((!single_defuse_cycle || code == COND_EXPR) && !slp_node)
-    {
-      new_temp = gimple_assign_lhs (*vec_stmt);
-      vect_defs[0] = new_temp;
-
-      /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
-	 which is updated with the current index of the loop for every match of
-	 the original loop's cond_expr (VEC_STMT).  This results in a vector
-	 containing the last time the condition passed for that vector lane.
-	 The first match will be a 1 to allow 0 to be used for non-matching
-	 indexes.  If there are no matches at all then the vector will be all
-	 zeroes.  */
-      if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
-	{
-	  int nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
-	  int k;
-
-	  gcc_assert (gimple_assign_rhs_code (*vec_stmt) == VEC_COND_EXPR);
-
-	  /* First we create a simple vector induction variable which starts
-	     with the values {1,2,3,...} (SERIES_VECT) and increments by the
-	     vector size (STEP).  */
-
-	  /* Create a {1,2,3,...} vector.  */
-	  tree *vtemp = XALLOCAVEC (tree, nunits_out);
-	  for (k = 0; k < nunits_out; ++k)
-	    vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
-	  tree series_vect = build_vector (cr_index_vector_type, vtemp);
-
-	  /* Create a vector of the step value.  */
-	  tree step = build_int_cst (cr_index_scalar_type, nunits_out);
-	  tree vec_step = build_vector_from_val (cr_index_vector_type, step);
-
-	  /* Create an induction variable.  */
-	  gimple_stmt_iterator incr_gsi;
-	  bool insert_after;
-	  standard_iv_increment_position (loop, &incr_gsi, &insert_after);
-	  create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
-		     insert_after, &indx_before_incr, &indx_after_incr);
-
-	  /* Next create a new phi node vector (NEW_PHI_TREE) which starts
-	     filled with zeros (VEC_ZERO).  */
-
-	  /* Create a vector of 0s.  */
-	  tree zero = build_zero_cst (cr_index_scalar_type);
-	  tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
-
-	  /* Create a vector phi node.  */
-	  tree new_phi_tree = make_ssa_name (cr_index_vector_type);
-	  new_phi = create_phi_node (new_phi_tree, loop->header);
-	  set_vinfo_for_stmt (new_phi,
-			      new_stmt_vec_info (new_phi, loop_vinfo));
-	  add_phi_arg (new_phi, vec_zero, loop_preheader_edge (loop),
-		       UNKNOWN_LOCATION);
-
-	  /* Now take the condition from the loops original cond_expr
-	     (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
-	     every match uses values from the induction variable
-	     (INDEX_BEFORE_INCR) otherwise uses values from the phi node
-	     (NEW_PHI_TREE).
-	     Finally, we update the phi (NEW_PHI_TREE) to take the value of
-	     the new cond_expr (INDEX_COND_EXPR).  */
-
-	  /* Duplicate the condition from vec_stmt.  */
-	  tree ccompare = unshare_expr (gimple_assign_rhs1 (*vec_stmt));
-
-	  /* Create a conditional, where the condition is taken from vec_stmt
-	     (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
-	     else is the phi (NEW_PHI_TREE).  */
-	  tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
-					 ccompare, indx_before_incr,
-					 new_phi_tree);
-	  cond_name = make_ssa_name (cr_index_vector_type);
-	  gimple *index_condition = gimple_build_assign (cond_name,
-							 index_cond_expr);
-	  gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
-	  stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
-							    loop_vinfo);
-	  STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
-	  set_vinfo_for_stmt (index_condition, index_vec_info);
-
-	  /* Update the phi with the vec cond.  */
-	  add_phi_arg (new_phi, cond_name, loop_latch_edge (loop),
-		       UNKNOWN_LOCATION);
-	}
-    }
+    vect_defs[0] = gimple_assign_lhs (*vec_stmt);
 
   vect_create_epilog_for_reduction (vect_defs, stmt, epilog_copies,
                                     epilog_reduc_code, phis, reduc_index,
-				    double_reduc, slp_node, cond_name);
+				    double_reduc, slp_node);
 
   return true;
 }
Index: gcc/tree-vect-slp.c
===================================================================
--- gcc/tree-vect-slp.c	(revision 249735)
+++ gcc/tree-vect-slp.c	(working copy)
@@ -3016,11 +3016,6 @@ vect_get_constant_vectors (tree op, slp_
 
   gcc_assert (op);
 
-  if (CONSTANT_CLASS_P (op))
-    constant_p = true;
-  else
-    constant_p = false;
-
   /* NUMBER_OF_COPIES is the number of times we need to use the same values in
      created vectors. It is greater than 1 if unrolling is performed.
 
@@ -3040,6 +3035,7 @@ vect_get_constant_vectors (tree op, slp_
   number_of_copies = nunits * number_of_vectors / group_size;
 
   number_of_places_left_in_vector = nunits;
+  constant_p = true;
   elts = XALLOCAVEC (tree, nunits);
   bool place_after_defs = false;
   for (j = 0; j < number_of_copies; j++)
@@ -3156,8 +3152,6 @@ vect_get_constant_vectors (tree op, slp_
 
           if (number_of_places_left_in_vector == 0)
             {
-              number_of_places_left_in_vector = nunits;
-
 	      if (constant_p)
 		vec_cst = build_vector (vector_type, elts);
 	      else
@@ -3188,6 +3182,8 @@ vect_get_constant_vectors (tree op, slp_
 		}
 	      voprnds.quick_push (init);
 	      place_after_defs = false;
+              number_of_places_left_in_vector = nunits;
+	      constant_p = true;
             }
         }
     }


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]