[PATCH] Refactor BB vectorization

Richard Biener rguenther@suse.de
Tue Nov 3 15:55:00 GMT 2015


This refactors BB vectorization in preparation to make it work
on sub-BB granularity.

Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.

Richard.

2015-11-03  Richard Biener  <rguenther@suse.de>

	* tree-vect-data-refs.c (vect_analyze_data_refs): Do not collect
	data references here.
	* tree-vect-loop.c: Include cgraph.h.
	(vect_analyze_loop_2): Collect data references here.
	* tree-vect-slp.c (find_bb_location): Inline ...
	(vect_slp_bb): ... here.  Renamed from vect_slp_analyze_bb.
	Factor in vect_slp_transform_bb.
	(vect_slp_transform_bb): Removed.
	(vect_slp_analyze_bb_1): Collect data references here.
	* tree-vectorizer.c (pass_slp_vectorize::execute): Call
	vect_slp_bb.
	* tree-vectorizer.h (vect_slp_bb): Declare.
	(vect_slp_analyze_bb): Remove.
	(vect_slp_transform_bb): Remove.
	(find_bb_location): Remove.
	(vect_analyze_data_refs): Remove stmt count reference parameter.

Index: trunk/gcc/tree-vect-data-refs.c
===================================================================
*** trunk.orig/gcc/tree-vect-data-refs.c	2015-11-03 12:57:31.538588865 +0100
--- trunk/gcc/tree-vect-data-refs.c	2015-11-03 15:05:51.537766823 +0100
*************** vect_check_gather_scatter (gimple *stmt,
*** 3286,3297 ****
  */
  
  bool
! vect_analyze_data_refs (vec_info *vinfo, int *min_vf, unsigned *n_stmts)
  {
    struct loop *loop = NULL;
-   basic_block bb = NULL;
    unsigned int i;
-   vec<data_reference_p> datarefs;
    struct data_reference *dr;
    tree scalar_type;
  
--- 3286,3295 ----
  */
  
  bool
! vect_analyze_data_refs (vec_info *vinfo, int *min_vf)
  {
    struct loop *loop = NULL;
    unsigned int i;
    struct data_reference *dr;
    tree scalar_type;
  
*************** vect_analyze_data_refs (vec_info *vinfo,
*** 3300,3405 ****
                       "=== vect_analyze_data_refs ===\n");
  
    if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
!     {
!       basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! 
!       loop = LOOP_VINFO_LOOP (loop_vinfo);
!       datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
!       if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)))
! 	{
! 	  if (dump_enabled_p ())
! 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! 	                     "not vectorized: loop contains function calls"
! 	                     " or data references that cannot be analyzed\n");
! 	  return false;
! 	}
! 
!       for (i = 0; i < loop->num_nodes; i++)
! 	{
! 	  gimple_stmt_iterator gsi;
! 
! 	  for (gsi = gsi_start_bb (bbs[i]); !gsi_end_p (gsi); gsi_next (&gsi))
! 	    {
! 	      gimple *stmt = gsi_stmt (gsi);
! 	      if (is_gimple_debug (stmt))
! 		continue;
! 	      ++*n_stmts;
! 	      if (!find_data_references_in_stmt (loop, stmt, &datarefs))
! 		{
! 		  if (is_gimple_call (stmt) && loop->safelen)
! 		    {
! 		      tree fndecl = gimple_call_fndecl (stmt), op;
! 		      if (fndecl != NULL_TREE)
! 			{
! 			  struct cgraph_node *node = cgraph_node::get (fndecl);
! 			  if (node != NULL && node->simd_clones != NULL)
! 			    {
! 			      unsigned int j, n = gimple_call_num_args (stmt);
! 			      for (j = 0; j < n; j++)
! 				{
! 				  op = gimple_call_arg (stmt, j);
! 				  if (DECL_P (op)
! 				      || (REFERENCE_CLASS_P (op)
! 					  && get_base_address (op)))
! 				    break;
! 				}
! 			      op = gimple_call_lhs (stmt);
! 			      /* Ignore #pragma omp declare simd functions
! 				 if they don't have data references in the
! 				 call stmt itself.  */
! 			      if (j == n
! 				  && !(op
! 				       && (DECL_P (op)
! 					   || (REFERENCE_CLASS_P (op)
! 					       && get_base_address (op)))))
! 				continue;
! 			    }
! 			}
! 		    }
! 		  LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs;
! 		  if (dump_enabled_p ())
! 		    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! 				     "not vectorized: loop contains function "
! 				     "calls or data references that cannot "
! 				     "be analyzed\n");
! 		  return false;
! 		}
! 	    }
! 	}
! 
!       LOOP_VINFO_DATAREFS (loop_vinfo) = datarefs;
!     }
!   else
!     {
!       bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
!       gimple_stmt_iterator gsi;
! 
!       bb = BB_VINFO_BB (bb_vinfo);
!       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
! 	{
! 	  gimple *stmt = gsi_stmt (gsi);
! 	  if (is_gimple_debug (stmt))
! 	    continue;
! 	  ++*n_stmts;
! 	  if (!find_data_references_in_stmt (NULL, stmt,
! 					     &BB_VINFO_DATAREFS (bb_vinfo)))
! 	    {
! 	      /* Mark the rest of the basic-block as unvectorizable.  */
! 	      for (; !gsi_end_p (gsi); gsi_next (&gsi))
! 		{
! 		  stmt = gsi_stmt (gsi);
! 		  STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
! 		}
! 	      break;
! 	    }
! 	}
! 
!       datarefs = BB_VINFO_DATAREFS (bb_vinfo);
!     }
  
    /* Go through the data-refs, check that the analysis succeeded.  Update
       pointer from stmt_vec_info struct to DR and vectype.  */
  
    FOR_EACH_VEC_ELT (datarefs, i, dr)
      {
        gimple *stmt;
--- 3298,3309 ----
                       "=== vect_analyze_data_refs ===\n");
  
    if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
!     loop = LOOP_VINFO_LOOP (loop_vinfo);
  
    /* Go through the data-refs, check that the analysis succeeded.  Update
       pointer from stmt_vec_info struct to DR and vectype.  */
  
+   vec<data_reference_p> datarefs = vinfo->datarefs;
    FOR_EACH_VEC_ELT (datarefs, i, dr)
      {
        gimple *stmt;
Index: trunk/gcc/tree-vect-loop.c
===================================================================
*** trunk.orig/gcc/tree-vect-loop.c	2015-11-02 12:37:11.007248642 +0100
--- trunk/gcc/tree-vect-loop.c	2015-11-03 13:48:50.290127748 +0100
*************** along with GCC; see the file COPYING3.
*** 46,51 ****
--- 46,52 ----
  #include "tree-scalar-evolution.h"
  #include "tree-vectorizer.h"
  #include "gimple-fold.h"
+ #include "cgraph.h"
  
  /* Loop Vectorization Pass.
  
*************** vect_analyze_loop_2 (loop_vec_info loop_
*** 1584,1596 ****
    unsigned int n_stmts = 0;
  
    /* Find all data references in the loop (which correspond to vdefs/vuses)
!      and analyze their evolution in the loop.  Also adjust the minimal
!      vectorization factor according to the loads and stores.
  
!      FORNOW: Handle only simple, array references, which
!      alignment can be forced, and aligned pointer-references.  */
  
!   ok = vect_analyze_data_refs (loop_vinfo, &min_vf, &n_stmts);
    if (!ok)
      {
        if (dump_enabled_p ())
--- 1585,1658 ----
    unsigned int n_stmts = 0;
  
    /* Find all data references in the loop (which correspond to vdefs/vuses)
!      and analyze their evolution in the loop.  */
  
!   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! 
!   loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
!   if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)))
!     {
!       if (dump_enabled_p ())
! 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! 			 "not vectorized: loop contains function calls"
! 			 " or data references that cannot be analyzed\n");
!       return false;
!     }
! 
!   for (unsigned i = 0; i < loop->num_nodes; i++)
!     for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
! 	 !gsi_end_p (gsi); gsi_next (&gsi))
!       {
! 	gimple *stmt = gsi_stmt (gsi);
! 	if (is_gimple_debug (stmt))
! 	  continue;
! 	++n_stmts;
! 	if (!find_data_references_in_stmt (loop, stmt,
! 					   &LOOP_VINFO_DATAREFS (loop_vinfo)))
! 	  {
! 	    if (is_gimple_call (stmt) && loop->safelen)
! 	      {
! 		tree fndecl = gimple_call_fndecl (stmt), op;
! 		if (fndecl != NULL_TREE)
! 		  {
! 		    cgraph_node *node = cgraph_node::get (fndecl);
! 		    if (node != NULL && node->simd_clones != NULL)
! 		      {
! 			unsigned int j, n = gimple_call_num_args (stmt);
! 			for (j = 0; j < n; j++)
! 			  {
! 			    op = gimple_call_arg (stmt, j);
! 			    if (DECL_P (op)
! 				|| (REFERENCE_CLASS_P (op)
! 				    && get_base_address (op)))
! 			      break;
! 			  }
! 			op = gimple_call_lhs (stmt);
! 			/* Ignore #pragma omp declare simd functions
! 			   if they don't have data references in the
! 			   call stmt itself.  */
! 			if (j == n
! 			    && !(op
! 				 && (DECL_P (op)
! 				     || (REFERENCE_CLASS_P (op)
! 					 && get_base_address (op)))))
! 			  continue;
! 		      }
! 		  }
! 	      }
! 	    if (dump_enabled_p ())
! 	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
! 			       "not vectorized: loop contains function "
! 			       "calls or data references that cannot "
! 			       "be analyzed\n");
! 	    return false;
! 	  }
!       }
! 
!   /* Analyze the data references and also adjust the minimal
!      vectorization factor according to the loads and stores.  */
  
!   ok = vect_analyze_data_refs (loop_vinfo, &min_vf);
    if (!ok)
      {
        if (dump_enabled_p ())
Index: trunk/gcc/tree-vect-slp.c
===================================================================
*** trunk.orig/gcc/tree-vect-slp.c	2015-11-03 12:57:31.538588865 +0100
--- trunk/gcc/tree-vect-slp.c	2015-11-03 14:04:53.845935110 +0100
*************** along with GCC; see the file COPYING3.
*** 40,67 ****
  #include "tree-vectorizer.h"
  #include "langhooks.h"
  #include "gimple-walk.h"
! 
! /* Extract the location of the basic block in the source code.
!    Return the basic block location if succeed and NULL if not.  */
! 
! source_location
! find_bb_location (basic_block bb)
! {
!   gimple *stmt = NULL;
!   gimple_stmt_iterator si;
! 
!   if (!bb)
!     return UNKNOWN_LOCATION;
! 
!   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
!     {
!       stmt = gsi_stmt (si);
!       if (gimple_location (stmt) != UNKNOWN_LOCATION)
!         return gimple_location (stmt);
!     }
! 
!   return UNKNOWN_LOCATION;
! }
  
  
  /* Recursively free the memory allocated for the SLP tree rooted at NODE.  */
--- 40,46 ----
  #include "tree-vectorizer.h"
  #include "langhooks.h"
  #include "gimple-walk.h"
! #include "dbgcnt.h"
  
  
  /* Recursively free the memory allocated for the SLP tree rooted at NODE.  */
*************** vect_slp_analyze_bb_1 (basic_block bb)
*** 2361,2367 ****
    if (!bb_vinfo)
      return NULL;
  
!   if (!vect_analyze_data_refs (bb_vinfo, &min_vf, &n_stmts))
      {
        if (dump_enabled_p ())
          dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
--- 2340,2370 ----
    if (!bb_vinfo)
      return NULL;
  
!   /* Gather all data references in the basic-block.  */
! 
!   for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
!        !gsi_end_p (gsi); gsi_next (&gsi))
!     {
!       gimple *stmt = gsi_stmt (gsi);
!       if (is_gimple_debug (stmt))
! 	continue;
!       ++n_stmts;
!       if (!find_data_references_in_stmt (NULL, stmt,
! 					 &BB_VINFO_DATAREFS (bb_vinfo)))
! 	{
! 	  /* Mark the rest of the basic-block as unvectorizable.  */
! 	  for (; !gsi_end_p (gsi); gsi_next (&gsi))
! 	    {
! 	      stmt = gsi_stmt (gsi);
! 	      STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
! 	    }
! 	  break;
! 	}
!     }
! 
!   /* Analyze the data references.  */
! 
!   if (!vect_analyze_data_refs (bb_vinfo, &min_vf))
      {
        if (dump_enabled_p ())
          dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
*************** vect_slp_analyze_bb_1 (basic_block bb)
*** 2499,2506 ****
  }
  
  
! bb_vec_info
! vect_slp_analyze_bb (basic_block bb)
  {
    bb_vec_info bb_vinfo;
    int insns = 0;
--- 2502,2512 ----
  }
  
  
! /* Main entry for the BB vectorizer.  Analyze and transform BB, returns
!    true if anything in the basic-block was vectorized.  */
! 
! bool
! vect_slp_bb (basic_block bb)
  {
    bb_vec_info bb_vinfo;
    int insns = 0;
*************** vect_slp_analyze_bb (basic_block bb)
*** 2517,2522 ****
--- 2523,2530 ----
            && !gimple_nop_p (stmt)
            && gimple_code (stmt) != GIMPLE_LABEL)
          insns++;
+       if (gimple_location (stmt) != UNKNOWN_LOCATION)
+ 	vect_location = gimple_location (stmt);
      }
  
    if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
*************** vect_slp_analyze_bb (basic_block bb)
*** 2526,2532 ****
  			 "not vectorized: too many instructions in "
  			 "basic block.\n");
  
!       return NULL;
      }
  
    /* Autodetect first vector size we try.  */
--- 2534,2540 ----
  			 "not vectorized: too many instructions in "
  			 "basic block.\n");
  
!       return false;
      }
  
    /* Autodetect first vector size we try.  */
*************** vect_slp_analyze_bb (basic_block bb)
*** 2537,2550 ****
      {
        bb_vinfo = vect_slp_analyze_bb_1 (bb);
        if (bb_vinfo)
!         return bb_vinfo;
  
        destroy_bb_vec_info (bb_vinfo);
  
        vector_sizes &= ~current_vector_size;
        if (vector_sizes == 0
            || current_vector_size == 0)
!         return NULL;
  
        /* Try the next biggest vector size.  */
        current_vector_size = 1 << floor_log2 (vector_sizes);
--- 2545,2577 ----
      {
        bb_vinfo = vect_slp_analyze_bb_1 (bb);
        if (bb_vinfo)
! 	{
! 	  if (!dbg_cnt (vect_slp))
! 	    {
! 	      destroy_bb_vec_info (bb_vinfo);
! 	      return false;
! 	    }
! 
! 	  if (dump_enabled_p ())
! 	    dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
! 
! 	  vect_schedule_slp (bb_vinfo);
! 
! 	  if (dump_enabled_p ())
! 	    dump_printf_loc (MSG_NOTE, vect_location,
! 			     "BASIC BLOCK VECTORIZED\n");
! 
! 	  destroy_bb_vec_info (bb_vinfo);
! 
! 	  return true;
! 	}
  
        destroy_bb_vec_info (bb_vinfo);
  
        vector_sizes &= ~current_vector_size;
        if (vector_sizes == 0
            || current_vector_size == 0)
!         return false;
  
        /* Try the next biggest vector size.  */
        current_vector_size = 1 << floor_log2 (vector_sizes);
*************** vect_schedule_slp (vec_info *vinfo)
*** 3533,3580 ****
  
    return is_store;
  }
- 
- 
- /* Vectorize the basic block.  */
- 
- void
- vect_slp_transform_bb (basic_block bb)
- {
-   bb_vec_info bb_vinfo = vec_info_for_bb (bb);
-   gimple_stmt_iterator si;
- 
-   gcc_assert (bb_vinfo);
- 
-   if (dump_enabled_p ())
-     dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
- 
-   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
-     {
-       gimple *stmt = gsi_stmt (si);
-       stmt_vec_info stmt_info;
- 
-       if (dump_enabled_p ())
-         {
-           dump_printf_loc (MSG_NOTE, vect_location,
-                            "------>SLPing statement: ");
-           dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
-           dump_printf (MSG_NOTE, "\n");
-         }
- 
-       stmt_info = vinfo_for_stmt (stmt);
-       gcc_assert (stmt_info);
- 
-       /* Schedule all the SLP instances when the first SLP stmt is reached.  */
-       if (STMT_SLP_TYPE (stmt_info))
-         {
-           vect_schedule_slp (bb_vinfo);
-           break;
-         }
-     }
- 
-   if (dump_enabled_p ())
-     dump_printf_loc (MSG_NOTE, vect_location,
- 		     "BASIC BLOCK VECTORIZED\n");
- 
-   destroy_bb_vec_info (bb_vinfo);
- }
--- 3560,3562 ----
Index: trunk/gcc/tree-vectorizer.c
===================================================================
*** trunk.orig/gcc/tree-vectorizer.c	2015-11-02 12:37:10.986248408 +0100
--- trunk/gcc/tree-vectorizer.c	2015-11-03 14:03:39.794106576 +0100
*************** pass_slp_vectorize::execute (function *f
*** 696,713 ****
  
    FOR_EACH_BB_FN (bb, fun)
      {
!       vect_location = find_bb_location (bb);
! 
!       if (vect_slp_analyze_bb (bb))
!         {
!           if (!dbg_cnt (vect_slp))
!             break;
! 
!           vect_slp_transform_bb (bb);
!           if (dump_enabled_p ())
!             dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
! 			     "basic block vectorized\n");
!         }
      }
  
    free_stmt_vec_info_vec ();
--- 696,704 ----
  
    FOR_EACH_BB_FN (bb, fun)
      {
!       if (vect_slp_bb (bb))
! 	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
! 			 "basic block vectorized\n");
      }
  
    free_stmt_vec_info_vec ();
Index: trunk/gcc/tree-vectorizer.h
===================================================================
*** trunk.orig/gcc/tree-vectorizer.h	2015-11-03 12:57:31.538588865 +0100
--- trunk/gcc/tree-vectorizer.h	2015-11-03 14:33:42.315259421 +0100
*************** extern bool vect_analyze_data_ref_access
*** 1010,1016 ****
  extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
  extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *,
  				       int *);
! extern bool vect_analyze_data_refs (vec_info *, int *, unsigned *);
  extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
  				      tree *, gimple_stmt_iterator *,
  				      gimple **, bool, bool *,
--- 1010,1016 ----
  extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
  extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *,
  				       int *);
! extern bool vect_analyze_data_refs (vec_info *, int *);
  extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
  				      tree *, gimple_stmt_iterator *,
  				      gimple **, bool, bool *,
*************** extern bool vect_make_slp_decision (loop
*** 1072,1081 ****
  extern void vect_detect_hybrid_slp (loop_vec_info);
  extern void vect_get_slp_defs (vec<tree> , slp_tree,
  			       vec<vec<tree> > *, int);
! 
! extern source_location find_bb_location (basic_block);
! extern bb_vec_info vect_slp_analyze_bb (basic_block);
! extern void vect_slp_transform_bb (basic_block);
  
  /* In tree-vect-patterns.c.  */
  /* Pattern recognition functions.
--- 1072,1078 ----
  extern void vect_detect_hybrid_slp (loop_vec_info);
  extern void vect_get_slp_defs (vec<tree> , slp_tree,
  			       vec<vec<tree> > *, int);
! extern bool vect_slp_bb (basic_block);
  
  /* In tree-vect-patterns.c.  */
  /* Pattern recognition functions.



More information about the Gcc-patches mailing list