This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Re: [patch, rfc] Make store motion use alias oracle


Hello,

> > here is the updated version of the patch that makes us use alias oracle
> > for both invariant motion and store motion, and adds a few trivial
> > improvements to the oracle, thus making it fix the fatigue problem
> > (I hope; I did not actually run the benchmark, I just looked at the
> > dumps).
> 
> After the patch has been applied to trunk it fails with ICE.  Here is
> a reduced test case:

a fixed version.

Zdenek

Index: tree-ssa-loop-im.c
===================================================================
*** tree-ssa-loop-im.c	(revision 121421)
--- tree-ssa-loop-im.c	(working copy)
*************** Software Foundation, 51 Franklin Street,
*** 39,44 ****
--- 39,45 ----
  #include "flags.h"
  #include "real.h"
  #include "hashtab.h"
+ #include "tree-affine.h"
  
  /* TODO:  Support for predicated code motion.  I.e.
  
*************** struct lim_aux_data
*** 86,95 ****
  				   the statement is executed if the loop
  				   is entered.  */
  
-   bool sm_done;			/* True iff the store motion for a memory
- 				   reference in the statement has already
- 				   been executed.  */
- 
    unsigned cost;		/* Cost of the computation performed by the
  				   statement.  */
  
--- 87,92 ----
*************** struct lim_aux_data
*** 104,137 ****
  			? NULL \
  			: (struct lim_aux_data *) (stmt_ann (STMT)->common.aux))
  
! /* Description of a memory reference location for store motion.  */
  
! struct mem_ref_loc
  {
    tree *ref;			/* The reference itself.  */
    tree stmt;			/* The statement in that it occurs.  */
!   struct mem_ref_loc *next;	/* Next use in the chain.  */
! };
  
! /* Description of a memory reference for store motion.  */
  
! struct mem_ref
  {
    tree mem;			/* The memory itself.  */
    hashval_t hash;		/* Its hash value.  */
!   bool is_stored;		/* True if there is a store to the location
! 				   in the loop.  */
!   struct mem_ref_loc *locs;	/* The locations where it is found.  */
    bitmap vops;			/* Vops corresponding to this memory
  				   location.  */
!   struct mem_ref *next;		/* Next memory reference in the list.
! 				   Memory references are stored in a hash
! 				   table, but the hash function depends
! 				   on values of pointers. Thus we cannot use
! 				   htab_traverse, since then we would get
! 				   miscompares during bootstrap (although the
! 				   produced code would be correct).  */
! };
  
  /* Minimum cost of an expensive expression.  */
  #define LIM_EXPENSIVE ((unsigned) PARAM_VALUE (PARAM_LIM_EXPENSIVE))
--- 101,198 ----
  			? NULL \
  			: (struct lim_aux_data *) (stmt_ann (STMT)->common.aux))
  
! /* Description of a memory reference location.  */
  
! typedef struct mem_ref_loc
  {
    tree *ref;			/* The reference itself.  */
    tree stmt;			/* The statement in that it occurs.  */
! } *mem_ref_loc_p;
! 
! DEF_VEC_P(mem_ref_loc_p);
! DEF_VEC_ALLOC_P(mem_ref_loc_p, heap);
! 
! /* The list of memory reference locations in a loop.  */
  
! typedef struct mem_ref_locs
! {
!   VEC (mem_ref_loc_p, heap) *locs;
! } *mem_ref_locs_p;
! 
! DEF_VEC_P(mem_ref_locs_p);
! DEF_VEC_ALLOC_P(mem_ref_locs_p, heap);
  
! /* Description of a memory reference.  */
! 
! typedef struct mem_ref
  {
    tree mem;			/* The memory itself.  */
+   unsigned id;			/* ID assigned to the memory reference
+ 				   (its index in memory_accesses.refs_list)  */
    hashval_t hash;		/* Its hash value.  */
!   bitmap stored;		/* The set of loops in that this memory locatio
! 				   is stored to.  */
!   VEC (mem_ref_locs_p, heap) *accesses_in_loop;
! 				/* The locations of the accesses.  Vector
! 				   indexed by the loop number.  */
    bitmap vops;			/* Vops corresponding to this memory
  				   location.  */
! 
!   /* The following sets are computed on demand.  We keep both set and
!      its complement, so that we know whether the information was
!      already computed or not.  */
!   bitmap indep_loop;		/* The set of loops in that the memory
! 				   reference is independent, meaning:
! 				   If it is stored in the loop, this store
! 				     is independent on all other loads and
! 				     stores.
! 				   If it is only loaded, then it is independent
! 				     on all stores in the loop.  */
!   bitmap dep_loop;		/* The complement of INDEP_LOOP.  */
! 
!   bitmap indep_ref;		/* The set of memory references on that
! 				   this reference is independent.  */
!   bitmap dep_ref;		/* The complement of DEP_REF.  */
! } *mem_ref_p;
! 
! DEF_VEC_P(mem_ref_p);
! DEF_VEC_ALLOC_P(mem_ref_p, heap);
! 
! DEF_VEC_P(bitmap);
! DEF_VEC_ALLOC_P(bitmap, heap);
! 
! DEF_VEC_P(htab_t);
! DEF_VEC_ALLOC_P(htab_t, heap);
! 
! /* Description of memory accesses in loops.  */
! 
! static struct
! {
!   /* The hash table of memory references accessed in loops.  */
!   htab_t refs;
! 
!   /* The list of memory references.  */
!   VEC (mem_ref_p, heap) *refs_list;
! 
!   /* The set of memory references accessed in each loop.  */
!   VEC (bitmap, heap) *refs_in_loop;
! 
!   /* The set of memory references accessed in each loop, including
!      subloops.  */
!   VEC (bitmap, heap) *all_refs_in_loop;
! 
!   /* The set of virtual operands clobbered in a given loop.  */
!   VEC (bitmap, heap) *clobbered_vops;
! 
!   /* Map from the pair (loop, virtual operand) to the set of refs that
!      touch the virtual operand in the loop.  */
!   VEC (htab_t, heap) *vop_ref_map;
! 
!   /* Cache for expanding memory addresses.  */
!   htab_t ttae_cache;
! } memory_accesses;
! 
! static bool ref_indep_loop_p (struct loop *, mem_ref_p);
  
  /* Minimum cost of an expensive expression.  */
  #define LIM_EXPENSIVE ((unsigned) PARAM_VALUE (PARAM_LIM_EXPENSIVE))
*************** movement_possibility (tree stmt)
*** 247,252 ****
--- 308,316 ----
    if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
      return MOVE_IMPOSSIBLE;
  
+   if (!ZERO_SSA_OPERANDS (stmt, SSA_OP_VIRTUAL_DEFS))
+     return MOVE_IMPOSSIBLE;
+ 
    if (stmt_ends_bb_p (stmt))
      return MOVE_IMPOSSIBLE;
  
*************** stmt_cost (tree stmt)
*** 466,471 ****
--- 530,615 ----
    return cost;
  }
  
+ /* Finds the outermost loop between OUTER and LOOP in that the memory reference
+    REF is independent.  If REF is not independent in LOOP, NULL is returned
+    instead.  */
+ 
+ static struct loop *
+ outermost_indep_loop (struct loop *outer, struct loop *loop, mem_ref_p ref)
+ {
+   struct loop *aloop;
+ 
+   if (bitmap_bit_p (ref->stored, loop->num))
+     return NULL;
+ 
+   for (aloop = outer;
+        aloop != loop;
+        aloop = superloop_at_depth (loop, aloop->depth + 1))
+     if (!bitmap_bit_p (ref->stored, aloop->num)
+ 	&& ref_indep_loop_p (aloop, ref))
+       return aloop;
+ 
+   if (ref_indep_loop_p (loop, ref))
+     return loop;
+   else
+     return NULL;
+ }
+ 
+ /* If there is a simple load or store to a memory reference in STMT, returns
+    the location of the memory reference, and sets IS_STORE accoring to whether
+    it is a store or load.  Otherwise, returns NULL.  */
+ 
+ static tree *
+ simple_mem_ref_in_stmt (tree stmt, bool *is_store)
+ {
+   tree *lhs, *rhs;
+ 
+   /* Recognize MEM = (SSA_NAME | invariant) and SSA_NAME = MEM patterns.  */
+   if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+     return NULL;
+ 
+   lhs = &GIMPLE_STMT_OPERAND (stmt, 0);
+   rhs = &GIMPLE_STMT_OPERAND (stmt, 1);
+ 
+   if (TREE_CODE (*lhs) == SSA_NAME)
+     {
+       if (!is_gimple_addressable (*rhs))
+ 	return NULL;
+ 
+       *is_store = false;
+       return rhs;
+     }
+   else if (TREE_CODE (*rhs) == SSA_NAME
+ 	   || is_gimple_min_invariant (*rhs))
+     {
+       *is_store = true;
+       return lhs;
+     }
+   else
+     return NULL;
+ }
+ 
+ /* Returns the memory reference contained in STMT.  */
+ 
+ static mem_ref_p
+ mem_ref_in_stmt (tree stmt)
+ {
+   bool store;
+   tree *mem = simple_mem_ref_in_stmt (stmt, &store);
+   hashval_t hash;
+   mem_ref_p ref;
+ 
+   if (!mem)
+     return NULL;
+   gcc_assert (!store);
+ 
+   hash = iterative_hash_expr (*mem, 0);
+   ref = htab_find_with_hash (memory_accesses.refs, *mem, hash);
+ 
+   gcc_assert (ref != NULL);
+   return ref;
+ }
+ 
  /* Determine the outermost loop to that it is possible to hoist a statement
     STMT and store it to LIM_DATA (STMT)->max_loop.  To do this we determine
     the outermost loop in that the value computed by STMT is invariant.
*************** determine_max_movement (tree stmt, bool 
*** 496,504 ****
      if (!add_dependency (val, lim_data, loop, true))
        return false;
  
!   FOR_EACH_SSA_TREE_OPERAND (val, stmt, iter, SSA_OP_VIRTUAL_USES)
!     if (!add_dependency (val, lim_data, loop, false))
!       return false;
  
    lim_data->cost += stmt_cost (stmt);
  
--- 640,665 ----
      if (!add_dependency (val, lim_data, loop, true))
        return false;
  
!   if (!ZERO_SSA_OPERANDS (stmt, SSA_OP_VIRTUAL_USES))
!     {
!       mem_ref_p ref = mem_ref_in_stmt (stmt);
! 
!       if (ref)
! 	{
! 	  lim_data->max_loop
! 		  = outermost_indep_loop (lim_data->max_loop, loop, ref);
! 	  if (!lim_data->max_loop)
! 	    return false;
! 	}
!       else
! 	{
! 	  FOR_EACH_SSA_TREE_OPERAND (val, stmt, iter, SSA_OP_VIRTUAL_USES)
! 	    {
! 	      if (!add_dependency (val, lim_data, loop, false))
! 		return false;
! 	    }
! 	}
!     }
  
    lim_data->cost += stmt_cost (stmt);
  
*************** determine_invariantness (void)
*** 690,695 ****
--- 851,872 ----
    fini_walk_dominator_tree (&walk_data);
  }
  
+ /* Marks all virtual operands of the statement STMT for renaming.  */
+ 
+ static void
+ mark_virtual_ops_for_renaming (tree stmt)
+ {
+   tree var;
+   ssa_op_iter iter;
+ 
+   FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_ALL_VIRTUALS)
+     {
+       if (TREE_CODE (var) == SSA_NAME)
+ 	var = SSA_NAME_VAR (var);
+       mark_sym_for_renaming (var);
+     }
+ }
+ 
  /* Hoist the statements in basic block BB out of the loops prescribed by
     data stored in LIM_DATA structures associated with each statement.  Callback
     for walk_dominator_tree.  */
*************** move_computations_stmt (struct dom_walk_
*** 739,744 ****
--- 916,923 ----
  	  fprintf (dump_file, "(cost %u) out of loop %d.\n\n",
  		   cost, level->num);
  	}
+ 
+       mark_virtual_ops_for_renaming (stmt);
        bsi_insert_on_edge (loop_preheader_edge (level), stmt);
        bsi_remove (&bsi, false);
      }
*************** force_move_till (tree ref, tree *index, 
*** 862,915 ****
    return true;
  }
  
! /* Records memory reference location *REF to the list MEM_REFS.  The reference
!    occurs in statement STMT.  */
  
  static void
! record_mem_ref_loc (struct mem_ref_loc **mem_refs, tree stmt, tree *ref)
  {
!   struct mem_ref_loc *aref = XNEW (struct mem_ref_loc);
  
    aref->stmt = stmt;
!   aref->ref = ref;
  
!   aref->next = *mem_refs;
!   *mem_refs = aref;
  }
  
! /* Releases list of memory reference locations MEM_REFS.  */
  
  static void
! free_mem_ref_locs (struct mem_ref_loc *mem_refs)
  {
!   struct mem_ref_loc *act;
  
!   while (mem_refs)
      {
!       act = mem_refs;
!       mem_refs = mem_refs->next;
!       free (act);
      }
  }
  
! /* Rewrites memory references in list MEM_REFS by variable TMP_VAR.  */
  
  static void
! rewrite_mem_refs (tree tmp_var, struct mem_ref_loc *mem_refs)
  {
!   tree var;
!   ssa_op_iter iter;
  
!   for (; mem_refs; mem_refs = mem_refs->next)
      {
!       FOR_EACH_SSA_TREE_OPERAND (var, mem_refs->stmt, iter, SSA_OP_ALL_VIRTUALS)
! 	mark_sym_for_renaming (SSA_NAME_VAR (var));
  
!       *mem_refs->ref = tmp_var;
!       update_stmt (mem_refs->stmt);
      }
  }
  
  /* The name and the length of the currently generated variable
     for lsm.  */
  #define MAX_LSM_NAME_LENGTH 40
--- 1041,1611 ----
    return true;
  }
  
! /* A hash function for struct mem_ref object OBJ.  */
! 
! static hashval_t
! memref_hash (const void *obj)
! {
!   const struct mem_ref *mem = obj;
! 
!   return mem->hash;
! }
! 
! /* An equality function for struct mem_ref object OBJ1 with
!    memory reference OBJ2.  */
! 
! static int
! memref_eq (const void *obj1, const void *obj2)
! {
!   const struct mem_ref *mem1 = obj1;
! 
!   return operand_equal_p (mem1->mem, (tree) obj2, 0);
! }
! 
! /* Releases list of memory reference locations ACCS.  */
! 
! static void
! free_mem_ref_locs (mem_ref_locs_p accs)
! {
!   unsigned i;
!   mem_ref_loc_p loc;
! 
!   if (!accs)
!     return;
! 
!   for (i = 0; VEC_iterate (mem_ref_loc_p, accs->locs, i, loc); i++)
!     free (loc);
!   VEC_free (mem_ref_loc_p, heap, accs->locs);
!   free (accs);
! }
! 
! /* A function to free the mem_ref object OBJ.  */
! 
! static void
! memref_free (void *obj)
! {
!   struct mem_ref *mem = obj;
!   unsigned i;
!   mem_ref_locs_p accs;
! 
!   BITMAP_FREE (mem->stored);
!   BITMAP_FREE (mem->indep_loop);
!   BITMAP_FREE (mem->dep_loop);
!   BITMAP_FREE (mem->indep_ref);
!   BITMAP_FREE (mem->dep_ref);
! 
!   for (i = 0; VEC_iterate (mem_ref_locs_p, mem->accesses_in_loop, i, accs); i++)
!     free_mem_ref_locs (accs);
!   VEC_free (mem_ref_locs_p, heap, mem->accesses_in_loop);
! 
!   BITMAP_FREE (mem->vops);
!   free (mem);
! }
! 
! /* Allocates and returns a memory reference description for MEM whose hash
!    value is HASH and id is ID.  */
! 
! static mem_ref_p
! mem_ref_alloc (tree mem, unsigned hash, unsigned id)
! {
!   mem_ref_p ref = XNEW (struct mem_ref);
!   ref->mem = mem;
!   ref->id = id;
!   ref->hash = hash;
!   ref->stored = BITMAP_ALLOC (NULL);
!   ref->indep_loop = BITMAP_ALLOC (NULL);
!   ref->dep_loop = BITMAP_ALLOC (NULL);
!   ref->indep_ref = BITMAP_ALLOC (NULL);
!   ref->dep_ref = BITMAP_ALLOC (NULL);
!   ref->accesses_in_loop = NULL;
!   ref->vops = BITMAP_ALLOC (NULL);
! 
!   return ref;
! }
! 
! /* Allocates and returns the new list of locations.  */
! 
! static mem_ref_locs_p
! mem_ref_locs_alloc (void)
! {
!   mem_ref_locs_p accs = XNEW (struct mem_ref_locs);
!   accs->locs = NULL;
!   return accs;
! }
! 
! /* Records memory reference location *LOC in LOOP to the memory reference
!    description REF.  The reference occurs in statement STMT.  */
  
  static void
! record_mem_ref_loc (mem_ref_p ref, struct loop *loop, tree stmt, tree *loc)
  {
!   mem_ref_loc_p aref = XNEW (struct mem_ref_loc);
!   mem_ref_locs_p accs;
!   bitmap ril = VEC_index (bitmap, memory_accesses.refs_in_loop, loop->num);
! 
!   if (VEC_length (mem_ref_locs_p, ref->accesses_in_loop)
!       <= (unsigned) loop->num)
!     VEC_safe_grow_cleared (mem_ref_locs_p, heap, ref->accesses_in_loop,
! 			   loop->num + 1);
!   accs = VEC_index (mem_ref_locs_p, ref->accesses_in_loop, loop->num);
!   if (!accs)
!     {
!       accs = mem_ref_locs_alloc ();
!       VEC_replace (mem_ref_locs_p, ref->accesses_in_loop, loop->num, accs);
!     }
  
    aref->stmt = stmt;
!   aref->ref = loc;
  
!   VEC_safe_push (mem_ref_loc_p, heap, accs->locs, aref);
!   bitmap_set_bit (ril, ref->id);
  }
  
! /* Marks reference REF as stored in LOOP.  */
  
  static void
! mark_ref_stored (mem_ref_p ref, struct loop *loop)
  {
!   for (;
!        loop != current_loops->tree_root
!        && !bitmap_bit_p (ref->stored, loop->num);
!        loop = loop->outer)
!     bitmap_set_bit (ref->stored, loop->num);
! }
! 
! /* Gathers memory references in statement STMT in LOOP, storing the
!    information about them in the memory_accesses structure.  Marks
!    the vops accessed through unrecognized statements there as
!    well.  */
! 
! static void
! gather_mem_refs_stmt (struct loop *loop, tree stmt)
! {
!   tree *mem = NULL;
!   hashval_t hash;
!   PTR *slot;
!   mem_ref_p ref;
!   ssa_op_iter oi;
!   tree vname;
!   bool is_stored;
!   bitmap clvops;
!   unsigned id;
! 
!   if (ZERO_SSA_OPERANDS (stmt, SSA_OP_ALL_VIRTUALS))
!     return;
! 
!   mem = simple_mem_ref_in_stmt (stmt, &is_stored);
!   if (!mem)
!     goto fail;
  
!   hash = iterative_hash_expr (*mem, 0);
!   slot = htab_find_slot_with_hash (memory_accesses.refs, *mem, hash, INSERT);
! 
!   if (*slot)
!     {
!       ref = *slot;
!       id = ref->id;
!     }
!   else
      {
!       id = VEC_length (mem_ref_p, memory_accesses.refs_list);
!       ref = mem_ref_alloc (*mem, hash, id);
!       VEC_safe_push (mem_ref_p, heap, memory_accesses.refs_list, ref);
!       *slot = ref;
!   
!       if (dump_file && (dump_flags & TDF_DETAILS))
! 	{
! 	  fprintf (dump_file, "Memory reference %u: ", id);
! 	  print_generic_expr (dump_file, ref->mem, TDF_SLIM);
! 	  fprintf (dump_file, "\n");
! 	}
      }
+   if (is_stored)
+     mark_ref_stored (ref, loop);
+ 
+   FOR_EACH_SSA_TREE_OPERAND (vname, stmt, oi, SSA_OP_VIRTUAL_USES)
+     bitmap_set_bit (ref->vops, DECL_UID (SSA_NAME_VAR (vname)));
+   record_mem_ref_loc (ref, loop, stmt, mem);
+   return;
+ 
+ fail:
+   clvops = VEC_index (bitmap, memory_accesses.clobbered_vops, loop->num);
+   FOR_EACH_SSA_TREE_OPERAND (vname, stmt, oi, SSA_OP_VIRTUAL_USES)
+     bitmap_set_bit (clvops, DECL_UID (SSA_NAME_VAR (vname)));
  }
  
! /* Gathers memory references in loops.  */
  
  static void
! gather_mem_refs_in_loops (void)
  {
!   block_stmt_iterator bsi;
!   basic_block bb;
!   struct loop *loop;
!   loop_iterator li;
!   bitmap clvo, clvi;
!   bitmap lrefs, alrefs, alrefso;
! 
!   FOR_EACH_BB (bb)
!     {
!       loop = bb->loop_father;
!       if (loop == current_loops->tree_root)
! 	continue;
! 	
!       for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
! 	gather_mem_refs_stmt (loop, bsi_stmt (bsi));
!     }
! 
!   /* Propagate the information about clobbered vops and accessed memory
!      references up the loop hierarchy.  */
!   FOR_EACH_LOOP (li, loop, LI_FROM_INNERMOST)
!     {
!       lrefs = VEC_index (bitmap, memory_accesses.refs_in_loop, loop->num);
!       alrefs = VEC_index (bitmap, memory_accesses.all_refs_in_loop, loop->num);
!       bitmap_ior_into (alrefs, lrefs);
! 
!       if (loop->outer == current_loops->tree_root)
! 	continue;
! 
!       clvi = VEC_index (bitmap, memory_accesses.clobbered_vops, loop->num);
!       clvo = VEC_index (bitmap, memory_accesses.clobbered_vops,
! 			loop->outer->num);
!       bitmap_ior_into (clvo, clvi);
! 
!       alrefso = VEC_index (bitmap, memory_accesses.all_refs_in_loop,
! 			   loop->outer->num);
!       bitmap_ior_into (alrefso, alrefs);
!     }
! }
! 
! /* Element of the hash table that maps vops to memory references.  */
! 
! struct vop_to_refs_elt
! {
!   /* DECL_UID of the vop.  */
!   unsigned uid;
! 
!   /* List of the all references.  */
!   bitmap refs_all;
! 
!   /* List of stored references.  */
!   bitmap refs_stored;
! };
! 
! /* A hash function for struct vop_to_refs_elt object OBJ.  */
! 
! static hashval_t
! vtoe_hash (const void *obj)
! {
!   const struct vop_to_refs_elt *vtoe = obj;
! 
!   return vtoe->uid;
! }
! 
! /* An equality function for struct vop_to_refs_elt object OBJ1 with
!    uid of a vop OBJ2.  */
! 
! static int
! vtoe_eq (const void *obj1, const void *obj2)
! {
!   const struct vop_to_refs_elt *vtoe = obj1;
!   const unsigned *uid = obj2;
! 
!   return vtoe->uid == *uid;
! }
! 
! /* A function to free the struct vop_to_refs_elt object.  */
! 
! static void
! vtoe_free (void *obj)
! {
!   struct vop_to_refs_elt *vtoe = obj;
! 
!   BITMAP_FREE (vtoe->refs_all);
!   BITMAP_FREE (vtoe->refs_stored);
!   free (vtoe);
! }
! 
! /* Records REF to hashtable VOP_TO_REFS for the index VOP.  STORED is true
!    if the reference REF is stored.  */
! 
! static void
! record_vop_access (htab_t vop_to_refs, unsigned vop, unsigned ref, bool stored)
! {
!   void **slot = htab_find_slot_with_hash (vop_to_refs, &vop, vop, INSERT);
!   struct vop_to_refs_elt *vtoe;
  
!   if (!*slot)
      {
!       vtoe = XNEW (struct vop_to_refs_elt);
!       vtoe->uid = vop;
!       vtoe->refs_all = BITMAP_ALLOC (NULL);
!       vtoe->refs_stored = BITMAP_ALLOC (NULL);
!       *slot = vtoe;
!     }
!   else
!     vtoe = *slot;
! 
!   bitmap_set_bit (vtoe->refs_all, ref);
!   if (stored)
!     bitmap_set_bit (vtoe->refs_stored, ref);
! }
! 
! /* Returns the set of references that access VOP according to the table
!    VOP_TO_REFS.  */
! 
! static bitmap
! get_vop_accesses (htab_t vop_to_refs, unsigned vop)
! {
!   struct vop_to_refs_elt *vtoe = htab_find_with_hash (vop_to_refs, &vop, vop);
!   return vtoe->refs_all;
! }
  
! /* Returns the set of stores that access VOP according to the table
!    VOP_TO_REFS.  */
! 
! static bitmap
! get_vop_stores (htab_t vop_to_refs, unsigned vop)
! {
!   struct vop_to_refs_elt *vtoe = htab_find_with_hash (vop_to_refs, &vop, vop);
!   return vtoe->refs_stored;
! }
! 
! /* Adds REF to mapping from virtual operands to references in LOOP.  */
! 
! static void
! add_vop_ref_mapping (struct loop *loop, mem_ref_p ref)
! {
!   htab_t map = VEC_index (htab_t, memory_accesses.vop_ref_map, loop->num);
!   bool stored = bitmap_bit_p (ref->stored, loop->num);
!   bitmap clobbers = VEC_index (bitmap, memory_accesses.clobbered_vops,
! 			       loop->num);
!   bitmap_iterator bi;
!   unsigned vop;
! 
!   EXECUTE_IF_AND_COMPL_IN_BITMAP (ref->vops, clobbers, 0, vop, bi)
!     {
!       record_vop_access (map, vop, ref->id, stored);
      }
  }
  
+ /* Create a mapping from virtual operands to references that touch them
+    in LOOP.  */
+ 
+ static void
+ create_vop_ref_mapping_loop (struct loop *loop)
+ {
+   bitmap refs = VEC_index (bitmap, memory_accesses.refs_in_loop, loop->num);
+   struct loop *sloop;
+   bitmap_iterator bi;
+   unsigned i;
+   mem_ref_p ref;
+ 
+   EXECUTE_IF_SET_IN_BITMAP (refs, 0, i, bi)
+     {
+       ref = VEC_index (mem_ref_p, memory_accesses.refs_list, i);
+       for (sloop = loop; sloop != current_loops->tree_root; sloop = sloop->outer)
+ 	add_vop_ref_mapping (sloop, ref);
+     }
+ }
+ 
+ /* For each non-clobbered virtual operand and each loop, record the memory
+    references in this loop that touch the operand.  */
+ 
+ static void
+ create_vop_ref_mapping (void)
+ {
+   loop_iterator li;
+   struct loop *loop;
+ 
+   FOR_EACH_LOOP (li, loop, 0)
+     {
+       create_vop_ref_mapping_loop (loop);
+     }
+ }
+ 
+ /* Gathers information about memory accesses in the loops.  */
+ 
+ static void
+ analyze_memory_references (void)
+ {
+   unsigned i;
+   bitmap empty;
+   htab_t hempty;
+ 
+   memory_accesses.refs
+ 	  = htab_create (100, memref_hash, memref_eq, memref_free);
+   memory_accesses.refs_list = NULL;
+   memory_accesses.refs_in_loop = VEC_alloc (bitmap, heap,
+ 					    number_of_loops ());
+   memory_accesses.all_refs_in_loop = VEC_alloc (bitmap, heap,
+ 						number_of_loops ());
+   memory_accesses.clobbered_vops = VEC_alloc (bitmap, heap,
+ 					      number_of_loops ());
+   memory_accesses.vop_ref_map = VEC_alloc (htab_t, heap,
+ 					   number_of_loops ());
+ 
+   for (i = 0; i < number_of_loops (); i++)
+     {
+       empty = BITMAP_ALLOC (NULL);
+       VEC_quick_push (bitmap, memory_accesses.refs_in_loop, empty);
+       empty = BITMAP_ALLOC (NULL);
+       VEC_quick_push (bitmap, memory_accesses.all_refs_in_loop, empty);
+       empty = BITMAP_ALLOC (NULL);
+       VEC_quick_push (bitmap, memory_accesses.clobbered_vops, empty);
+       hempty = htab_create (10, vtoe_hash, vtoe_eq, vtoe_free);
+       VEC_quick_push (htab_t, memory_accesses.vop_ref_map, hempty);
+     }
+ 
+   memory_accesses.ttae_cache = NULL;
+ 
+   gather_mem_refs_in_loops ();
+   create_vop_ref_mapping ();
+ }
+ 
+ /* Returns true if a region of size SIZE1 at position 0 and a region of
+    size SIZE2 at position DIFF cannot overlap.  */
+ 
+ static bool
+ cannot_overlap_p (aff_tree *diff, double_int size1, double_int size2)
+ {
+   double_int d, bound;
+ 
+   /* Unless the difference is a constant, we fail.  */
+   if (diff->n != 0)
+     return false;
+ 
+   d = diff->offset;
+   if (double_int_negative_p (d))
+     {
+       /* The second object is before the first one, we succeed if the last
+ 	 element of the second object is before the start of the first one.  */
+       bound = double_int_add (d, double_int_add (size2, double_int_minus_one));
+       return double_int_negative_p (bound);
+     }
+   else
+     {
+       /* We succeed if the second object starts after the first one ends.  */
+       return double_int_scmp (size1, d) <= 0;
+     }
+ }
+ 
+ /* Returns true if MEM1 and MEM2 may alias.  TTAE_CACHE is used as a cache in
+    tree_to_aff_combination_expand.  */
+ 
+ static bool
+ mem_refs_may_alias_p (tree mem1, tree mem2, htab_t *ttae_cache)
+ {
+   /* Perform BASE + OFFSET analysis -- if MEM1 and MEM2 are based on the same
+      object and their offset differ in such a way that the locations cannot
+      overlap, then they cannot alias.  */
+   aff_tree off1, off2;
+   double_int size1, size2;
+   tree base1, base2;
+ 
+   /* If MEM1 and MEM2 are based on different variables, they cannot alias.  */
+   base1 = get_base_address (mem1);
+   base2 = get_base_address (mem2);
+ 
+   if (base1
+       && !INDIRECT_REF_P (base1)
+       && base2
+       && !INDIRECT_REF_P (base2)
+       && !operand_equal_p (base1, base2, 0))
+     return false;
+ 
+   /* With strict aliasing, it is impossible to access a scalar variable through
+      anything but a pointer dereference or through a union (gcc extension).  */
+   if (flag_strict_aliasing)
+     {
+       if (!INDIRECT_REF_P (mem1)
+ 	  && base1
+ 	  && TREE_CODE (TREE_TYPE (base1)) != UNION_TYPE
+ 	  && SSA_VAR_P (mem2)
+ 	  && !AGGREGATE_TYPE_P (TREE_TYPE (mem2)))
+ 	return false;
+       if (!INDIRECT_REF_P (mem2)
+ 	  && base2
+ 	  && TREE_CODE (TREE_TYPE (base2)) != UNION_TYPE
+ 	  && SSA_VAR_P (mem1)
+ 	  && !AGGREGATE_TYPE_P (TREE_TYPE (mem1)))
+ 	return false;
+     }
+ 
+   /* The expansion of addresses may be a bit expensive, thus we only do
+      the check at -O2 and higher optimization levels.  */
+   if (optimize < 2)
+     return true;
+ 
+   get_inner_reference_aff (mem1, &off1, &size1);
+   get_inner_reference_aff (mem2, &off2, &size2);
+   aff_combination_expand (&off1, ttae_cache);
+   aff_combination_expand (&off2, ttae_cache);
+   aff_combination_scale (&off1, double_int_minus_one);
+   aff_combination_add (&off2, &off1);
+ 
+   if (cannot_overlap_p (&off2, size1, size2))
+     return false;
+ 
+   return true;
+ }
+ 
+ /* Rewrites location LOC by TMP_VAR.  */
+ 
+ static void
+ rewrite_mem_ref_loc (mem_ref_loc_p loc, tree tmp_var)
+ {
+   mark_virtual_ops_for_renaming (loc->stmt);
+   *loc->ref = tmp_var;
+   update_stmt (loc->stmt);
+ }
+ 
+ /* Adds all locations of REF in LOOP and its subloops to LOCS.  */
+ 
+ static void
+ get_all_locs_in_loop (struct loop *loop, mem_ref_p ref,
+ 		      VEC (mem_ref_loc_p, heap) **locs)
+ {
+   mem_ref_locs_p accs;
+   unsigned i;
+   mem_ref_loc_p loc;
+   bitmap refs = VEC_index (bitmap, memory_accesses.all_refs_in_loop,
+ 			   loop->num);
+   struct loop *subloop;
+ 
+   if (!bitmap_bit_p (refs, ref->id))
+     return;
+ 
+   if (VEC_length (mem_ref_locs_p, ref->accesses_in_loop)
+       > (unsigned) loop->num)
+     {
+       accs = VEC_index (mem_ref_locs_p, ref->accesses_in_loop, loop->num);
+       if (accs)
+ 	{
+ 	  for (i = 0; VEC_iterate (mem_ref_loc_p, accs->locs, i, loc); i++)
+ 	    VEC_safe_push (mem_ref_loc_p, heap, *locs, loc);
+ 	}
+     }
+ 
+   for (subloop = loop->inner; subloop != NULL; subloop = subloop->next)
+     get_all_locs_in_loop (subloop, ref, locs);
+ }
+ 
+ /* Rewrites all references to REF in LOOP by variable TMP_VAR.  */
+ 
+ static void
+ rewrite_mem_refs (struct loop *loop, mem_ref_p ref, tree tmp_var)
+ {
+   unsigned i;
+   mem_ref_loc_p loc;
+   VEC (mem_ref_loc_p, heap) *locs = NULL;
+ 
+   get_all_locs_in_loop (loop, ref, &locs);
+   for (i = 0; VEC_iterate (mem_ref_loc_p, locs, i, loc); i++)
+     rewrite_mem_ref_loc (loc, tmp_var);
+   VEC_free (mem_ref_loc_p, heap, locs);
+ }
+ 
  /* The name and the length of the currently generated variable
     for lsm.  */
  #define MAX_LSM_NAME_LENGTH 40
*************** get_lsm_tmp_name (tree ref)
*** 1013,1030 ****
    return lsm_tmp_name;
  }
  
! /* Records request for store motion of memory reference REF from LOOP.
!    MEM_REFS is the list of occurrences of the reference REF inside LOOP;
!    these references are rewritten by a new temporary variable.
     Exits from the LOOP are stored in EXITS.  The initialization of the
     temporary variable is put to the preheader of the loop, and assignments
     to the reference from the temporary variable are emitted to exits.  */
  
  static void
! schedule_sm (struct loop *loop, VEC (edge, heap) *exits, tree ref,
! 	     struct mem_ref_loc *mem_refs)
  {
-   struct mem_ref_loc *aref;
    tree tmp_var;
    unsigned i;
    tree load, store;
--- 1709,1722 ----
    return lsm_tmp_name;
  }
  
! /* Executes store motion of memory reference REF from LOOP.
     Exits from the LOOP are stored in EXITS.  The initialization of the
     temporary variable is put to the preheader of the loop, and assignments
     to the reference from the temporary variable are emitted to exits.  */
  
  static void
! execute_sm (struct loop *loop, VEC (edge, heap) *exits, mem_ref_p ref)
  {
    tree tmp_var;
    unsigned i;
    tree load, store;
*************** schedule_sm (struct loop *loop, VEC (edg
*** 1034,1057 ****
    if (dump_file && (dump_flags & TDF_DETAILS))
      {
        fprintf (dump_file, "Executing store motion of ");
!       print_generic_expr (dump_file, ref, 0);
        fprintf (dump_file, " from loop %d\n", loop->num);
      }
  
!   tmp_var = make_rename_temp (TREE_TYPE (ref),
! 			      get_lsm_tmp_name (ref));
  
    fmt_data.loop = loop;
    fmt_data.orig_loop = loop;
!   for_each_index (&ref, force_move_till, &fmt_data);
  
!   rewrite_mem_refs (tmp_var, mem_refs);
!   for (aref = mem_refs; aref; aref = aref->next)
!     if (LIM_DATA (aref->stmt))
!       LIM_DATA (aref->stmt)->sm_done = true;
  
    /* Emit the load & stores.  */
!   load = build2_gimple (GIMPLE_MODIFY_STMT, tmp_var, ref);
    get_stmt_ann (load)->common.aux = xcalloc (1, sizeof (struct lim_aux_data));
    LIM_DATA (load)->max_loop = loop;
    LIM_DATA (load)->tgt_loop = loop;
--- 1726,1746 ----
    if (dump_file && (dump_flags & TDF_DETAILS))
      {
        fprintf (dump_file, "Executing store motion of ");
!       print_generic_expr (dump_file, ref->mem, 0);
        fprintf (dump_file, " from loop %d\n", loop->num);
      }
  
!   tmp_var = make_rename_temp (TREE_TYPE (ref->mem),
! 			      get_lsm_tmp_name (ref->mem));
  
    fmt_data.loop = loop;
    fmt_data.orig_loop = loop;
!   for_each_index (&ref->mem, force_move_till, &fmt_data);
  
!   rewrite_mem_refs (loop, ref, tmp_var);
  
    /* Emit the load & stores.  */
!   load = build2_gimple (GIMPLE_MODIFY_STMT, tmp_var, ref->mem);
    get_stmt_ann (load)->common.aux = xcalloc (1, sizeof (struct lim_aux_data));
    LIM_DATA (load)->max_loop = loop;
    LIM_DATA (load)->tgt_loop = loop;
*************** schedule_sm (struct loop *loop, VEC (edg
*** 1062,1390 ****
  
    for (i = 0; VEC_iterate (edge, exits, i, ex); i++)
      {
!       store = build2_gimple (GIMPLE_MODIFY_STMT, unshare_expr (ref), tmp_var);
        bsi_insert_on_edge (ex, store);
      }
  }
  
! /* Check whether memory reference REF can be hoisted out of the LOOP.  If this
!    is true, prepare the statements that load the value of the memory reference
!    to a temporary variable in the loop preheader, store it back on the loop
!    exits, and replace all the references inside LOOP by this temporary variable.
!    EXITS is the list of exits of LOOP.  CLOBBERED_VOPS is the bitmap of virtual
!    operands that are clobbered by a call or accessed through multiple references
!    in loop.  */
  
  static void
! determine_lsm_ref (struct loop *loop, VEC (edge, heap) *exits,
! 		   bitmap clobbered_vops, struct mem_ref *ref)
  {
!   struct mem_ref_loc *aref;
!   struct loop *must_exec;
! 
!   /* In case the memory is not stored to, there is nothing for SM to do.  */
!   if (!ref->is_stored)
!     return;
! 
!   /* If the reference is aliased with any different ref, or killed by call
!      in function, then fail.  */
!   if (bitmap_intersect_p (ref->vops, clobbered_vops))
!     return;
  
!   if (tree_could_trap_p (ref->mem))
      {
!       /* If the memory access is unsafe (i.e. it might trap), ensure that some
! 	 of the statements in that it occurs is always executed when the loop
! 	 is entered.  This way we know that by moving the load from the
! 	 reference out of the loop we will not cause the error that would not
! 	 occur otherwise.
! 
! 	 TODO -- in fact we would like to check for anticipability of the
! 	 reference, i.e. that on each path from loop entry to loop exit at
! 	 least one of the statements containing the memory reference is
! 	 executed.  */
! 
!       for (aref = ref->locs; aref; aref = aref->next)
! 	{
! 	  if (!LIM_DATA (aref->stmt))
! 	    continue;
! 
! 	  must_exec = LIM_DATA (aref->stmt)->always_executed_in;
! 	  if (!must_exec)
! 	    continue;
! 
! 	  if (must_exec == loop
! 	      || flow_loop_nested_p (must_exec, loop))
! 	    break;
! 	}
! 
!       if (!aref)
! 	return;
      }
- 
-   schedule_sm (loop, exits, ref->mem, ref->locs);
- }
- 
- /* Hoists memory references MEM_REFS out of LOOP.  CLOBBERED_VOPS is the list
-    of vops clobbered by call in loop or accessed by multiple memory references.
-    EXITS is the list of exit edges of the LOOP.  */
- 
- static void
- hoist_memory_references (struct loop *loop, struct mem_ref *mem_refs,
- 			 bitmap clobbered_vops, VEC (edge, heap) *exits)
- {
-   struct mem_ref *ref;
- 
-   for (ref = mem_refs; ref; ref = ref->next)
-     determine_lsm_ref (loop, exits, clobbered_vops, ref);
  }
  
! /* Checks whether LOOP (with exits stored in EXITS array) is suitable
!    for a store motion optimization (i.e. whether we can insert statement
!    on its exits).  */
  
  static bool
! loop_suitable_for_sm (struct loop *loop ATTRIBUTE_UNUSED,
! 		      VEC (edge, heap) *exits)
  {
    unsigned i;
!   edge ex;
! 
!   for (i = 0; VEC_iterate (edge, exits, i, ex); i++)
!     if (ex->flags & EDGE_ABNORMAL)
!       return false;
  
!   return true;
! }
  
! /* A hash function for struct mem_ref object OBJ.  */
  
! static hashval_t
! memref_hash (const void *obj)
! {
!   const struct mem_ref *mem = obj;
  
!   return mem->hash;
  }
  
! /* An equality function for struct mem_ref object OBJ1 with
!    memory reference OBJ2.  */
  
! static int
! memref_eq (const void *obj1, const void *obj2)
  {
!   const struct mem_ref *mem1 = obj1;
  
!   return operand_equal_p (mem1->mem, (tree) obj2, 0);
  }
  
! /* Gathers memory references in statement STMT in LOOP, storing the
!    information about them in MEM_REFS hash table.  Note vops accessed through
!    unrecognized statements in CLOBBERED_VOPS.  The newly created references
!    are also stored to MEM_REF_LIST.  */
  
  static void
! gather_mem_refs_stmt (struct loop *loop, htab_t mem_refs,
! 		      bitmap clobbered_vops, tree stmt,
! 		      struct mem_ref **mem_ref_list)
  {
!   tree *lhs, *rhs, *mem = NULL;
!   hashval_t hash;
!   PTR *slot;
!   struct mem_ref *ref = NULL;
!   ssa_op_iter oi;
!   tree vname;
!   bool is_stored;
  
!   if (ZERO_SSA_OPERANDS (stmt, SSA_OP_ALL_VIRTUALS))
!     return;
  
!   /* Recognize MEM = (SSA_NAME | invariant) and SSA_NAME = MEM patterns.  */
!   if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
!     goto fail;
  
!   lhs = &GIMPLE_STMT_OPERAND (stmt, 0);
!   rhs = &GIMPLE_STMT_OPERAND (stmt, 1);
  
!   if (TREE_CODE (*lhs) == SSA_NAME)
      {
!       if (!is_gimple_addressable (*rhs))
! 	goto fail;
  
!       mem = rhs;
!       is_stored = false;
      }
!   else if (TREE_CODE (*rhs) == SSA_NAME
! 	   || is_gimple_min_invariant (*rhs))
      {
!       mem = lhs;
!       is_stored = true;
      }
!   else
!     goto fail;
  
!   /* If we cannot create an SSA name for the result, give up.  */
!   if (!is_gimple_reg_type (TREE_TYPE (*mem))
!       || TREE_THIS_VOLATILE (*mem))
!     goto fail;
  
!   /* If we cannot move the reference out of the loop, fail.  */
!   if (!for_each_index (mem, may_move_till, loop))
!     goto fail;
  
!   hash = iterative_hash_expr (*mem, 0);
!   slot = htab_find_slot_with_hash (mem_refs, *mem, hash, INSERT);
  
!   if (*slot)
!     ref = *slot;
!   else
!     {
!       ref = XNEW (struct mem_ref);
!       ref->mem = *mem;
!       ref->hash = hash;
!       ref->locs = NULL;
!       ref->is_stored = false;
!       ref->vops = BITMAP_ALLOC (NULL);
!       ref->next = *mem_ref_list;
!       *mem_ref_list = ref;
!       *slot = ref;
!     }
!   ref->is_stored |= is_stored;
  
!   FOR_EACH_SSA_TREE_OPERAND (vname, stmt, oi, SSA_OP_VIRTUAL_USES)
!     bitmap_set_bit (ref->vops, DECL_UID (SSA_NAME_VAR (vname)));
!   record_mem_ref_loc (&ref->locs, stmt, mem);
!   return;
  
! fail:
!   FOR_EACH_SSA_TREE_OPERAND (vname, stmt, oi, SSA_OP_VIRTUAL_USES)
!     bitmap_set_bit (clobbered_vops, DECL_UID (SSA_NAME_VAR (vname)));
  }
  
! /* Gathers memory references in LOOP.  Notes vops accessed through unrecognized
!    statements in CLOBBERED_VOPS.  The list of the references found by
!    the function is returned.  */
  
! static struct mem_ref *
! gather_mem_refs (struct loop *loop, bitmap clobbered_vops)
  {
!   basic_block *body = get_loop_body (loop);
!   block_stmt_iterator bsi;
!   unsigned i;
!   struct mem_ref *mem_ref_list = NULL;
!   htab_t mem_refs = htab_create (100, memref_hash, memref_eq, NULL);
  
!   for (i = 0; i < loop->num_nodes; i++)
!     {
!       for (bsi = bsi_start (body[i]); !bsi_end_p (bsi); bsi_next (&bsi))
! 	gather_mem_refs_stmt (loop, mem_refs, clobbered_vops, bsi_stmt (bsi),
! 			      &mem_ref_list);
!     }
  
!   free (body);
  
!   htab_delete (mem_refs);
!   return mem_ref_list;
  }
  
! /* Finds the vops accessed by more than one of the memory references described
!    in MEM_REFS and marks them in CLOBBERED_VOPS.  */
  
  static void
! find_more_ref_vops (struct mem_ref *mem_refs, bitmap clobbered_vops)
  {
!   bitmap_head tmp, all_vops;
!   struct mem_ref *ref;
! 
!   bitmap_initialize (&tmp, &bitmap_default_obstack);
!   bitmap_initialize (&all_vops, &bitmap_default_obstack);
  
!   for (ref = mem_refs; ref; ref = ref->next)
      {
!       /* The vops that are already in all_vops are accessed by more than
! 	 one memory reference.  */
!       bitmap_and (&tmp, &all_vops, ref->vops);
!       bitmap_ior_into (clobbered_vops, &tmp);
!       bitmap_clear (&tmp);
! 
!       bitmap_ior_into (&all_vops, ref->vops);
      }
- 
-   bitmap_clear (&all_vops);
  }
  
! /* Releases the memory occupied by REF.  */
  
! static void
! free_mem_ref (struct mem_ref *ref)
  {
!   free_mem_ref_locs (ref->locs);
!   BITMAP_FREE (ref->vops);
!   free (ref);
! }
! 
! /* Releases the memory occupied by REFS.  */
  
! static void
! free_mem_refs (struct mem_ref *refs)
! {
!   struct mem_ref *ref, *next;
  
!   for (ref = refs; ref; ref = next)
!     {
!       next = ref->next;
!       free_mem_ref (ref);
!     }
  }
  
  /* Try to perform store motion for all memory references modified inside
!    LOOP.  */
  
  static void
! determine_lsm_loop (struct loop *loop)
  {
    VEC (edge, heap) *exits = get_loop_exit_edges (loop);
!   bitmap clobbered_vops;
!   struct mem_ref *mem_refs;
  
!   if (!loop_suitable_for_sm (loop, exits))
      {
!       VEC_free (edge, heap, exits);
!       return;
      }
- 
-   /* Find the memory references in LOOP.  */
-   clobbered_vops = BITMAP_ALLOC (NULL);
-   mem_refs = gather_mem_refs (loop, clobbered_vops);
- 
-   /* Find the vops that are used for more than one reference.  */
-   find_more_ref_vops (mem_refs, clobbered_vops);
- 
-   /* Hoist all suitable memory references.  */
-   hoist_memory_references (loop, mem_refs, clobbered_vops, exits);
- 
-   free_mem_refs (mem_refs);
    VEC_free (edge, heap, exits);
!   BITMAP_FREE (clobbered_vops);
  }
  
  /* Try to perform store motion for all memory references modified inside
     loops.  */
  
  static void
! determine_lsm (void)
  {
    struct loop *loop;
!   loop_iterator li;
  
!   /* Pass the loops from the outermost and perform the store motion as
!      suitable.  */
! 
!   FOR_EACH_LOOP (li, loop, 0)
!     {
!       determine_lsm_loop (loop);
!     }
  
    bsi_commit_edge_inserts ();
  }
  
--- 1751,2034 ----
  
    for (i = 0; VEC_iterate (edge, exits, i, ex); i++)
      {
!       store = build2_gimple (GIMPLE_MODIFY_STMT,
! 			     unshare_expr (ref->mem), tmp_var);
        bsi_insert_on_edge (ex, store);
      }
  }
  
! /* Hoists memory references MEM_REFS out of LOOP.  EXITS is the list of exit
!    edges of the LOOP.  */
  
  static void
! hoist_memory_references (struct loop *loop, bitmap mem_refs,
! 			 VEC (edge, heap) *exits)
  {
!   mem_ref_p ref;
!   unsigned  i;
!   bitmap_iterator bi;
  
!   EXECUTE_IF_SET_IN_BITMAP (mem_refs, 0, i, bi)
      {
!       ref = VEC_index (mem_ref_p, memory_accesses.refs_list, i);
!       execute_sm (loop, exits, ref);
      }
  }
  
! /* Returns true if REF is always accessed in LOOP.  */
  
  static bool
! ref_always_accessed_p (struct loop *loop, mem_ref_p ref)
  {
+   VEC (mem_ref_loc_p, heap) *locs = NULL;
    unsigned i;
!   mem_ref_loc_p loc;
!   bool ret = false;
!   struct loop *must_exec;
  
!   get_all_locs_in_loop (loop, ref, &locs);
!   for (i = 0; VEC_iterate (mem_ref_loc_p, locs, i, loc); i++)
!     {
!       if (!LIM_DATA (loc->stmt))
! 	continue;
  
!       must_exec = LIM_DATA (loc->stmt)->always_executed_in;
!       if (!must_exec)
! 	continue;
  
!       if (must_exec == loop
! 	  || flow_loop_nested_p (must_exec, loop))
! 	{
! 	  ret = true;
! 	  break;
! 	}
!     }
!   VEC_free (mem_ref_loc_p, heap, locs);
  
!   return ret;
  }
  
! /* Returns true if REF1 and REF2 are independent.  */
  
! static bool
! refs_independent_p (mem_ref_p ref1, mem_ref_p ref2)
  {
!   if (ref1 == ref2
!       || bitmap_bit_p (ref1->indep_ref, ref2->id))
!     return true;
!   if (bitmap_bit_p (ref1->dep_ref, ref2->id))
!     return false;
  
!   if (dump_file && (dump_flags & TDF_DETAILS))
!     fprintf (dump_file, "Querying dependency of refs %u and %u: ",
! 	     ref1->id, ref2->id);
! 
!   if (mem_refs_may_alias_p (ref1->mem, ref2->mem,
! 			    &memory_accesses.ttae_cache))
!     {
!       bitmap_set_bit (ref1->dep_ref, ref2->id);
!       bitmap_set_bit (ref2->dep_ref, ref1->id);
!       if (dump_file && (dump_flags & TDF_DETAILS))
! 	fprintf (dump_file, "dependent.\n");
!       return false;
!     }
!   else
!     {
!       bitmap_set_bit (ref1->indep_ref, ref2->id);
!       bitmap_set_bit (ref2->indep_ref, ref1->id);
!       if (dump_file && (dump_flags & TDF_DETAILS))
! 	fprintf (dump_file, "independent.\n");
!       return true;
!     }
  }
  
! /* Records the information whether REF is independent in LOOP (according
!    to INDEP).  */
  
  static void
! record_indep_loop (struct loop *loop, mem_ref_p ref, bool indep)
  {
!   if (indep)
!     bitmap_set_bit (ref->indep_loop, loop->num);
!   else
!     bitmap_set_bit (ref->dep_loop, loop->num);
! }
  
! /* Returns true if REF is independent on all other memory references in
!    LOOP.  */
  
! static bool
! ref_indep_loop_p_1 (struct loop *loop, mem_ref_p ref)
! {
!   bitmap clobbers, refs_to_check, refs;
!   unsigned i;
!   bitmap_iterator bi;
!   bool ret = true, stored = bitmap_bit_p (ref->stored, loop->num);
!   htab_t map;
!   mem_ref_p aref;
! 
!   /* If the reference is clobbered, it is not independent.  */
!   clobbers = VEC_index (bitmap, memory_accesses.clobbered_vops, loop->num);
!   if (bitmap_intersect_p (ref->vops, clobbers))
!     return false;
  
!   refs_to_check = BITMAP_ALLOC (NULL);
  
!   map = VEC_index (htab_t, memory_accesses.vop_ref_map, loop->num);
!   EXECUTE_IF_AND_COMPL_IN_BITMAP (ref->vops, clobbers, 0, i, bi)
      {
!       if (stored)
! 	refs = get_vop_accesses (map, i);
!       else
! 	refs = get_vop_stores (map, i);
  
!       bitmap_ior_into (refs_to_check, refs);
      }
! 
!   EXECUTE_IF_SET_IN_BITMAP (refs_to_check, 0, i, bi)
      {
!       aref = VEC_index (mem_ref_p, memory_accesses.refs_list, i);
!       if (!refs_independent_p (ref, aref))
! 	{
! 	  ret = false;
! 	  record_indep_loop (loop, aref, false);
! 	  break;
! 	}
      }
!   
!   BITMAP_FREE (refs_to_check);
!   return ret;
! }
  
! /* Returns true if REF is independent on all other memory references in
!    LOOP.  Wrapper over ref_indep_loop_p_1, caching its results.  */
  
! static bool
! ref_indep_loop_p (struct loop *loop, mem_ref_p ref)
! {
!   bool ret;
  
!   if (bitmap_bit_p (ref->indep_loop, loop->num))
!     return true;
!   if (bitmap_bit_p (ref->dep_loop, loop->num))
!     return false;
  
!   ret = ref_indep_loop_p_1 (loop, ref);
  
!   if (dump_file && (dump_flags & TDF_DETAILS))
!     fprintf (dump_file, "Querying dependencies of ref %u in loop %d: %s\n",
! 	     ref->id, loop->num, ret ? "independent" : "dependent");
  
!   record_indep_loop (loop, ref, ret);
! 
!   return ret;
  }
  
! /* Returns true if we can perform store motion of REF from LOOP.  */
  
! static bool
! can_sm_ref_p (struct loop *loop, mem_ref_p ref)
  {
!   /* Unless the reference is stored in the loop, there is nothing to do.  */
!   if (!bitmap_bit_p (ref->stored, loop->num))
!     return false;
  
!   /* It should be movable.  */
!   if (!is_gimple_reg_type (TREE_TYPE (ref->mem))
!       || TREE_THIS_VOLATILE (ref->mem)
!       || !for_each_index (&ref->mem, may_move_till, loop))
!     return false;
! 
!   /* If it can trap, it must be always executed in LOOP.  */
!   if (tree_could_trap_p (ref->mem)
!       && !ref_always_accessed_p (loop, ref))
!     return false;
  
!   /* And it must be independent on all other memory references
!      in LOOP.  */
!   if (!ref_indep_loop_p (loop, ref))
!     return false;
  
!   return true;
  }
  
! /* Marks the references in LOOP for that store motion should be performed
!    in REFS_TO_SM.  SM_EXECUTED is the set of references for that store
!    motion was performed in one of the outer loops.  */
  
  static void
! find_refs_for_sm (struct loop *loop, bitmap sm_executed, bitmap refs_to_sm)
  {
!   bitmap refs = VEC_index (bitmap, memory_accesses.all_refs_in_loop,
! 			   loop->num);
!   unsigned i;
!   bitmap_iterator bi;
!   mem_ref_p ref;
  
!   EXECUTE_IF_AND_COMPL_IN_BITMAP (refs, sm_executed, 0, i, bi)
      {
!       ref = VEC_index (mem_ref_p, memory_accesses.refs_list, i);
!       if (can_sm_ref_p (loop, ref))
! 	bitmap_set_bit (refs_to_sm, i);
      }
  }
  
! /* Checks whether LOOP (with exits stored in EXITS array) is suitable
!    for a store motion optimization (i.e. whether we can insert statement
!    on its exits).  */
  
! static bool
! loop_suitable_for_sm (struct loop *loop ATTRIBUTE_UNUSED,
! 		      VEC (edge, heap) *exits)
  {
!   unsigned i;
!   edge ex;
  
!   for (i = 0; VEC_iterate (edge, exits, i, ex); i++)
!     if (ex->flags & EDGE_ABNORMAL)
!       return false;
  
!   return true;
  }
  
  /* Try to perform store motion for all memory references modified inside
!    LOOP.  SM_EXECUTED is the bitmap of the memory references for that
!    store motion was executed in one of the outer loops.  */
  
  static void
! store_motion_loop (struct loop *loop, bitmap sm_executed)
  {
    VEC (edge, heap) *exits = get_loop_exit_edges (loop);
!   struct loop *subloop;
!   bitmap sm_in_loop = BITMAP_ALLOC (NULL);
  
!   if (loop_suitable_for_sm (loop, exits))
      {
!       find_refs_for_sm (loop, sm_executed, sm_in_loop);
!       hoist_memory_references (loop, sm_in_loop, exits);
      }
    VEC_free (edge, heap, exits);
! 
!   bitmap_ior_into (sm_executed, sm_in_loop);
!   for (subloop = loop->inner; subloop != NULL; subloop = subloop->next)
!     store_motion_loop (subloop, sm_executed);
!   bitmap_and_compl_into (sm_executed, sm_in_loop);
!   BITMAP_FREE (sm_in_loop);
  }
  
  /* Try to perform store motion for all memory references modified inside
     loops.  */
  
  static void
! store_motion (void)
  {
    struct loop *loop;
!   bitmap sm_executed = BITMAP_ALLOC (NULL);
  
!   for (loop = current_loops->tree_root->inner; loop != NULL; loop = loop->next)
!     store_motion_loop (loop, sm_executed);
  
+   BITMAP_FREE (sm_executed);
    bsi_commit_edge_inserts ();
  }
  
*************** static void
*** 1491,1501 ****
--- 2135,2170 ----
  tree_ssa_lim_finalize (void)
  {
    basic_block bb;
+   unsigned i;
+   bitmap b;
+   htab_t h;
  
    FOR_EACH_BB (bb)
      {
        bb->aux = NULL;
      }
+ 
+   VEC_free (mem_ref_p, heap, memory_accesses.refs_list);
+   htab_delete (memory_accesses.refs);
+ 
+   for (i = 0; VEC_iterate (bitmap, memory_accesses.refs_in_loop, i, b); i++)
+     BITMAP_FREE (b);
+   VEC_free (bitmap, heap, memory_accesses.refs_in_loop);
+ 
+   for (i = 0; VEC_iterate (bitmap, memory_accesses.all_refs_in_loop, i, b); i++)
+     BITMAP_FREE (b);
+   VEC_free (bitmap, heap, memory_accesses.all_refs_in_loop);
+ 
+   for (i = 0; VEC_iterate (bitmap, memory_accesses.clobbered_vops, i, b); i++)
+     BITMAP_FREE (b);
+   VEC_free (bitmap, heap, memory_accesses.clobbered_vops);
+ 
+   for (i = 0; VEC_iterate (htab_t, memory_accesses.vop_ref_map, i, h); i++)
+     htab_delete (h);
+   VEC_free (htab_t, heap, memory_accesses.vop_ref_map);
+ 
+   if (memory_accesses.ttae_cache)
+     htab_delete (memory_accesses.ttae_cache);
  }
  
  /* Moves invariants from loops.  Only "expensive" invariants are moved out --
*************** tree_ssa_lim (void)
*** 1506,1519 ****
  {
    tree_ssa_lim_initialize ();
  
    /* For each statement determine the outermost loop in that it is
       invariant and cost for computing the invariant.  */
    determine_invariantness ();
  
!   /* For each memory reference determine whether it is possible to hoist it
!      out of the loop.  Force the necessary invariants to be moved out of the
!      loops as well.  */
!   determine_lsm ();
  
    /* Move the expressions that are expensive enough.  */
    move_computations ();
--- 2175,2190 ----
  {
    tree_ssa_lim_initialize ();
  
+   /* Gathers information about memory accesses in the loops.  */
+   analyze_memory_references ();
+ 
    /* For each statement determine the outermost loop in that it is
       invariant and cost for computing the invariant.  */
    determine_invariantness ();
  
!   /* Execute store motion.  Force the necessary invariants to be moved
!      out of the loops as well.  */
!   store_motion ();
  
    /* Move the expressions that are expensive enough.  */
    move_computations ();
Index: testsuite/gcc.dg/tree-ssa/loop-24.c
===================================================================
*** testsuite/gcc.dg/tree-ssa/loop-24.c	(revision 0)
--- testsuite/gcc.dg/tree-ssa/loop-24.c	(revision 0)
***************
*** 0 ****
--- 1,53 ----
+ /* { dg-do compile } */
+ /* { dg-options "-O2 -fdump-tree-lim-details" } */
+ 
+ int x;
+ int a[100];
+ 
+ struct a
+ {
+   int X;
+   int Y;
+ };
+ 
+ struct a arr[100];
+ 
+ void bla(void);
+ 
+ void foo (struct a *A, unsigned b)
+ {
+   unsigned i;
+ 
+   /* We should perform store motion here.  */
+   for (x = 0; x < 100; x++)
+     a[x] = x;
+ 
+   /* But not here.  */
+   for (x = 0; x < 100; x++)
+     bla ();
+ 
+   /* But we should here (using base + offset analysis).  */
+   for (i = 0; i < 100; i++)
+     {
+       A[5].X += i;
+       A[5].Y += i;
+     }
+ 
+   /* And here.  */
+   for (i = 0; i < 100; i++)
+     {
+       arr[b+8].X += i;
+       arr[b+9].X += i;
+     }
+ 
+   /* And here as well.  */
+   for (i = 0; i < 100; i++)
+     {
+       A[b].X += i;
+       A[b+1].Y += i;
+     }
+ }
+ 
+ /* { dg-final { scan-tree-dump-times "Executing store motion of" 7 "lim" } } */
+ 
+ /* { dg-final { cleanup-tree-dump "lim" } } */
Index: tree-affine.c
===================================================================
*** tree-affine.c	(revision 121421)
--- tree-affine.c	(working copy)
*************** Software Foundation, 51 Franklin Street,
*** 30,35 ****
--- 30,37 ----
  #include "diagnostic.h"
  #include "tree-dump.h"
  #include "tree-affine.h"
+ #include "tree-gimple.h"
+ #include "hashtab.h"
  
  /* Extends CST as appropriate for the affine combinations COMB.  */
  
*************** aff_combination_mult (aff_tree *c1, aff_
*** 493,495 ****
--- 495,653 ----
      aff_combination_add_product (c1, double_int_one, c2->rest, r);
    aff_combination_add_product (c1, c2->offset, NULL, r);
  }
+ 
+ /* Element of the cache that maps ssa name NAME to its expanded form
+    as an affine expression EXPANSION.  */
+ 
+ struct name_expansion
+ {
+   aff_tree expansion;
+   tree name;
+ 
+   /* True if the expansion for the name is just being generated.  */
+   unsigned in_progress : 1;
+ };
+ 
+ /* Hash function for struct name_expansion.  */
+ 
+ static hashval_t
+ name_expansion_hash (const void *e)
+ {
+   return SSA_NAME_VERSION (((struct name_expansion *) e)->name);
+ }
+ 
+ /* Equality function for struct name_expansion.  The second argument is an
+    SSA name.  */
+ 
+ static int
+ name_expansion_eq (const void *e, const void *n)
+ {
+   return ((struct name_expansion *) e)->name == n;
+ }
+ 
+ /* Expands SSA names in COMB recursively.  CACHE is used to cache the
+    results.  */
+ 
+ void
+ aff_combination_expand (aff_tree *comb, htab_t *cache)
+ {
+   unsigned i;
+   aff_tree to_add, current, curre;
+   tree e, def, rhs;
+   double_int scale;
+   void **slot;
+   struct name_expansion *exp;
+ 
+   aff_combination_zero (&to_add, comb->type);
+   for (i = 0; i < comb->n; i++)
+     {
+       e = comb->elts[i].val;
+       if (TREE_CODE (e) != SSA_NAME)
+ 	continue;
+       def = SSA_NAME_DEF_STMT (e);
+       if (TREE_CODE (def) != GIMPLE_MODIFY_STMT
+ 	  || GIMPLE_STMT_OPERAND (def, 0) != e)
+ 	continue;
+ 
+       rhs = GIMPLE_STMT_OPERAND (def, 1);
+       if (TREE_CODE (rhs) != SSA_NAME
+ 	  && !EXPR_P (rhs)
+ 	  && !is_gimple_min_invariant (rhs))
+ 	continue;
+ 
+       /* We do not know whether the reference retains its value at the
+ 	 place where the expansion is used.  */
+       if (REFERENCE_CLASS_P (rhs))
+ 	continue;
+ 
+       /* Also, we do not want to return call_exprs.  */
+       if (get_call_expr_in (def) != NULL)
+ 	continue;
+ 
+       if (!*cache)
+ 	*cache = htab_create (10, name_expansion_hash, name_expansion_eq,
+ 			      free);
+       slot = htab_find_slot_with_hash (*cache, e, SSA_NAME_VERSION (e), INSERT);
+       exp = *slot;
+ 
+       if (exp)
+ 	{
+ 	  /* Since we follow the definitions in the SSA form, we should not
+ 	     enter a cycle unless we pass through a phi node.  */
+ 	  gcc_assert (!exp->in_progress);
+ 	  current = exp->expansion;
+ 	}
+       else
+ 	{
+ 	  exp = XNEW (struct name_expansion);
+ 	  exp->name = e;
+ 	  exp->in_progress = 1;
+ 	  *slot = exp;
+ 	  tree_to_aff_combination_expand (rhs, comb->type, &current, cache);
+ 	  exp->expansion = current;
+ 	  exp->in_progress = 0;
+ 	}
+ 
+       /* Accumulate the new terms to TO_ADD, so that we do not modify
+ 	 COMB while traversing it; include the term -coef * E, to remove
+          it from COMB.  */
+       scale = comb->elts[i].coef;
+       aff_combination_zero (&curre, comb->type);
+       aff_combination_add_elt (&curre, e, double_int_neg (scale));
+       aff_combination_scale (&current, scale);
+       aff_combination_add (&to_add, &current);
+       aff_combination_add (&to_add, &curre);
+     }
+   aff_combination_add (comb, &to_add);
+ }
+ 
+ /* Similar to tree_to_aff_combination, but follows SSA name definitions
+    and expands them recursively.  CACHE is used to cache the expansions
+    of the ssa names, to avoid exponential time complexity for cases
+    like
+  
+    a1 = a0 + a0;
+    a2 = a1 + a1;
+    a3 = a2 + a2;
+    ...  */
+ 
+ void
+ tree_to_aff_combination_expand (tree expr, tree type, aff_tree *comb,
+ 				htab_t *cache)
+ {
+   tree_to_aff_combination (expr, type, comb);
+   aff_combination_expand (comb, cache);
+ }
+ 
+ /* Returns address of the reference REF in ADDR.  The size of the accessed
+    location is stored to SIZE.  */
+ 
+ void
+ get_inner_reference_aff (tree ref, aff_tree *addr, double_int *size)
+ {
+   HOST_WIDE_INT bitsize, bitpos;
+   tree toff;
+   enum machine_mode mode;
+   int uns, vol;
+   aff_tree tmp;
+   tree base = get_inner_reference (ref, &bitsize, &bitpos, &toff, &mode,
+ 				   &uns, &vol, false);
+   tree base_addr = build_fold_addr_expr (base);
+ 
+   /* ADDR = &BASE + TOFF + BITPOS / BITS_PER_UNIT.  */
+ 
+   tree_to_aff_combination (base_addr, sizetype, addr);
+ 
+   if (toff)
+     {
+       tree_to_aff_combination (toff, sizetype, &tmp);
+       aff_combination_add (addr, &tmp);
+     }
+ 
+   aff_combination_const (&tmp, sizetype,
+ 			 shwi_to_double_int (bitpos / BITS_PER_UNIT));
+   aff_combination_add (addr, &tmp);
+ 
+   *size = shwi_to_double_int ((bitsize + BITS_PER_UNIT - 1) / BITS_PER_UNIT);
+ }
+ 
Index: tree-affine.h
===================================================================
*** tree-affine.h	(revision 121421)
--- tree-affine.h	(working copy)
*************** void aff_combination_convert (aff_tree *
*** 70,72 ****
--- 70,75 ----
  void tree_to_aff_combination (tree, tree, aff_tree *);
  tree aff_combination_to_tree (aff_tree *);
  void unshare_aff_combination (aff_tree *);
+ void aff_combination_expand (aff_tree *, htab_t *);
+ void tree_to_aff_combination_expand (tree, tree, aff_tree *, htab_t *);
+ void get_inner_reference_aff (tree, aff_tree *, double_int *);
Index: Makefile.in
===================================================================
*** Makefile.in	(revision 121422)
--- Makefile.in	(working copy)
*************** tree-ssa-loop-ivopts.o : tree-ssa-loop-i
*** 2141,2147 ****
     $(CFGLOOP_H) $(PARAMS_H) langhooks.h $(BASIC_BLOCK_H) hard-reg-set.h \
     tree-chrec.h $(VARRAY_H) tree-affine.h
  tree-affine.o : tree-affine.c tree-affine.h $(CONFIG_H) \
!    $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) \
     output.h $(DIAGNOSTIC_H) $(TM_H) coretypes.h $(TREE_DUMP_H)
  tree-ssa-loop-manip.o : tree-ssa-loop-manip.c $(TREE_FLOW_H) $(CONFIG_H) \
     $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(CFGLOOP_H) \
--- 2141,2147 ----
     $(CFGLOOP_H) $(PARAMS_H) langhooks.h $(BASIC_BLOCK_H) hard-reg-set.h \
     tree-chrec.h $(VARRAY_H) tree-affine.h
  tree-affine.o : tree-affine.c tree-affine.h $(CONFIG_H) \
!    $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(TREE_GIMPLE_H) \
     output.h $(DIAGNOSTIC_H) $(TM_H) coretypes.h $(TREE_DUMP_H)
  tree-ssa-loop-manip.o : tree-ssa-loop-manip.c $(TREE_FLOW_H) $(CONFIG_H) \
     $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(CFGLOOP_H) \
*************** tree-ssa-loop-im.o : tree-ssa-loop-im.c 
*** 2152,2158 ****
     $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(CFGLOOP_H) domwalk.h \
     $(PARAMS_H) output.h $(DIAGNOSTIC_H) $(TIMEVAR_H) $(TM_H) coretypes.h \
     $(TREE_DUMP_H) tree-pass.h $(FLAGS_H) $(REAL_H) $(BASIC_BLOCK_H) \
!    hard-reg-set.h
  tree-ssa-math-opts.o : tree-ssa-math-opts.c $(TREE_FLOW_H) $(CONFIG_H) \
     $(SYSTEM_H) $(TREE_H) $(TIMEVAR_H) tree-pass.h $(TM_H) $(FLAGS_H) \
     alloc-pool.h $(BASIC_BLOCK_H) $(TARGET_H)
--- 2152,2158 ----
     $(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(CFGLOOP_H) domwalk.h \
     $(PARAMS_H) output.h $(DIAGNOSTIC_H) $(TIMEVAR_H) $(TM_H) coretypes.h \
     $(TREE_DUMP_H) tree-pass.h $(FLAGS_H) $(REAL_H) $(BASIC_BLOCK_H) \
!    hard-reg-set.h tree-affine.h
  tree-ssa-math-opts.o : tree-ssa-math-opts.c $(TREE_FLOW_H) $(CONFIG_H) \
     $(SYSTEM_H) $(TREE_H) $(TIMEVAR_H) tree-pass.h $(TM_H) $(FLAGS_H) \
     alloc-pool.h $(BASIC_BLOCK_H) $(TARGET_H)


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]