Index: tree-pretty-print.c
===================================================================
*** tree-pretty-print.c	(revision 128603)
--- tree-pretty-print.c	(working copy)
*************** dump_generic_node (pretty_printer *buffe
*** 1941,1946 ****
--- 1941,1963 ----
        dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
        break;
  
+     case OMP_ATOMIC_LOAD:
+       pp_string (buffer, "#pragma omp atomic_load");
+       newline_and_indent (buffer, spc + 2);
+       dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+       pp_space (buffer);
+       pp_character (buffer, '=');
+       pp_space (buffer);
+       pp_character (buffer, '*');
+       dump_generic_node (buffer, TREE_OPERAND (node, 1), spc, flags, false);
+       break;
+ 
+     case OMP_ATOMIC_STORE:
+       pp_string (buffer, "#pragma omp atomic_store (");
+       dump_generic_node (buffer, TREE_OPERAND (node, 0), spc, flags, false);
+       pp_character (buffer, ')');
+       break;
+ 
      case OMP_SINGLE:
        pp_string (buffer, "#pragma omp single");
        dump_omp_clauses (buffer, OMP_SINGLE_CLAUSES (node), spc, flags);
Index: tree.h
===================================================================
*** tree.h	(revision 128603)
--- tree.h	(working copy)
*************** extern const enum tree_code_class tree_c
*** 195,200 ****
--- 195,202 ----
       || TREE_CODE (NODE) == OMP_ORDERED			\
       || TREE_CODE (NODE) == OMP_CRITICAL		\
       || TREE_CODE (NODE) == OMP_RETURN			\
+      || TREE_CODE (NODE) == OMP_ATOMIC_LOAD                            \
+      || TREE_CODE (NODE) == OMP_ATOMIC_STORE                           \
       || TREE_CODE (NODE) == OMP_CONTINUE)
  
  /* Number of argument-words in each kind of tree-node.  */
Index: omp-low.c
===================================================================
*** omp-low.c	(revision 128603)
--- omp-low.c	(working copy)
*************** along with GCC; see the file COPYING3.  
*** 41,47 ****
  #include "ggc.h"
  #include "except.h"
  #include "splay-tree.h"
! 
  
  /* Lowering of OpenMP parallel and workshare constructs proceeds in two 
     phases.  The first phase scans the function looking for OMP statements
--- 41,48 ----
  #include "ggc.h"
  #include "except.h"
  #include "splay-tree.h"
! #include "optabs.h"
! #include "cfgloop.h"
  
  /* Lowering of OpenMP parallel and workshare constructs proceeds in two 
     phases.  The first phase scans the function looking for OMP statements
*************** expand_omp_parallel (struct omp_region *
*** 2597,2603 ****
        rebuild_cgraph_edges ();
        pop_cfun ();
      }
! 
    /* Emit a library call to launch the children threads.  */
    expand_parallel_call (region, new_bb, entry_stmt, ws_args);
    update_ssa (TODO_update_ssa_only_virtuals);
--- 2598,2604 ----
        rebuild_cgraph_edges ();
        pop_cfun ();
      }
!   
    /* Emit a library call to launch the children threads.  */
    expand_parallel_call (region, new_bb, entry_stmt, ws_args);
    update_ssa (TODO_update_ssa_only_virtuals);
*************** expand_omp_synch (struct omp_region *reg
*** 3540,3545 ****
--- 3541,3869 ----
      }
  }
  
+ /* A subroutine of expand_omp_atomic.  Attempt to implement the atomic
+    operation as a __sync_fetch_and_op builtin.  INDEX is log2 of the
+    size of the data type, and thus usable to find the index of the builtin
+    decl.  Returns false if the expression is not of the proper form.  */
+ 
+ static bool
+ expand_omp_atomic_fetch_op (basic_block load_bb,
+ 		 		 		     tree addr, tree loaded_val,
+ 		 		 		     tree stored_val, int index)
+ {
+   enum built_in_function base;
+   tree decl, itype, call;
+   enum insn_code *optab;
+   tree rhs;
+   basic_block store_bb = single_succ (load_bb);
+   block_stmt_iterator bsi;
+   tree stmt;
+ 
+   /* There should be just one statement computing the stored value.  */
+   bsi = bsi_after_labels (store_bb);
+   stmt = bsi_stmt (bsi);
+   if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+     return false;
+   bsi_next (&bsi);
+   if (TREE_CODE (bsi_stmt (bsi)) != OMP_ATOMIC_STORE)
+     return false;
+ 
+   if (!operand_equal_p (GIMPLE_STMT_OPERAND (stmt, 0),
+ 		 		 		 stored_val, 0))
+     return false;
+ 
+   rhs = GIMPLE_STMT_OPERAND (stmt, 1);
+  
+   /* Check for one of the supported fetch-op operations.  */
+   switch (TREE_CODE (rhs))
+     {
+     case PLUS_EXPR:
+     case POINTER_PLUS_EXPR:
+       base = BUILT_IN_FETCH_AND_ADD_N;
+       optab = sync_add_optab;
+       break;
+     case MINUS_EXPR:
+       base = BUILT_IN_FETCH_AND_SUB_N;
+       optab = sync_add_optab;
+       break;
+     case BIT_AND_EXPR:
+       base = BUILT_IN_FETCH_AND_AND_N;
+       optab = sync_and_optab;
+       break;
+     case BIT_IOR_EXPR:
+       base = BUILT_IN_FETCH_AND_OR_N;
+       optab = sync_ior_optab;
+       break;
+     case BIT_XOR_EXPR:
+       base = BUILT_IN_FETCH_AND_XOR_N;
+       optab = sync_xor_optab;
+       break;
+     default:
+       return false;
+     }
+   /* Make sure the expression is of the proper form.  */
+   if (operand_equal_p (TREE_OPERAND (rhs, 0), loaded_val, 0))
+     rhs = TREE_OPERAND (rhs, 1);
+   else if (commutative_tree_code (TREE_CODE (rhs))
+ 		    && operand_equal_p (TREE_OPERAND (rhs, 1), loaded_val, 0))
+     rhs = TREE_OPERAND (rhs, 0);
+   else
+     return false;
+ 
+   decl = built_in_decls[base + index + 1];
+   itype = TREE_TYPE (TREE_TYPE (decl));
+ 
+   if (optab[TYPE_MODE (itype)] == CODE_FOR_nothing)
+     return false;
+ 
+   bsi = bsi_last (load_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
+   call = build_call_expr (decl, 2, addr, fold_convert (itype, rhs));
+   force_gimple_operand_bsi (&bsi, call, true, NULL_TREE,
+ 		 		 		     true, BSI_SAME_STMT);
+   bsi_remove (&bsi, true);
+ 
+   bsi = bsi_last (store_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
+   bsi_remove (&bsi, true);
+   bsi = bsi_last (store_bb);
+   bsi_remove (&bsi, true);		 
+ 
+   if (gimple_in_ssa_p (cfun))
+     update_ssa (TODO_update_ssa_no_phi);
+ 
+   return true;
+ }
+ 
+ /* A subroutine of expand_omp_atomic.  Implement the atomic operation as:
+ 
+ 		 		  oldval = *addr;
+       repeat:
+ 		 		  newval = rhs;		 		  // with oldval replacing *addr in rhs
+ 		 		  oldval = __sync_val_compare_and_swap (addr, oldval, newval);
+ 		 		  if (oldval != newval)
+ 		 		    goto repeat;
+ 
+    INDEX is log2 of the size of the data type, and thus usable to find the
+    index of the builtin decl.  */
+ 
+ static bool
+ expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
+ 		 		 		     tree addr, tree loaded_val, tree stored_val,
+ 		 		 		     int index)
+ {
+   tree loadedi, storedi, initial, new_stored, new_storedi, old_vali;
+   tree type, itype, cmpxchg, iaddr;
+   block_stmt_iterator bsi;
+   basic_block loop_header = single_succ (load_bb);
+   tree phi, x;
+   edge e;
+ 
+   cmpxchg = built_in_decls[BUILT_IN_VAL_COMPARE_AND_SWAP_N + index + 1];
+   type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
+   itype = TREE_TYPE (TREE_TYPE (cmpxchg));
+  
+   if (sync_compare_and_swap[TYPE_MODE (itype)] == CODE_FOR_nothing)
+     return false;
+   
+   /* Load the initial value, replacing the OMP_ATOMIC_LOAD.  */
+   bsi = bsi_last (load_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
+   initial = force_gimple_operand_bsi (&bsi, build_fold_indirect_ref (addr),
+ 		 		 		 		       true, NULL_TREE,
+ 		 		 		 		       true, BSI_SAME_STMT);
+   /* Move the value to the LOADED_VAL temporary.  */
+   if (gimple_in_ssa_p (cfun))
+     {
+       gcc_assert (phi_nodes (loop_header) == NULL_TREE);
+       phi = create_phi_node (loaded_val, loop_header);
+       SSA_NAME_DEF_STMT (loaded_val) = phi;
+       SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
+ 		        initial);
+     }
+   else
+     bsi_insert_before (&bsi,
+ 		 		        build_gimple_modify_stmt (loaded_val, initial),
+ 		 		        BSI_SAME_STMT);
+   bsi_remove (&bsi, true);
+ 
+   bsi = bsi_last (store_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
+ 
+   /* For floating-point values, we'll need to view-convert them to integers
+      so that we can perform the atomic compare and swap.  Simplify the 
+      following code by always setting up the "i"ntegral variables.  */
+   if (INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
+     {
+       loadedi = loaded_val;
+       storedi = stored_val;
+       iaddr = addr;
+     }
+   else
+     {
+       loadedi = force_gimple_operand_bsi (&bsi,
+ 		 		 		 build1 (VIEW_CONVERT_EXPR, itype, loaded_val),
+ 		 		 		 true, NULL_TREE, true, BSI_SAME_STMT);
+       storedi = force_gimple_operand_bsi (&bsi,
+ 		 		 		 build1 (VIEW_CONVERT_EXPR, itype, stored_val),
+ 		 		 		 true, NULL_TREE, true, BSI_SAME_STMT);
+       iaddr = fold_convert (build_pointer_type (itype), addr);
+     }
+   
+   /* Build the compare&swap statement.  */
+   new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
+   new_storedi = force_gimple_operand_bsi (&bsi,
+ 		 		 		 		 		   fold_convert (itype, new_storedi),
+ 		 		 		 		 		   true, NULL_TREE,
+ 		 		 		 		 		   true, BSI_SAME_STMT);
+   if (storedi == stored_val)
+     new_stored = new_storedi;
+   else
+     new_stored = force_gimple_operand_bsi (&bsi,
+ 		 		 		 build1 (VIEW_CONVERT_EXPR, type, new_storedi),
+ 		 		 		 true, NULL_TREE, true, BSI_SAME_STMT);
+ 
+   if (gimple_in_ssa_p (cfun))
+     old_vali = loadedi;
+   else
+     {
+       old_vali = create_tmp_var (itype, NULL);
+       x = build_gimple_modify_stmt (old_vali, loadedi);
+       bsi_insert_before (&bsi, x, BSI_SAME_STMT);
+ 
+       x = build_gimple_modify_stmt (loaded_val, new_stored);
+       bsi_insert_before (&bsi, x, BSI_SAME_STMT);
+     }
+ 
+   /* Note that we always perform the comparison as an integer, even for
+      floating point.  This allows the atomic operation to properly 
+      succeed even with NaNs and -0.0.  */
+   x = build3 (COND_EXPR, void_type_node,
+ 		       build2 (NE_EXPR, boolean_type_node,
+ 		 		       new_storedi, old_vali),
+ 		       NULL_TREE, NULL_TREE);
+   bsi_insert_before (&bsi, x, BSI_SAME_STMT);
+ 
+   /* Update cfg.  */
+   e = single_succ_edge (store_bb);
+   e->flags &= ~EDGE_FALLTHRU;
+   e->flags |= EDGE_FALSE_VALUE;
+ 
+   e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
+ 
+   /* Copy the new value to loaded_val (we already did that before the condition
+      if we are not in SSA).  */
+   if (gimple_in_ssa_p (cfun))
+     {
+       phi = phi_nodes (loop_header);
+       SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_stored);
+     }
+ 
+   /* Remove OMP_ATOMIC_STORE.  */
+   bsi_remove (&bsi, true);
+ 
+   if (gimple_in_ssa_p (cfun))
+     update_ssa (TODO_update_ssa_no_phi);
+ 
+   return true;
+ }
+ 
+ /* A subroutine of expand_omp_atomic.  Implement the atomic operation as:
+ 
+ 		 		  GOMP_atomic_start ();
+ 		 		  *addr = rhs;
+ 		 		  GOMP_atomic_end ();
+ 
+    The result is not globally atomic, but works so long as all parallel
+    references are within #pragma omp atomic directives.  According to
+    responses received from omp@openmp.org, appears to be within spec.
+    Which makes sense, since that's how several other compilers handle
+    this situation as well.  */
+ 
+ static bool
+ expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
+ 		 		 		  tree addr, tree loaded_val, tree stored_val)
+ {
+   block_stmt_iterator bsi;
+   tree t;
+ 
+   bsi = bsi_last (load_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
+ 
+   t = built_in_decls[BUILT_IN_GOMP_ATOMIC_START];
+   t = build_function_call_expr (t,0);
+   force_gimple_operand_bsi (&bsi, t, true, NULL_TREE,
+ 		 		 		     true, BSI_SAME_STMT);
+ 
+   t = build_gimple_modify_stmt (loaded_val,
+ 		 		 		 		 build_fold_indirect_ref (addr));
+   if (gimple_in_ssa_p (cfun))
+     SSA_NAME_DEF_STMT (loaded_val) = t;
+   bsi_insert_before (&bsi, t, BSI_SAME_STMT);
+   bsi_remove (&bsi, true);
+ 
+   bsi = bsi_last (store_bb);
+   gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
+ 
+   t = build_gimple_modify_stmt (build_fold_indirect_ref (unshare_expr (addr)),
+ 		 		 		 		 stored_val);
+   bsi_insert_before (&bsi, t, BSI_SAME_STMT);
+ 
+   t = built_in_decls[BUILT_IN_GOMP_ATOMIC_END];
+   t = build_function_call_expr (t,0);
+   force_gimple_operand_bsi (&bsi, t, true, NULL_TREE,
+ 		 		 		     true, BSI_SAME_STMT);
+   bsi_remove (&bsi, true);
+ 
+   if (gimple_in_ssa_p (cfun))
+     update_ssa (TODO_update_ssa_no_phi);
+   return true;
+ }
+ 
+ /* Gimplify an OMP_ATOMIC statement.  */
+ 
+ static void
+ expand_omp_atomic (struct omp_region *region)
+ {
+   basic_block load_bb = region->entry, store_bb = region->exit;
+   tree load = last_stmt (load_bb), store = last_stmt (store_bb);
+   tree loaded_val = TREE_OPERAND (load, 0);
+   tree addr = TREE_OPERAND (load, 1);
+   tree stored_val = TREE_OPERAND (store, 0);
+   tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
+   HOST_WIDE_INT index;
+ 
+   /* Make sure the type is one of the supported sizes.  */
+   index = tree_low_cst (TYPE_SIZE_UNIT (type), 1);
+   index = exact_log2 (index);
+   if (index >= 0 && index <= 4)
+     {
+       unsigned int align = TYPE_ALIGN_UNIT (type);
+ 
+       /* __sync builtins require strict data alignment.  */
+       if (exact_log2 (align) >= index)
+ 		 {
+ 		   /* When possible, use specialized atomic update functions.  */
+ 		   if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
+ 		       && store_bb == single_succ (load_bb))
+ 		     {
+ 		       if (expand_omp_atomic_fetch_op (load_bb, addr,
+ 		 		 		 		 		       loaded_val, stored_val, index))
+ 		 		 return;
+ 		     }
+ 
+ 		   /* If we don't have specialized __sync builtins, try and implement
+ 		      as a compare and swap loop.  */
+ 		   if (expand_omp_atomic_pipeline (load_bb, store_bb, addr, 
+ 		 		 		 		 		   loaded_val, stored_val, index))
+ 		     return;
+ 		 }
+     }
+ 
+   /* The ultimate fallback is wrapping the operation in a mutex.  */
+   expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
+ }
+ 
  
  /* Expand the parallel region tree rooted at REGION.  Expansion
     proceeds in depth-first order.  Innermost regions are expanded
*************** expand_omp (struct omp_region *region)
*** 3584,3589 ****
--- 3908,3918 ----
  	  expand_omp_synch (region);
  	  break;
  
+         case OMP_ATOMIC_LOAD:
+            expand_omp_atomic (region);
+            break; 
+                    
+ 
  	default:
  	  gcc_unreachable ();
  	}
*************** build_omp_regions_1 (basic_block bb, str
*** 3614,3620 ****
  
        stmt = bsi_stmt (si);
        code = TREE_CODE (stmt);
- 
        if (code == OMP_RETURN)
  	{
  	  /* STMT is the return point out of region PARENT.  Mark it
--- 3943,3948 ----
*************** build_omp_regions_1 (basic_block bb, str
*** 3630,3635 ****
--- 3958,3974 ----
  	  if (region->type == OMP_PARALLEL)
  	    determine_parallel_type (region);
  	}
+         else if (code == OMP_ATOMIC_STORE)
+          {
+            /* OMP_ATOMIC_STORE is analoguous to OMP_RETURN, but matches with
+                      OMP_ATOMIC_LOAD.  */
+                   gcc_assert (parent);
+                   gcc_assert (parent->type == OMP_ATOMIC_LOAD);
+                   region = parent;
+                   region->exit = bb;
+                   parent = parent->outer;
+          }
+ 
        else if (code == OMP_CONTINUE)
  	{
  	  gcc_assert (parent);
*************** build_omp_regions_1 (basic_block bb, str
*** 3638,3644 ****
        else if (code == OMP_SECTIONS_SWITCH)
  	{
  	  /* OMP_SECTIONS_SWITCH is part of OMP_SECTIONS, and we do nothing for
! 	     it.  */
  	}
        else
  	{
--- 3977,3983 ----
        else if (code == OMP_SECTIONS_SWITCH)
  	{
  	  /* OMP_SECTIONS_SWITCH is part of OMP_SECTIONS, and we do nothing for
! 	     it.  */;
  	}
        else
  	{
Index: tree-gimple.c
===================================================================
*** tree-gimple.c	(revision 128603)
--- tree-gimple.c	(working copy)
*************** is_gimple_stmt (tree t)
*** 240,245 ****
--- 240,247 ----
      case OMP_CRITICAL:
      case OMP_RETURN:
      case OMP_CONTINUE:
+     case OMP_ATOMIC_LOAD:
+     case OMP_ATOMIC_STORE:
        /* These are always void.  */
        return true;
  
Index: testsuite/gcc.dg/gomp/atomic-3.c
===================================================================
*** testsuite/gcc.dg/gomp/atomic-3.c	(revision 128603)
--- testsuite/gcc.dg/gomp/atomic-3.c	(working copy)
***************
*** 1,5 ****
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-gimple" } */
  
  int *xyzzy;
  
--- 1,5 ----
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-ompexp" } */
  
  int *xyzzy;
  
*************** void f1(void)
*** 9,13 ****
      xyzzy++;
  }
  
! /* { dg-final { scan-tree-dump-times "xyzzy, 4" 1 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "gimple" } } */
--- 9,13 ----
      xyzzy++;
  }
  
! /* { dg-final { scan-tree-dump-times "xyzzy, 4" 1 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "ompexp" } } */
Index: testsuite/gcc.dg/gomp/atomic-9.c
===================================================================
*** testsuite/gcc.dg/gomp/atomic-9.c	(revision 128603)
--- testsuite/gcc.dg/gomp/atomic-9.c	(working copy)
***************
*** 1,5 ****
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-gimple" } */
  
  volatile int *bar(void);
  
--- 1,5 ----
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-ompexp" } */
  
  volatile int *bar(void);
  
*************** void f1(void)
*** 9,13 ****
      *bar() += 1;
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 1 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "gimple" } } */
--- 9,13 ----
      *bar() += 1;
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 1 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "ompexp" } } */
Index: testsuite/gcc.dg/gomp/atomic-10.c
===================================================================
*** testsuite/gcc.dg/gomp/atomic-10.c	(revision 128603)
--- testsuite/gcc.dg/gomp/atomic-10.c	(working copy)
***************
*** 1,6 ****
  /* PR middle-end/28046 */
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-gimple" } */
  
  int a[3], b;
  struct C { int x; int y; } c;
--- 1,6 ----
  /* PR middle-end/28046 */
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-ompexp" } */
  
  int a[3], b;
  struct C { int x; int y; } c;
*************** foo (void)
*** 20,24 ****
    *baz () += bar ();
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 4 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "gimple" } } */
--- 20,24 ----
    *baz () += bar ();
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 4 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "ompexp" } } */
Index: testsuite/g++.dg/gomp/atomic-3.C
===================================================================
*** testsuite/g++.dg/gomp/atomic-3.C	(revision 128603)
--- testsuite/g++.dg/gomp/atomic-3.C	(working copy)
***************
*** 1,5 ****
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-gimple" } */
  
  int *xyzzy;
  
--- 1,5 ----
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-ompexp" } */
  
  int *xyzzy;
  
*************** void f1(void)
*** 9,13 ****
      xyzzy++;
  }
  
! /* { dg-final { scan-tree-dump-times "xyzzy, 4" 1 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "gimple" } } */
--- 9,13 ----
      xyzzy++;
  }
  
! /* { dg-final { scan-tree-dump-times "xyzzy, 4" 1 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "ompexp" } } */
Index: testsuite/g++.dg/gomp/atomic-9.C
===================================================================
*** testsuite/g++.dg/gomp/atomic-9.C	(revision 128603)
--- testsuite/g++.dg/gomp/atomic-9.C	(working copy)
***************
*** 1,5 ****
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-gimple" } */
  
  volatile int *bar(void);
  
--- 1,5 ----
  /* { dg-do compile } */
! /* { dg-options "-fopenmp -fdump-tree-ompexp" } */
  
  volatile int *bar(void);
  
*************** void f1(void)
*** 9,13 ****
      *bar() += 1;
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 1 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "gimple" } } */
--- 9,13 ----
      *bar() += 1;
  }
  
! /* { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 1 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } } */
! /* { dg-final { cleanup-tree-dump "ompexp" } } */
Index: testsuite/g++.dg/gomp/atomic-10.C
===================================================================
*** testsuite/g++.dg/gomp/atomic-10.C	(revision 128603)
--- testsuite/g++.dg/gomp/atomic-10.C	(working copy)
***************
*** 1,6 ****
  // PR middle-end/28046
  // { dg-do compile }
! // { dg-options "-fopenmp -fdump-tree-gimple" }
  
  int a[3], b;
  struct C { int x; int y; } c;
--- 1,6 ----
  // PR middle-end/28046
  // { dg-do compile }
! // { dg-options "-fopenmp -fdump-tree-ompexp" }
  
  int a[3], b;
  struct C { int x; int y; } c;
*************** foo (void)
*** 20,24 ****
    *baz () += bar ();
  }
  
! // { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 4 "gimple" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } }
! // { dg-final { cleanup-tree-dump "gimple" } }
--- 20,24 ----
    *baz () += bar ();
  }
  
! // { dg-final { scan-tree-dump-times "__sync_fetch_and_add" 4 "ompexp" { target i?86-*-* x86_64-*-* ia64-*-* powerpc*-*-* alpha*-*-* } } }
! // { dg-final { cleanup-tree-dump "ompexp" } }
Index: gimple-low.c
===================================================================
*** gimple-low.c	(revision 128603)
--- gimple-low.c	(working copy)
*************** lower_stmt (tree_stmt_iterator *tsi, str
*** 251,256 ****
--- 251,258 ----
      case OMP_ORDERED:
      case OMP_CRITICAL:
      case OMP_RETURN:
+     case OMP_ATOMIC_LOAD:
+     case OMP_ATOMIC_STORE:
      case OMP_CONTINUE:
        break;
  
Index: expr.c
===================================================================
*** expr.c	(revision 128603)
--- expr.c	(working copy)
*************** along with GCC; see the file COPYING3.  
*** 53,58 ****
--- 53,59 ----
  #include "target.h"
  #include "timevar.h"
  #include "df.h"
+ #include "diagnostic.h"
  
  /* Decide whether a function's arguments should be processed
     from first to last or from last to first.
*************** expand_expr_real_1 (tree exp, rtx target
*** 9296,9301 ****
--- 9297,9309 ----
  	goto binop;
        }
  
+     case OMP_ATOMIC_LOAD:
+     case OMP_ATOMIC_STORE:
+       /* OMP expansion is not run when there were errors, so these codes
+ 		  can get here.  */
+       gcc_assert (errorcount != 0);
+       return NULL_RTX;
+ 
      default:
        return lang_hooks.expand_expr (exp, original_target, tmode,
  				     modifier, alt_rtl);
Index: tree-parloops.c
===================================================================
*** tree-parloops.c	(revision 128603)
--- tree-parloops.c	(working copy)
*************** Software Foundation, 51 Franklin Street,
*** 35,40 ****
--- 35,41 ----
  #include "tree-scalar-evolution.h"
  #include "hashtab.h"
  #include "langhooks.h"
+ #include "tree-vectorizer.h"
  
  /* This pass tries to distribute iterations of loops into several threads.
     The implementation is straightforward -- for each loop we test whether its
*************** Software Foundation, 51 Franklin Street,
*** 61,70 ****
--- 62,296 ----
     -- handling of common scalar dependence patterns (accumulation, ...)
     -- handling of non-innermost loops  */
  
+ /*  
+   Reduction handling:
+   currently we use vect_is_simple_reduction() to detect reduction patterns.
+   The code transformation will be introduced by an example.
+   
+   source code:
+ 
+ parloop
+ {
+   int sum=1;
+ 
+   for (i = 0; i < N/1000; i++)
+    {
+     x[i] = i + 3;
+     sum+=x[i];
+    }
+ }
+ 
+ gimple code:
+ 
+ header_bb:
+ 
+   # sum_24 = PHI <sum_14(3), 1(2)>;
+   # i_21 = PHI <i_15(3), 0(2)>;
+ <L0>:;
+   D.2191_10 = i_21 + 3;
+   x[i_21] = D.2191_10;
+   sum_14 = D.2191_10 + sum_24;
+   i_15 = i_21 + 1;
+   if (N_8 > i_15) goto <L0>; else goto <L2>;
+ 
+ exit_bb:
+ 
+   # sum_25 = PHI <sum_14(3)>;
+ <L2>:;
+ 
+ 
+ after reduction transformation (only relevant parts):
+ 
+ parloop
+ {
+ 
+ ....
+ 
+ <L16>:;
+   D.2241_2 = (unsigned int) N_8;
+   D.2242_26 = D.2241_2 - 1;
+   if (D.2242_26 > 399) goto <L26>; else goto <L27>;
+ 
+ #two new variables are created for each reduction: 
+ "reduction" is the variable holding the proper init
+ value related to this reduction operation.
+ "reduction_initial" is the initial value given by the user.
+ It is kept and will be used  after the parallel computing 
+ is done.#
+ 
+ <L26>:;
+   reduction.38_42 = 0;     
+   reduction_initial.39_43 = 1;        
+   x.40_44 = &x;
+   .paral_data_store.47.D.2261 = D.2242_26;
+   .paral_data_store.47.reduction.38 = reduction.38_42;
+   .paral_data_store.47.x.40 = x.40_44;
+   __builtin_GOMP_parallel_start (parloop._loopfn.0, &.paral_data_store.47, 4);
+   parloop._loopfn.0 (&.paral_data_store.47);
+   __builtin_GOMP_parallel_end ();
+ 
+ # collecting the result after the join of the threads is done at
+   create_loads_for_reductions().  
+   a new variable "reduction_final" is created.  It calculates the
+   final value from the initial value and the value computed by 
+   the threads.  #
+   
+   .paral_data_load.48_49 = &.paral_data_store.47;        
+   reduction_final.49_50 = .paral_data_load.48_49->reduction.38;
+   reduction_final.49_51 = reduction_initial.39_43 + reduction_final.49_50;
+   ivtmp.37_36 = D.2242_26;
+   i_37 = (int) ivtmp.37_36;
+   D.2191_38 = i_37 + 3;
+   x[i_37] = D.2191_38;
+   sum_40 = D.2191_38 + reduction_final.49_51;
+   i_41 = i_37 + 1;
+   goto <bb 8> (<L2>);
+ 
+   # sum_25 = PHI <sum_40(4), sum_9(6)>;
+ <L2>:;
+   printf (&"sum is %d\n"[0], sum_25);
+ 
+ ...
+ 
+ }
+ 
+ parloop._loopfn.0 (.paral_data_param)
+ {
+  ...
+ 
+ <L28>:;
+   .paral_data_param_52 = .paral_data_param_75;
+   .paral_data_load.48_48 = (struct .paral_data.46 *) .paral_data_param_52;
+   D.2289_46 = .paral_data_load.48_48->D.2261;
+   reduction.43_45 = .paral_data_load.48_48->reduction.38;
+    x.45_47 = .paral_data_load.48_48->x.40;
+   # SUCC: 23 [100.0%]  (fallthru)
+ 
+   # BLOCK 23
+   # PRED: 21 [100.0%]  (fallthru)
+ <L30>:;
+   D.2292_60 = __builtin_omp_get_num_threads ();
+   D.2293_61 = (unsigned int) D.2292_60;
+   D.2294_62 = __builtin_omp_get_thread_num ();
+   D.2295_63 = (unsigned int) D.2294_62;
+   D.2296_64 = D.2289_46 / D.2293_61;
+   D.2297_65 = D.2293_61 * D.2296_64;
+   D.2298_66 = D.2297_65 != D.2289_46;
+   D.2299_67 = D.2296_64 + D.2298_66;
+   D.2300_68 = D.2299_67 * D.2295_63;
+   D.2301_69 = D.2299_67 + D.2300_68;
+   D.2302_70 = MIN_EXPR <D.2301_69, D.2289_46>;
+   ivtmp.41_54 = D.2300_68;
+   if (D.2300_68 >= D.2302_70) goto <L31>; else goto <L32>;
+   # SUCC: 26 [100.0%]  (false) 24 (true)
+ 
+   # Adding this reduction phi is done at 
+     create_phi_for_local_result() #
+ 
+   # BLOCK 24
+   # PRED: 5 (false) 23 (true)
+   # reduction.38_56 = PHI <sum.42_14(5), 0(23)>;  
+ <L31>:;
+   __builtin_GOMP_barrier ();
+   # SUCC: 25 [100.0%]  (fallthru)
+ 
+ 
+   # Creating the atomic operation is
+     done at create_call_for_reduction_1()  #
+ 
+ # BLOCK 25    
+   # PRED: 24 [100.0%]  (fallthru)
+   D.2306_57 = &.paral_data_load.48_48->reduction.38;
+   D.2307_58 = (unsigned int) reduction.38_56;
+   D.2308_59 = __sync_fetch_and_add_4 (D.2306_57, D.2307_58);
+   # SUCC: 22 [100.0%]  (fallthru)
+ 
+   # BLOCK 22
+   # PRED: 25 [100.0%]  (fallthru)
+ <L29>:;
+   return;
+   # SUCC: EXIT
+ 
+   # BLOCK 26
+   # PRED: 23 [100.0%]  (false)
+ <L32>:;
+   # SUCC: 4 [100.0%]  (fallthru)
+ 
+   # BLOCK 4
+   # PRED: 5 [100.0%]  (true) 26 [100.0%]  (fallthru)
+   # ivtmp.41_31 = PHI <ivtmp.41_30(5), ivtmp.41_54(26)>;
+   # sum.42_32 = PHI <sum.42_14(5), reduction.43_45(26)>;
+ <L0>:;
+   # SUCC: 19 [100.0%]  (fallthru)
+ 
+   # BLOCK 19
+   # PRED: 4 [100.0%]  (fallthru)
+   # sum.42_24 = PHI <sum.42_32(4)>;
+   # ivtmp.41_17 = PHI <ivtmp.41_31(4)>;
+   i.44_21 = (int) ivtmp.41_17;
+   D.2310_10 = i.44_21 + 3;
+   (*x.45_47)[i.44_21] = D.2310_10;
+   sum.42_14 = D.2310_10 + sum.42_24;
+   i.44_15 = i.44_21 + 1;
+   # SUCC: 5 [100.0%]  (fallthru)
+   # BLOCK 5
+   # PRED: 19 [100.0%]  (fallthru)
+ <L17>:;
+   ivtmp.41_30 = ivtmp.41_31 + 1;
+   if (ivtmp.41_30 < D.2302_70) goto <L0>; else goto <L31>;
+   # SUCC: 4 [100.0%]  (true) 24 (false)
+ 
+ }
+   
+ */
+ 
  /* Minimal number of iterations of a loop that should be executed in each
     thread.  */
  #define MIN_PER_THREAD 100
  
+ /* List of reductions in the current loop.  */
+ static htab_t reduction_list = NULL;
+ 
+ /* Element of the hashtable, representing a 
+    reduction in the current loop.  */
+ struct reduction_info
+ {
+   tree reduc_stmt;		/* reduction statement.  */
+   tree reduc_phi;		/* The phi node defining the reduction.  */
+   enum tree_code reduction_code;	/* code for the reduction operation.  */
+   tree keep_res;		/* The PHI_RESULT of this phi is the resulting value 
+ 				   of the reduction variable when existing the loop. */
+   tree initial_value;		/* An ssa name representing a new variable holding
+ 				   the initial value of the reduction var before entering the loop.   */
+   tree field;			/*  the name of the field in the parloop data structure intended for reduction.  */
+   tree reduction_init;		/* An ssa name representing a new variable which will be 
+ 				   assigned the proper reduction initialization value (init).  */
+   tree init;			/* reduction initialization value.  */
+   tree new_phi;			/* (helper field) Newly created phi node whose result 
+ 				   will be passed to the atomic operation.  Represents
+ 				   the local result each thread computed for the reduction
+ 				   operation.  */
+ };
+ 
+ /* Equality and hash functions for hashtab code.  */
+ 
+ static int
+ reduction_info_eq (const void *aa, const void *bb)
+ {
+   const struct reduction_info *a = (const struct reduction_info *) aa;
+   const struct reduction_info *b = (const struct reduction_info *) bb;
+ 
+   return (a->reduc_phi == b->reduc_phi);
+ }
+ 
+ static hashval_t
+ reduction_info_hash (const void *aa)
+ {
+   const struct reduction_info *a = (const struct reduction_info *) aa;
+ 
+   return htab_hash_pointer (a->reduc_phi);
+ }
+ 
  /* Element of hashtable of names to copy.  */
  
  struct name_to_copy_elt
*************** struct name_to_copy_elt
*** 80,87 ****
  static int
  name_to_copy_elt_eq (const void *aa, const void *bb)
  {
!   struct name_to_copy_elt *a = (struct name_to_copy_elt *) aa;
!   struct name_to_copy_elt *b = (struct name_to_copy_elt *) bb;
  
    return a->version == b->version;
  }
--- 306,313 ----
  static int
  name_to_copy_elt_eq (const void *aa, const void *bb)
  {
!   const struct name_to_copy_elt *a = (const struct name_to_copy_elt *) aa;
!   const struct name_to_copy_elt *b = (const struct name_to_copy_elt *) bb;
  
    return a->version == b->version;
  }
*************** name_to_copy_elt_eq (const void *aa, con
*** 89,95 ****
  static hashval_t
  name_to_copy_elt_hash (const void *aa)
  {
!   struct name_to_copy_elt *a = (struct name_to_copy_elt *) aa;
  
    return (hashval_t) a->version;
  }
--- 315,321 ----
  static hashval_t
  name_to_copy_elt_hash (const void *aa)
  {
!   const struct name_to_copy_elt *a = (const struct name_to_copy_elt *) aa;
  
    return (hashval_t) a->version;
  }
*************** loop_parallel_p (struct loop *loop, stru
*** 108,113 ****
--- 334,340 ----
    lambda_trans_matrix trans;
    bool ret = false;
    tree phi;
+   loop_vec_info simple_loop_info;
  
    /* Only consider innermost loops with just one exit.  The innermost-loop
       restriction is not necessary, but it makes things simpler.  */
*************** loop_parallel_p (struct loop *loop, stru
*** 120,125 ****
--- 347,353 ----
    /* We need to know # of iterations, and there should be no uses of values
       defined inside loop outside of it, unless the values are invariants of
       the loop.  */
+ 
    if (!number_of_iterations_exit (loop, exit, niter, false))
      {
        if (dump_file && (dump_flags & TDF_DETAILS))
*************** loop_parallel_p (struct loop *loop, stru
*** 127,141 ****
        return false;
      }
  
    for (phi = phi_nodes (exit->dest); phi; phi = PHI_CHAIN (phi))
      {
        tree val = PHI_ARG_DEF_FROM_EDGE (phi, exit);
  
        if (is_gimple_reg (val))
  	{
  	  if (dump_file && (dump_flags & TDF_DETAILS))
! 	    fprintf (dump_file, "  FAILED: value used outside loop\n");
! 	  return false;
  	}
      }
  
--- 355,462 ----
        return false;
      }
  
+   simple_loop_info = vect_analyze_loop_form (loop);
+ 
+   for (phi = phi_nodes (loop->header); phi; phi = PHI_CHAIN (phi))
+     {
+       tree reduc_stmt = NULL, operation;
+ 
+       /* ??? TODO: Change this into a generic function that 
+          recognizes reductions.  */
+       if (!is_gimple_reg (PHI_RESULT (phi)))
+         continue;
+       if (simple_loop_info)
+         reduc_stmt = vect_is_simple_reduction (simple_loop_info, phi);
+ 	
+       /* ??? Handle only PLUS reductions for now.  
+          Create a reduction_info struct, initialize it and insert it to 
+          the reduction list.  */
+ 
+       if (reduc_stmt)
+ 	{
+ 	  PTR *slot;
+ 	  struct reduction_info *new_reduction;
+ 
+ 	  if (dump_file && (dump_flags & TDF_DETAILS))
+ 	    {
+ 	      fprintf (dump_file,
+ 		       "Detected reduction. reduction stmt is: \n");
+ 	      print_generic_stmt (dump_file, reduc_stmt, 0);
+ 	      fprintf (dump_file, "\n");
+ 	    }
+ 	  if (!reduction_list)
+ 	    reduction_list = htab_create (10, reduction_info_hash,
+ 					  reduction_info_eq, free);
+ 
+ 	  new_reduction =
+ 	    (struct reduction_info *) xcalloc (1,
+ 					       sizeof (struct
+ 						       reduction_info));
+ 
+ 	  new_reduction->reduc_stmt = reduc_stmt;
+ 	  new_reduction->reduc_phi = phi;
+ 	  operation =  GIMPLE_STMT_OPERAND (reduc_stmt, 1);
+ 	  new_reduction->reduction_code = TREE_CODE (operation);
+ 	  slot = htab_find_slot (reduction_list, new_reduction, INSERT);
+ 	  *slot = new_reduction;
+ 	}
+     }
+ 
    for (phi = phi_nodes (exit->dest); phi; phi = PHI_CHAIN (phi))
      {
+       struct reduction_info tmpred, *red;
+       imm_use_iterator imm_iter;
+       use_operand_p use_p;
+       tree reduc_phi;
+ 
        tree val = PHI_ARG_DEF_FROM_EDGE (phi, exit);
  
        if (is_gimple_reg (val))
  	{
  	  if (dump_file && (dump_flags & TDF_DETAILS))
! 	    {
! 	      fprintf (dump_file, "phi is ");
! 	      print_generic_expr (dump_file, phi, 0);
! 	      fprintf (dump_file, "arg of phi to exit:   value ");
! 	      print_generic_expr (dump_file, val, 0);
! 	      fprintf (dump_file, " used outside loop\n");
! 	      fprintf (dump_file,
! 		       "  checking if it a part of reduction pattern:  \n");
! 	    }
! 	  if (!reduction_list)
! 	    {
! 	      if (dump_file && (dump_flags & TDF_DETAILS))
! 		fprintf (dump_file,
! 			 "  FAILED: it is not a part of reduction.\n");
! 	      return false;
! 	    }
! 	  reduc_phi = NULL;
! 	  FOR_EACH_IMM_USE_FAST (use_p, imm_iter, val)
! 	  {
! 	    if (flow_bb_inside_loop_p (loop, bb_for_stmt (USE_STMT (use_p))))
! 	      {
! 		reduc_phi = USE_STMT (use_p);
! 		break;
! 	      }
! 	  }
! 
! 	  tmpred.reduc_phi = reduc_phi;
! 	  red = htab_find (reduction_list, &tmpred);
! 	  if (red == NULL)
! 	    {
! 	      if (dump_file && (dump_flags & TDF_DETAILS))
! 		fprintf (dump_file,
! 			 "  FAILED: it is not a part of reduction.\n");
! 	      return false;
! 	    }
! 	  if (dump_file && (dump_flags & TDF_DETAILS))
! 	    {
! 	      fprintf (dump_file, "reduction phi is  ");
! 	      print_generic_expr (dump_file, red->reduc_phi, 0);
! 	      fprintf (dump_file, "reduction stmt is  ");
! 	      print_generic_expr (dump_file, red->reduc_stmt, 0);
! 	    }
! 
  	}
      }
  
*************** loop_parallel_p (struct loop *loop, stru
*** 149,158 ****
        if (is_gimple_reg (def)
  	  && !simple_iv (loop, phi, def, &iv, true))
  	{
! 	  if (dump_file && (dump_flags & TDF_DETAILS))
! 	    fprintf (dump_file,
! 		     "  FAILED: scalar dependency between iterations\n");
! 	  return false;
  	}
      }
  
--- 470,492 ----
        if (is_gimple_reg (def)
  	  && !simple_iv (loop, phi, def, &iv, true))
  	{
! 	  struct reduction_info tmpred, *red;
! 
! 	  if (!reduction_list)
! 	    {
! 	      if (dump_file && (dump_flags & TDF_DETAILS))
! 		fprintf (dump_file, "  FAILED: non-biv phi node\n");
! 	      return false;
! 	    }
! 	  /* There is recution list.  Check if this is a reduction_phi.  */
! 	  tmpred.reduc_phi = phi;
! 	  red = htab_find (reduction_list, &tmpred);
! 	  if (red == NULL)
! 	    {
! 	      if (dump_file && (dump_flags & TDF_DETAILS))
! 		fprintf (dump_file, "  FAILED: non-biv phi node\n");
! 	      return false;
! 	    }
  	}
      }
  
*************** struct elv_data
*** 251,256 ****
--- 585,669 ----
    bool changed;
  };
  
+ /* Callback for htab_traverse.  Create the initialization statement
+    for reduction described in SLOT, and place it at the preheader of 
+    the loop described in DATA.  */
+ 
+ static int
+ initialize_reductions (void **slot, void *data)
+ {
+   tree t, stmt;
+   tree init, c;
+   basic_block bb;
+   block_stmt_iterator bsi;
+   tree name, name1;
+   int i, n;
+   tree bvar, type;
+ 
+   struct reduction_info *reduc = *slot;
+   struct loop *loop = (struct loop *) data;
+ 
+   /* Create initialization in preheader: 
+      reduction_variable = initialization value of reduction.  */
+ 
+   /* In the phi node at the header, replace the argument coming 
+      from the preheader with the reduction initialization value.  */
+ 
+   /* Create a new variable to initialize the reduction.  */
+   type = TREE_TYPE (PHI_RESULT (reduc->reduc_phi));
+   bvar = create_tmp_var (type, "reduction");
+   add_referenced_var (bvar);
+   /*bvar = SSA_NAME_VAR (TREE_OPERAND (reduc->reduc_stmt,0)); */
+ 
+ 
+   c = build_omp_clause (OMP_CLAUSE_REDUCTION);
+   OMP_CLAUSE_REDUCTION_CODE (c) = /*PLUS_EXPR */ reduc->reduction_code;
+   OMP_CLAUSE_DECL (c) = SSA_NAME_VAR (GIMPLE_STMT_OPERAND (reduc->reduc_stmt, 0));
+ 
+   init = omp_reduction_init (c, TREE_TYPE (bvar));
+   reduc->init = init;
+   
+   t = build_gimple_modify_stmt (bvar, init);
+   name = make_ssa_name (bvar, t);
+ 
+   GIMPLE_STMT_OPERAND (t, 0) = name;
+   SSA_NAME_DEF_STMT (name) = t;
+ 
+   /* Traverse the arguments of REDUC_PHI, to replace the argument 
+      representing the initialization value.  Keeping the old value 
+      in a new variable "reduction_initial", that will be taken in 
+      consideration after the parallel computing is done.  */
+ 
+   for (i = 0, n = PHI_NUM_ARGS (reduc->reduc_phi); i < n; ++i)
+     {
+       tree arg;
+ 
+       if (!flow_bb_inside_loop_p
+ 	  (loop, PHI_ARG_EDGE (reduc->reduc_phi, i)->src))
+ 	{
+ 	  arg = PHI_ARG_DEF (reduc->reduc_phi, i);
+ 
+ 	  /* Create new  variable to hold the initial value.  */
+ 	  type = TREE_TYPE (bvar);
+ 	  bvar = create_tmp_var (type, "reduction_initial");
+ 	  add_referenced_var (bvar);
+ 
+           stmt = build_gimple_modify_stmt (bvar, arg);
+ 	  name1 = make_ssa_name (bvar, stmt);
+ 	  GIMPLE_STMT_OPERAND (stmt, 0) = name1;
+ 	  SSA_NAME_DEF_STMT (name1) = stmt;
+ 	  bb = loop_preheader_edge (loop)->src;
+ 	  bsi = bsi_after_labels (bb);
+ 	  bsi_insert_before (&bsi, stmt, BSI_NEW_STMT);
+ 	  bsi_insert_before (&bsi, t, BSI_NEW_STMT);
+ 	  SET_PHI_ARG_DEF (reduc->reduc_phi, i, name);
+ 	  reduc->initial_value = name1;
+ 	  reduc->reduction_init = name;
+ 	}
+     }
+   return 1;
+ }
+ 
  static tree
  eliminate_local_variables_1 (tree *tp, int *walk_subtrees, void *data)
  {
*************** separate_decls_in_loop_stmt (struct loop
*** 466,471 ****
--- 879,887 ----
  static int
  add_field_for_name (void **slot, void *data)
  {
+   tree stmt;
+   use_operand_p use_p = NULL;
+ 
    struct name_to_copy_elt *elt = *slot;
    tree type = data;
    tree name = ssa_name (elt->version);
*************** add_field_for_name (void **slot, void *d
*** 474,479 ****
--- 890,911 ----
  
    insert_field_into_struct (type, field);
    elt->field = field;
+ 
+   /* Find uses of name to locate if this name is related to 
+      the reduction phi, and if so, record the filed in the reduction struct.  */
+ 
+   if (reduction_list && single_imm_use (elt->new_name, &use_p, &stmt)
+       && TREE_CODE (stmt) == PHI_NODE)
+     {
+       /* check if STMT is a REDUC_PHI of some reduction.  */
+       struct reduction_info tmpred, *red;
+       tmpred.reduc_phi = stmt;
+ 
+       red = htab_find (reduction_list, &tmpred);
+       if (red)
+ 	red->field = field;
+     }
+ 
    return 1;
  }
  
*************** struct clsn_data
*** 490,495 ****
--- 922,1120 ----
    basic_block load_bb;
  };
  
+ /* Since thread computes its own result (local result),
+    this function creates a phi node whose result is the 
+    thread's local result.  */
+ 
+ static int
+ create_phi_for_local_result (void **slot, void *data)
+ {
+   struct reduction_info *reduc = *slot;
+   struct loop *loop = data;
+   edge e;
+   tree new_phi;
+   basic_block store_bb;
+   tree local_res;
+ 
+   store_bb = FALLTHRU_EDGE (loop->latch)->dest;
+ 
+   if (EDGE_PRED (store_bb, 0) == FALLTHRU_EDGE (loop->latch))
+     e = EDGE_PRED (store_bb, 1);
+   else
+     e = EDGE_PRED (store_bb, 0);
+   local_res = make_ssa_name (SSA_NAME_VAR (reduc->reduction_init), NULL_TREE);
+   new_phi = create_phi_node (local_res, store_bb);
+   SSA_NAME_DEF_STMT (local_res) = new_phi;
+   add_phi_arg (new_phi, reduc->init, e);
+   add_phi_arg (new_phi, GIMPLE_STMT_OPERAND (reduc->reduc_stmt, 0),
+ 	       FALLTHRU_EDGE (loop->latch));
+   reduc->new_phi = new_phi;
+ 
+   return 1;
+ }
+ 
+ 
+ 
+ /* Create an atomic instruction for the reduction described in SLOT.  
+    DATA annotates the place in memory the atomic operation relates to,
+    and the basic block it needs to be generated in.  */
+ 
+ static int
+ create_call_for_reduction_1 (void **slot, void *data)
+ {
+   struct reduction_info *reduc = *slot;
+   struct clsn_data *clsn_data = data;
+   block_stmt_iterator bsi;
+   tree type = TREE_TYPE (PHI_RESULT (reduc->reduc_phi));
+   tree struct_type = TREE_TYPE (TREE_TYPE (clsn_data->load));
+   tree load_struct;
+   basic_block bb;
+   basic_block new_bb;
+   edge e;
+   tree t, addr, addr_type, ref, x;
+   tree tmp_load, load, name, stmt, tmp_store;
+ 
+   load_struct = fold_build1 (INDIRECT_REF, struct_type, clsn_data->load);
+   t = build3 (COMPONENT_REF, type, load_struct, reduc->field, NULL_TREE);
+   addr_type = build_pointer_type (type);
+ 
+   addr = build_addr (t, current_function_decl);
+ 
+   /* Create phi node.  */
+   bb = clsn_data->load_bb;
+ 
+   e = split_block (bb, t);
+   new_bb = e->dest;
+   single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
+   /*if (current_loops)
+ 	{
+           remove_bb_from_loops (e->dest);
+ 	  add_bb_to_loop (e->dest, single_succ_edge (new_bb)->dest->loop_father);
+ 	}*/
+ 
+   tmp_load = create_tmp_var (TREE_TYPE (TREE_TYPE (addr)), NULL);
+   add_referenced_var (tmp_load);
+   tmp_load = make_ssa_name (tmp_load, NULL);
+   load = build2 (OMP_ATOMIC_LOAD, void_type_node, tmp_load, addr);
+   SSA_NAME_DEF_STMT (tmp_load) = load;
+   bsi = bsi_start (new_bb);
+   bsi_insert_after (&bsi, load,BSI_NEW_STMT);
+ 
+   e = split_block (bsi.bb, load);
+   new_bb = e->dest;
+   single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
+   /*if (current_loops)
+         {
+           remove_bb_from_loops (e->dest);
+           add_bb_to_loop (e->dest, single_succ_edge (new_bb)->dest->loop_father);
+         }*/
+   bsi = bsi_start (new_bb); 
+   ref = tmp_load;
+   x =
+     fold_build2 (reduc->reduction_code, TREE_TYPE (PHI_RESULT (reduc->new_phi)), ref, PHI_RESULT (reduc->new_phi));
+ 
+   tmp_store = create_tmp_var (TREE_TYPE (PHI_RESULT (reduc->new_phi)), NULL);
+   add_referenced_var (tmp_store);
+   stmt = build_gimple_modify_stmt (tmp_store, x);
+   name = make_ssa_name (tmp_store, stmt);
+   GIMPLE_STMT_OPERAND (stmt, 0) = name;
+   SSA_NAME_DEF_STMT (name) = stmt;
+   bsi_insert_after (&bsi, stmt, BSI_NEW_STMT);
+  
+   x= build1 (OMP_ATOMIC_STORE, void_type_node, name);
+ 
+   bsi_insert_after (&bsi, x,BSI_NEW_STMT);
+   return 1;
+ }
+ 
+ /* Create the atomic operation at the join point of the threads.  */
+ static void
+ create_call_for_reduction (struct loop *loop, struct clsn_data *clsn_data)
+ {
+   htab_traverse (reduction_list, create_phi_for_local_result, loop);
+   clsn_data->load_bb = FALLTHRU_EDGE (loop->latch)->dest;
+   htab_traverse (reduction_list, create_call_for_reduction_1, clsn_data);
+ }
+ 
+ /* Create a new variable that loads the final reduction value at the  
+    join point of all threads, adds the initial value the reduction 
+    variable had before the parallel computation started, and 
+    inserts it in the right place.  */
+ 
+ static int
+ create_loads_for_reductions (void **slot, void *data)
+ {
+   struct reduction_info *red = *slot;
+   struct clsn_data *clsn_data = data;
+   tree stmt;
+   block_stmt_iterator bsi;
+   tree type = TREE_TYPE (red->reduction_init);
+   tree struct_type = TREE_TYPE (TREE_TYPE (clsn_data->load));
+   tree load_struct;
+   tree bvar, name;
+   imm_use_iterator imm_iter;
+   use_operand_p use_p;
+   tree use_stmt;
+   tree x;
+ 
+   bsi = bsi_after_labels (clsn_data->load_bb);
+   load_struct = fold_build1 (INDIRECT_REF, struct_type, clsn_data->load);
+   load_struct = build3 (COMPONENT_REF, type, load_struct, red->field,
+ 			NULL_TREE);
+   bvar = create_tmp_var (type, "reduction_final");
+   add_referenced_var (bvar);
+ 
+   /* Apply operation between the new variable which is the result
+      of computation all threads, and the initial value which is kept
+      at reduction->inital_value.  */
+ 
+   stmt = build_gimple_modify_stmt (bvar, load_struct);
+   name = make_ssa_name (bvar, stmt);
+   GIMPLE_STMT_OPERAND (stmt, 0) = name;
+   SSA_NAME_DEF_STMT (name) = stmt;
+ 
+   bsi_insert_after (&bsi, stmt, BSI_NEW_STMT);
+ 
+   x =
+     fold_build2 (red->reduction_code, TREE_TYPE (load_struct),
+ 		 name, red->initial_value);
+   stmt = build_gimple_modify_stmt (bvar, x);
+   name = make_ssa_name (bvar, stmt);
+   GIMPLE_STMT_OPERAND (stmt, 0) = name;
+   SSA_NAME_DEF_STMT (name) = stmt;
+ 
+   bsi_insert_after (&bsi, stmt, BSI_NEW_STMT);
+ 
+   FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, PHI_RESULT (red->keep_res))
+     FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter) 
+       SET_USE (use_p, name);
+ 
+   SET_PHI_RESULT (red->keep_res, NULL_TREE);
+   remove_phi_node (red->keep_res, NULL_TREE, false);
+ 
+   return 1;
+ }
+ 
+ 
+ static void
+ create_final_loads_for_reduction (struct clsn_data *clsn_data)
+ {
+   block_stmt_iterator bsi;
+   tree t;
+ 
+   bsi = bsi_after_labels (clsn_data->load_bb);
+   t = build_fold_addr_expr (clsn_data->store);
+   t = build_gimple_modify_stmt (clsn_data->load,build_fold_addr_expr (clsn_data->store));
+ 
+   bsi_insert_before (&bsi, t, BSI_NEW_STMT);
+   SSA_NAME_DEF_STMT (clsn_data->load) = t;
+   GIMPLE_STMT_OPERAND (t, 0) = clsn_data->load;
+ 
+   htab_traverse (reduction_list, create_loads_for_reductions, clsn_data);
+ 
+ }
+ 
+ 
  static int
  create_loads_and_stores_for_name (void **slot, void *data)
  {
*************** create_loads_and_stores_for_name (void *
*** 552,559 ****
     */
  
  static void
! separate_decls_in_loop (struct loop *loop, tree *arg_struct,
! 			tree *new_arg_struct)
  {
    basic_block bb1 = split_edge (loop_preheader_edge (loop));
    basic_block bb0 = single_pred (bb1);
--- 1177,1184 ----
     */
  
  static void
! separate_decls_in_loop (struct loop *loop, tree * arg_struct,
! 			tree * new_arg_struct, struct clsn_data *clsn_data)
  {
    basic_block bb1 = split_edge (loop_preheader_edge (loop));
    basic_block bb0 = single_pred (bb1);
*************** separate_decls_in_loop (struct loop *loo
*** 565,571 ****
    unsigned i;
    tree phi, type, type_name, nvar;
    block_stmt_iterator bsi;
!   struct clsn_data clsn_data;
  
    /* Find and rename the ssa names defined outside of loop.  */
    for (i = 0; i < loop->num_nodes; i++)
--- 1190,1196 ----
    unsigned i;
    tree phi, type, type_name, nvar;
    block_stmt_iterator bsi;
!   struct clsn_data clsn_data1;
  
    /* Find and rename the ssa names defined outside of loop.  */
    for (i = 0; i < loop->num_nodes; i++)
*************** separate_decls_in_loop (struct loop *loo
*** 606,617 ****
        add_referenced_var (nvar);
        *new_arg_struct = make_ssa_name (nvar, NULL_TREE);
  
!       clsn_data.store = *arg_struct;
!       clsn_data.load = *new_arg_struct;
!       clsn_data.store_bb = bb0;
!       clsn_data.load_bb = bb1;
        htab_traverse (name_copies, create_loads_and_stores_for_name,
! 		     &clsn_data);
      }
  
    htab_delete (decl_copies);
--- 1231,1257 ----
        add_referenced_var (nvar);
        *new_arg_struct = make_ssa_name (nvar, NULL_TREE);
  
!       /* We should mark *arg_struct call clobbered.  However, this means
! 		  we would need to update virtual operands for every function call.
! 		  To avoid this, we pretend this variable is volatile.  */
!       TREE_THIS_VOLATILE (*arg_struct) = 1;
! 
!       clsn_data->store = *arg_struct;
!       clsn_data->load = *new_arg_struct;
!       clsn_data->store_bb = bb0;
!       clsn_data->load_bb = bb1;
        htab_traverse (name_copies, create_loads_and_stores_for_name,
! 		     clsn_data);
! 
!       /* Load the calculation from memory into a new 
!          reduction variable (after the join of the threads).  */
!       if (reduction_list)
! 	{
! 	  clsn_data1.load = make_ssa_name (nvar, NULL_TREE);
! 	  clsn_data1.load_bb = single_dom_exit (loop)->dest;
! 	  clsn_data1.store = clsn_data->store;
! 	  create_final_loads_for_reduction (&clsn_data1);
! 	}
      }
  
    htab_delete (decl_copies);
*************** canonicalize_loop_ivs (struct loop *loop
*** 727,741 ****
        res = PHI_RESULT (phi);
  
        if (!is_gimple_reg (res)
! 	  || res == var_before)
  	{
  	  prev = phi;
  	  continue;
  	}
-       
-       ok = simple_iv (loop, phi, res, &iv, true);
-       gcc_assert (ok);
- 
        remove_phi_node (phi, prev, false);
  
        atype = TREE_TYPE (res);
--- 1367,1384 ----
        res = PHI_RESULT (phi);
  
        if (!is_gimple_reg (res)
! 		   || res == var_before)
! 		 {
! 		   prev = phi;
! 		   continue;
! 		 }
!       
!       ok = simple_iv (loop, phi, res, &iv, true);
!       if (!ok)
  	{
  	  prev = phi;
  	  continue;
  	}
        remove_phi_node (phi, prev, false);
  
        atype = TREE_TYPE (res);
*************** transform_to_exit_first_loop (struct loo
*** 834,845 ****
        if (!is_gimple_reg (res))
  	continue;
  
!       gcc_assert (control_name == NULL_TREE
! 		  && SSA_NAME_VAR (res) == SSA_NAME_VAR (control));
        control_name = res;
      }
    gcc_assert (control_name != NULL_TREE);
    phi = SSA_NAME_DEF_STMT (control_name);
    remove_phi_node (phi, NULL_TREE, false);
  
    /* Initialize the control variable to NIT.  */
--- 1477,1509 ----
        if (!is_gimple_reg (res))
  	continue;
  
!       /* check if it is a part of reduction.  If it is,
!          keep the phi at the reduction's keep_res field.  The  
!          PHI_RESULT of this phi is the resulting value of the reduction 
!          variable when exiting the loop.  */
! 
!       exit = single_dom_exit (loop);
! 
!       if (reduction_list)
! 	{
! 	  struct reduction_info tmpred, *red;
! 
! 	  tree val = PHI_ARG_DEF_FROM_EDGE (phi, exit);
! 
! 	  tmpred.reduc_phi = SSA_NAME_DEF_STMT (val);
! 
! 	  red = htab_find (reduction_list, &tmpred);
! 	  if (red)
! 	    red->keep_res = phi;
! 	}
!       else
! 	gcc_assert (control_name == NULL_TREE
! 		    && SSA_NAME_VAR (res) == SSA_NAME_VAR (control));
        control_name = res;
      }
    gcc_assert (control_name != NULL_TREE);
    phi = SSA_NAME_DEF_STMT (control_name);
+   SET_PHI_RESULT (phi, NULL_TREE);
    remove_phi_node (phi, NULL_TREE, false);
  
    /* Initialize the control variable to NIT.  */
*************** gen_parallel_loop (struct loop *loop, un
*** 985,990 ****
--- 1649,1655 ----
    tree many_iterations_cond, type, nit;
    tree stmts, arg_struct, new_arg_struct;
    basic_block parallel_head;
+   struct clsn_data clsn_data;
    unsigned prob;
  
    /* From
*************** gen_parallel_loop (struct loop *loop, un
*** 1082,1097 ****
    /* Ensure that the exit condition is the first statement in the loop.  */
    transform_to_exit_first_loop (loop, nit);
  
    /* Eliminate the references to local variables from the loop.  */
    eliminate_local_variables (loop);
  
    /* In the old loop, move all variables non-local to the loop to a structure
       and back, and create separate decls for the variables used in loop.  */
!   separate_decls_in_loop (loop, &arg_struct, &new_arg_struct);
  
    /* Create the parallel constructs.  */
    parallel_head = create_parallel_loop (loop, create_loop_fn (), arg_struct,
  					new_arg_struct, n_threads);
  
    scev_reset ();
  
--- 1747,1769 ----
    /* Ensure that the exit condition is the first statement in the loop.  */
    transform_to_exit_first_loop (loop, nit);
  
+ 
+   /* Generate intializations for reductions.  */
+   if (reduction_list)
+     htab_traverse (reduction_list, initialize_reductions, loop);
+ 
    /* Eliminate the references to local variables from the loop.  */
    eliminate_local_variables (loop);
  
    /* In the old loop, move all variables non-local to the loop to a structure
       and back, and create separate decls for the variables used in loop.  */
!   separate_decls_in_loop (loop, &arg_struct, &new_arg_struct, &clsn_data);
  
    /* Create the parallel constructs.  */
    parallel_head = create_parallel_loop (loop, create_loop_fn (), arg_struct,
  					new_arg_struct, n_threads);
+   if (reduction_list)
+     create_call_for_reduction (loop, &clsn_data);
  
    scev_reset ();
  
*************** parallelize_loops (void)
*** 1125,1130 ****
--- 1797,1803 ----
  
    FOR_EACH_LOOP (li, loop, 0)
      {
+       reduction_list = NULL;
        if (/* Do not bother with loops in cold areas.  */
  	  !maybe_hot_bb_p (loop->header)
  	  /* Or loops that roll too little.  */
Index: tree-vectorizer.h
===================================================================
*** tree-vectorizer.h	(revision 128603)
--- tree-vectorizer.h	(working copy)
*************** extern stmt_vec_info new_stmt_vec_info (
*** 666,672 ****
  /* Driver for analysis stage.  */
  extern loop_vec_info vect_analyze_loop (struct loop *);
  extern void vect_free_slp_tree (slp_tree);
! 
  
  /** In tree-vect-patterns.c  **/
  /* Pattern recognition functions.
--- 666,672 ----
  /* Driver for analysis stage.  */
  extern loop_vec_info vect_analyze_loop (struct loop *);
  extern void vect_free_slp_tree (slp_tree);
! extern loop_vec_info vect_analyze_loop_form (struct loop *);
  
  /** In tree-vect-patterns.c  **/
  /* Pattern recognition functions.
Index: tree-vect-analyze.c
===================================================================
*** tree-vect-analyze.c	(revision 128603)
--- tree-vect-analyze.c	(working copy)
*************** along with GCC; see the file COPYING3.  
*** 42,48 ****
  #include "recog.h"
  
  /* Main analysis functions.  */
- static loop_vec_info vect_analyze_loop_form (struct loop *);
  static bool vect_analyze_data_refs (loop_vec_info);
  static bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
  static void vect_analyze_scalar_cycles (loop_vec_info);
--- 42,47 ----
*************** vect_analyze_loop_1 (struct loop *loop)
*** 3949,3955 ****
     - the loop exit condition is simple enough, and the number of iterations
       can be analyzed (a countable loop).  */
  
! static loop_vec_info
  vect_analyze_loop_form (struct loop *loop)
  {
    loop_vec_info loop_vinfo;
--- 3948,3954 ----
     - the loop exit condition is simple enough, and the number of iterations
       can be analyzed (a countable loop).  */
  
! loop_vec_info
  vect_analyze_loop_form (struct loop *loop)
  {
    loop_vec_info loop_vinfo;
Index: gimplify.c
===================================================================
*** gimplify.c	(revision 128603)
--- gimplify.c	(working copy)
*************** gimplify_omp_workshare (tree *expr_p, tr
*** 5248,5254 ****
     EXPR is this stabilized form.  */
  
  static bool
! goa_lhs_expr_p (const_tree expr, const_tree addr)
  {
    /* Also include casts to other type variants.  The C front end is fond
       of adding these for e.g. volatile variables.  This is like 
--- 5248,5254 ----
     EXPR is this stabilized form.  */
  
  static bool
! goa_lhs_expr_p (tree expr, tree addr)
  {
    /* Also include casts to other type variants.  The C front end is fond
       of adding these for e.g. volatile variables.  This is like 
*************** goa_lhs_expr_p (const_tree expr, const_t
*** 5268,5333 ****
    return false;
  }
  
! /* A subroutine of gimplify_omp_atomic.  Attempt to implement the atomic
!    operation as a __sync_fetch_and_op builtin.  INDEX is log2 of the
!    size of the data type, and thus usable to find the index of the builtin
!    decl.  Returns GS_UNHANDLED if the expression is not of the proper form.  */
! 
! static enum gimplify_status
! gimplify_omp_atomic_fetch_op (tree *expr_p, tree addr, tree rhs, int index)
! {
!   enum built_in_function base;
!   tree decl, itype;
!   enum insn_code *optab;
! 
!   /* Check for one of the supported fetch-op operations.  */
!   switch (TREE_CODE (rhs))
!     {
!     case POINTER_PLUS_EXPR:
!     case PLUS_EXPR:
!       base = BUILT_IN_FETCH_AND_ADD_N;
!       optab = sync_add_optab;
!       break;
!     case MINUS_EXPR:
!       base = BUILT_IN_FETCH_AND_SUB_N;
!       optab = sync_add_optab;
!       break;
!     case BIT_AND_EXPR:
!       base = BUILT_IN_FETCH_AND_AND_N;
!       optab = sync_and_optab;
!       break;
!     case BIT_IOR_EXPR:
!       base = BUILT_IN_FETCH_AND_OR_N;
!       optab = sync_ior_optab;
!       break;
!     case BIT_XOR_EXPR:
!       base = BUILT_IN_FETCH_AND_XOR_N;
!       optab = sync_xor_optab;
!       break;
!     default:
!       return GS_UNHANDLED;
!     }
! 
!   /* Make sure the expression is of the proper form.  */
!   if (goa_lhs_expr_p (TREE_OPERAND (rhs, 0), addr))
!     rhs = TREE_OPERAND (rhs, 1);
!   else if (commutative_tree_code (TREE_CODE (rhs))
! 	   && goa_lhs_expr_p (TREE_OPERAND (rhs, 1), addr))
!     rhs = TREE_OPERAND (rhs, 0);
!   else
!     return GS_UNHANDLED;
! 
!   decl = built_in_decls[base + index + 1];
!   itype = TREE_TYPE (TREE_TYPE (decl));
! 
!   if (optab[TYPE_MODE (itype)] == CODE_FOR_nothing)
!     return GS_UNHANDLED;
! 
!   *expr_p = build_call_expr (decl, 2, addr, fold_convert (itype, rhs));
!   return GS_OK;
! }
! 
! /* A subroutine of gimplify_omp_atomic_pipeline.  Walk *EXPR_P and replace
     appearances of *LHS_ADDR with LHS_VAR.  If an expression does not involve
     the lhs, evaluate it into a temporary.  Return 1 if the lhs appeared as
     a subexpression, 0 if it did not, or -1 if an error was encountered.  */
--- 5268,5274 ----
    return false;
  }
  
! /* Walk *EXPR_P and replace
     appearances of *LHS_ADDR with LHS_VAR.  If an expression does not involve
     the lhs, evaluate it into a temporary.  Return 1 if the lhs appeared as
     a subexpression, 0 if it did not, or -1 if an error was encountered.  */
*************** goa_stabilize_expr (tree *expr_p, tree *
*** 5371,5514 ****
    return saw_lhs;
  }
  
- /* A subroutine of gimplify_omp_atomic.  Implement the atomic operation as:
- 
- 	oldval = *addr;
-       repeat:
- 	newval = rhs;	// with oldval replacing *addr in rhs
- 	oldval = __sync_val_compare_and_swap (addr, oldval, newval);
- 	if (oldval != newval)
- 	  goto repeat;
- 
-    INDEX is log2 of the size of the data type, and thus usable to find the
-    index of the builtin decl.  */
- 
- static enum gimplify_status
- gimplify_omp_atomic_pipeline (tree *expr_p, tree *pre_p, tree addr,
- 			      tree rhs, int index)
- {
-   tree oldval, oldival, oldival2, newval, newival, label;
-   tree type, itype, cmpxchg, x, iaddr;
- 
-   cmpxchg = built_in_decls[BUILT_IN_VAL_COMPARE_AND_SWAP_N + index + 1];
-   type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
-   itype = TREE_TYPE (TREE_TYPE (cmpxchg));
- 
-   if (sync_compare_and_swap[TYPE_MODE (itype)] == CODE_FOR_nothing)
-     return GS_UNHANDLED;
- 
-   oldval = create_tmp_var (type, NULL);
-   newval = create_tmp_var (type, NULL);
- 
-   /* Precompute as much of RHS as possible.  In the same walk, replace
-      occurrences of the lhs value with our temporary.  */
-   if (goa_stabilize_expr (&rhs, pre_p, addr, oldval) < 0)
-     return GS_ERROR;
- 
-   x = build_fold_indirect_ref (addr);
-   x = build_gimple_modify_stmt (oldval, x);
-   gimplify_and_add (x, pre_p);
- 
-   /* For floating-point values, we'll need to view-convert them to integers
-      so that we can perform the atomic compare and swap.  Simplify the 
-      following code by always setting up the "i"ntegral variables.  */
-   if (INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
-     {
-       oldival = oldval;
-       newival = newval;
-       iaddr = addr;
-     }
-   else
-     {
-       oldival = create_tmp_var (itype, NULL);
-       newival = create_tmp_var (itype, NULL);
- 
-       x = build1 (VIEW_CONVERT_EXPR, itype, oldval);
-       x = build_gimple_modify_stmt (oldival, x);
-       gimplify_and_add (x, pre_p);
-       iaddr = fold_convert (build_pointer_type (itype), addr);
-     }
- 
-   oldival2 = create_tmp_var (itype, NULL);
- 
-   label = create_artificial_label ();
-   x = build1 (LABEL_EXPR, void_type_node, label);
-   gimplify_and_add (x, pre_p);
- 
-   x = build_gimple_modify_stmt (newval, rhs);
-   gimplify_and_add (x, pre_p);
- 
-   if (newval != newival)
-     {
-       x = build1 (VIEW_CONVERT_EXPR, itype, newval);
-       x = build_gimple_modify_stmt (newival, x);
-       gimplify_and_add (x, pre_p);
-     }
- 
-   x = build_gimple_modify_stmt (oldival2, fold_convert (itype, oldival));
-   gimplify_and_add (x, pre_p);
- 
-   x = build_call_expr (cmpxchg, 3, iaddr, fold_convert (itype, oldival),
- 		       fold_convert (itype, newival));
-   if (oldval == oldival)
-     x = fold_convert (type, x);
-   x = build_gimple_modify_stmt (oldival, x);
-   gimplify_and_add (x, pre_p);
- 
-   /* For floating point, be prepared for the loop backedge.  */
-   if (oldval != oldival)
-     {
-       x = build1 (VIEW_CONVERT_EXPR, type, oldival);
-       x = build_gimple_modify_stmt (oldval, x);
-       gimplify_and_add (x, pre_p);
-     }
- 
-   /* Note that we always perform the comparison as an integer, even for
-      floating point.  This allows the atomic operation to properly 
-      succeed even with NaNs and -0.0.  */
-   x = build3 (COND_EXPR, void_type_node,
- 	      build2 (NE_EXPR, boolean_type_node,
- 		      fold_convert (itype, oldival), oldival2),
- 	      build1 (GOTO_EXPR, void_type_node, label), NULL);
-   gimplify_and_add (x, pre_p);
- 
-   *expr_p = NULL;
-   return GS_ALL_DONE;
- }
- 
- /* A subroutine of gimplify_omp_atomic.  Implement the atomic operation as:
- 
- 	GOMP_atomic_start ();
- 	*addr = rhs;
- 	GOMP_atomic_end ();
- 
-    The result is not globally atomic, but works so long as all parallel
-    references are within #pragma omp atomic directives.  According to
-    responses received from omp@openmp.org, appears to be within spec.
-    Which makes sense, since that's how several other compilers handle
-    this situation as well.  */
- 
- static enum gimplify_status
- gimplify_omp_atomic_mutex (tree *expr_p, tree *pre_p, tree addr, tree rhs)
- {
-   tree t;
- 
-   t = built_in_decls[BUILT_IN_GOMP_ATOMIC_START];
-   t = build_call_expr (t, 0);
-   gimplify_and_add (t, pre_p);
- 
-   t = build_fold_indirect_ref (addr);
-   t = build_gimple_modify_stmt (t, rhs);
-   gimplify_and_add (t, pre_p);
-   
-   t = built_in_decls[BUILT_IN_GOMP_ATOMIC_END];
-   t = build_call_expr (t, 0);
-   gimplify_and_add (t, pre_p);
- 
-   *expr_p = NULL;
-   return GS_ALL_DONE;
- }
- 
  /* Gimplify an OMP_ATOMIC statement.  */
  
  static enum gimplify_status
--- 5312,5317 ----
*************** gimplify_omp_atomic (tree *expr_p, tree 
*** 5517,5562 ****
    tree addr = TREE_OPERAND (*expr_p, 0);
    tree rhs = TREE_OPERAND (*expr_p, 1);
    tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
!   HOST_WIDE_INT index;
  
!   /* Make sure the type is one of the supported sizes.  */
!   index = tree_low_cst (TYPE_SIZE_UNIT (type), 1);
!   index = exact_log2 (index);
!   if (index >= 0 && index <= 4)
!     {
!       enum gimplify_status gs;
!       unsigned int align;
  
!       if (DECL_P (TREE_OPERAND (addr, 0)))
! 	align = DECL_ALIGN_UNIT (TREE_OPERAND (addr, 0));
!       else if (TREE_CODE (TREE_OPERAND (addr, 0)) == COMPONENT_REF
! 	       && TREE_CODE (TREE_OPERAND (TREE_OPERAND (addr, 0), 1))
! 		  == FIELD_DECL)
! 	align = DECL_ALIGN_UNIT (TREE_OPERAND (TREE_OPERAND (addr, 0), 1));
!       else
! 	align = TYPE_ALIGN_UNIT (type);
  
-       /* __sync builtins require strict data alignment.  */
-       if (exact_log2 (align) >= index)
- 	{
- 	  /* When possible, use specialized atomic update functions.  */
- 	  if (INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
- 	    {
- 	      gs = gimplify_omp_atomic_fetch_op (expr_p, addr, rhs, index);
- 	      if (gs != GS_UNHANDLED)
- 		return gs;
- 	    }
- 
- 	  /* If we don't have specialized __sync builtins, try and implement
- 	     as a compare and swap loop.  */
- 	  gs = gimplify_omp_atomic_pipeline (expr_p, pre_p, addr, rhs, index);
- 	  if (gs != GS_UNHANDLED)
- 	    return gs;
- 	}
-     }
- 
-   /* The ultimate fallback is wrapping the operation in a mutex.  */
-   return gimplify_omp_atomic_mutex (expr_p, pre_p, addr, rhs);
  }
  
  /*  Gimplifies the expression tree pointed to by EXPR_P.  Return 0 if
--- 5320,5345 ----
    tree addr = TREE_OPERAND (*expr_p, 0);
    tree rhs = TREE_OPERAND (*expr_p, 1);
    tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
!   tree tmp_load, load, store;
  
!    tmp_load = create_tmp_var (type, NULL);
!    if (goa_stabilize_expr (&rhs, pre_p, addr, tmp_load) < 0)
!      return GS_ERROR;
! 
!    if (gimplify_expr (&addr, pre_p, NULL, is_gimple_val, fb_rvalue)
!        != GS_ALL_DONE)
!      return GS_ERROR;
! 
!    load = build2 (OMP_ATOMIC_LOAD, void_type_node, tmp_load, addr);
!    append_to_statement_list (load, pre_p);
!    if (gimplify_expr (&rhs, pre_p, NULL, is_gimple_val, fb_rvalue)
!        != GS_ALL_DONE)
!      return GS_ERROR;
!    store = build1 (OMP_ATOMIC_STORE, void_type_node, rhs);
!    *expr_p = store;
  
!    return GS_ALL_DONE;
  
  }
  
  /*  Gimplifies the expression tree pointed to by EXPR_P.  Return 0 if
*************** gimplify_expr (tree *expr_p, tree *pre_p
*** 6032,6037 ****
--- 5815,5823 ----
  
  	case OMP_RETURN:
  	case OMP_CONTINUE:
+         case OMP_ATOMIC_LOAD:
+         case OMP_ATOMIC_STORE:
+ 
  	  ret = GS_ALL_DONE;
  	  break;
  
Index: tree.def
===================================================================
*** tree.def	(revision 128603)
--- tree.def	(working copy)
*************** DEFTREECODE (OMP_CONTINUE, "omp_continue
*** 1059,1064 ****
--- 1059,1076 ----
  	build_fold_indirect_ref of the address.  */
  DEFTREECODE (OMP_ATOMIC, "omp_atomic", tcc_statement, 2)
  
+ /* Codes used for lowering of OMP_ATOMIC.  Although the form of the OMP_ATOMIC
+    statement is very simple (just in form mem op= expr), various implicit
+    conversions may cause the expression become more complex, so that it does
+    not fit the gimple grammar very well.  To overcome this problem, OMP_ATOMIC
+    is rewritten as a sequence of two codes in gimplification:
+ 
+    OMP_LOAD (tmp, mem)
+    val = some computations involving tmp;
+    OMP_STORE (val)  */
+ DEFTREECODE (OMP_ATOMIC_LOAD, "omp_atomic_load", tcc_statement, 2)
+ DEFTREECODE (OMP_ATOMIC_STORE, "omp_atomic_store", tcc_statement, 1)
+ 
  /* OpenMP clauses.  */
  DEFTREECODE (OMP_CLAUSE, "omp_clause", tcc_exceptional, 0)
  
Index: tree-inline.c
===================================================================
*** tree-inline.c	(revision 128603)
--- tree-inline.c	(working copy)
*************** estimate_num_insns_1 (tree *tp, int *wal
*** 2110,2115 ****
--- 2110,2116 ----
      case OMP_RETURN:
      case OMP_CONTINUE:
      case OMP_SECTIONS_SWITCH:
+     case OMP_ATOMIC_STORE:
        break;
  
      /* We don't account constants for now.  Assume that the cost is amortized
*************** estimate_num_insns_1 (tree *tp, int *wal
*** 2336,2341 ****
--- 2337,2343 ----
      case OMP_ORDERED:
      case OMP_CRITICAL:
      case OMP_ATOMIC:
+     case OMP_ATOMIC_LOAD:
        /* OpenMP directives are generally very expensive.  */
        d->count += d->weights->omp_cost;
        break;
Index: Makefile.in
===================================================================
*** Makefile.in	(revision 128603)
--- Makefile.in	(working copy)
*************** tree-loop-linear.o: tree-loop-linear.c $
*** 2273,2279 ****
     $(TARGET_H) tree-chrec.h $(OBSTACK_H)
  tree-parloops.o: tree-parloops.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
     $(TREE_FLOW_H) $(TREE_H) $(RTL_H) $(CFGLOOP_H) $(TREE_DATA_REF_H) $(GGC_H) \
!    $(DIAGNOSTIC_H) tree-pass.h $(SCEV_H) langhooks.h gt-tree-parloops.h
  tree-stdarg.o: tree-stdarg.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
     $(TREE_H) $(FUNCTION_H) $(DIAGNOSTIC_H) $(TREE_FLOW_H) tree-pass.h \
     tree-stdarg.h $(TARGET_H) langhooks.h
--- 2273,2280 ----
     $(TARGET_H) tree-chrec.h $(OBSTACK_H)
  tree-parloops.o: tree-parloops.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
     $(TREE_FLOW_H) $(TREE_H) $(RTL_H) $(CFGLOOP_H) $(TREE_DATA_REF_H) $(GGC_H) \
!    $(DIAGNOSTIC_H) tree-pass.h $(SCEV_H) langhooks.h gt-tree-parloops.h \
!    tree-vectorizer.h
  tree-stdarg.o: tree-stdarg.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
     $(TREE_H) $(FUNCTION_H) $(DIAGNOSTIC_H) $(TREE_FLOW_H) tree-pass.h \
     tree-stdarg.h $(TARGET_H) langhooks.h
*************** expr.o : expr.c $(CONFIG_H) $(SYSTEM_H) 
*** 2385,2391 ****
     typeclass.h hard-reg-set.h toplev.h hard-reg-set.h except.h reload.h \
     $(GGC_H) langhooks.h intl.h $(TM_P_H) $(REAL_H) $(TARGET_H) \
     tree-iterator.h gt-expr.h $(MACHMODE_H) $(TIMEVAR_H) $(TREE_FLOW_H) \
!    tree-pass.h $(DF_H)
  dojump.o : dojump.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(RTL_H) $(TREE_H) \
     $(FLAGS_H) $(FUNCTION_H) $(EXPR_H) $(OPTABS_H) $(INSN_ATTR_H) insn-config.h \
     langhooks.h $(GGC_H) gt-dojump.h
--- 2386,2392 ----
     typeclass.h hard-reg-set.h toplev.h hard-reg-set.h except.h reload.h \
     $(GGC_H) langhooks.h intl.h $(TM_P_H) $(REAL_H) $(TARGET_H) \
     tree-iterator.h gt-expr.h $(MACHMODE_H) $(TIMEVAR_H) $(TREE_FLOW_H) \
!    tree-pass.h $(DF_H) $(DIAGNOSTIC_H)
  dojump.o : dojump.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(RTL_H) $(TREE_H) \
     $(FLAGS_H) $(FUNCTION_H) $(EXPR_H) $(OPTABS_H) $(INSN_ATTR_H) insn-config.h \
     langhooks.h $(GGC_H) gt-dojump.h
Index: tree-cfg.c
===================================================================
*** tree-cfg.c	(revision 128603)
--- tree-cfg.c	(working copy)
*************** make_edges (void)
*** 525,530 ****
--- 525,537 ----
  	      fallthru = false;
  	      break;
  
+ 
+             case OMP_ATOMIC_LOAD:
+             case OMP_ATOMIC_STORE:
+                fallthru = true;
+                break;
+ 
+ 
  	    case OMP_RETURN:
  	      /* In the case of an OMP_SECTION, the edge will go somewhere
  		 other than the next block.  This will be created later.  */
Index: tree-ssa-operands.c
===================================================================
*** tree-ssa-operands.c	(revision 128603)
--- tree-ssa-operands.c	(working copy)
*************** add_stmt_operand (tree *var_p, stmt_ann_
*** 1620,1657 ****
      add_virtual_operand (var, s_ann, flags, NULL_TREE, 0, -1, false);
  }
  
! 
! /* A subroutine of get_expr_operands to handle INDIRECT_REF,
!    ALIGN_INDIRECT_REF and MISALIGNED_INDIRECT_REF.  
! 
!    STMT is the statement being processed, EXPR is the INDIRECT_REF
!       that got us here.
!    
!    FLAGS is as in get_expr_operands.
! 
!    FULL_REF contains the full pointer dereference expression, if we
!       have it, or NULL otherwise.
! 
!    OFFSET and SIZE are the location of the access inside the
!       dereferenced pointer, if known.
! 
!    RECURSE_ON_BASE should be set to true if we want to continue
!       calling get_expr_operands on the base pointer, and false if
!       something else will do it for us.  */
  
  static void
! get_indirect_ref_operands (tree stmt, tree expr, int flags,
! 			   tree full_ref,
! 			   HOST_WIDE_INT offset, HOST_WIDE_INT size,
! 			   bool recurse_on_base)
! {
!   tree *pptr = &TREE_OPERAND (expr, 0);
!   tree ptr = *pptr;
    stmt_ann_t s_ann = stmt_ann (stmt);
  
    s_ann->references_memory = true;
-   if (TREE_THIS_VOLATILE (expr))
-     s_ann->has_volatile_ops = true; 
  
    if (SSA_VAR_P (ptr))
      {
--- 1620,1639 ----
      add_virtual_operand (var, s_ann, flags, NULL_TREE, 0, -1, false);
  }
  
! /* Subroutine of get_indirect_ref_operands.  ADDR is the address
!    that is dereferenced, the meaning of the rest of the arguments
!    is the same as in get_indirect_ref_operands.  */
  
  static void
! get_addr_dereference_operands (tree stmt, tree *addr, int flags,
!                                                       tree full_ref,
!                                                        HOST_WIDE_INT offset, HOST_WIDE_INT size,
!                                                        bool recurse_on_base)
!   {
!  tree ptr = *addr;
    stmt_ann_t s_ann = stmt_ann (stmt);
  
    s_ann->references_memory = true;
  
    if (SSA_VAR_P (ptr))
      {
*************** get_indirect_ref_operands (tree stmt, tr
*** 1718,1726 ****
  
    /* If requested, add a USE operand for the base pointer.  */
    if (recurse_on_base)
!     get_expr_operands (stmt, pptr, opf_use);
  }
  
  
  /* A subroutine of get_expr_operands to handle TARGET_MEM_REF.  */
  
--- 1700,1741 ----
  
    /* If requested, add a USE operand for the base pointer.  */
    if (recurse_on_base)
!     get_expr_operands (stmt, addr, opf_use);
  }
  
+ /* A subroutine of get_expr_operands to handle INDIRECT_REF,
+    ALIGN_INDIRECT_REF and MISALIGNED_INDIRECT_REF.  
+ 
+    STMT is the statement being processed, EXPR is the INDIRECT_REF
+       that got us here.
+    
+    FLAGS is as in get_expr_operands.
+ 
+    FULL_REF contains the full pointer dereference expression, if we
+       have it, or NULL otherwise.
+ 
+    OFFSET and SIZE are the location of the access inside the
+       dereferenced pointer, if known.
+ 
+    RECURSE_ON_BASE should be set to true if we want to continue
+       calling get_expr_operands on the base pointer, and false if
+       something else will do it for us.  */
+ 
+ static void
+ get_indirect_ref_operands (tree stmt, tree expr, int flags,
+ 		 		 		    tree full_ref,
+ 		 		 		    HOST_WIDE_INT offset, HOST_WIDE_INT size,
+ 		 		 		    bool recurse_on_base)
+ {
+   tree *pptr = &TREE_OPERAND (expr, 0);
+   stmt_ann_t s_ann = stmt_ann (stmt);
+ 
+   if (TREE_THIS_VOLATILE (expr))
+     s_ann->has_volatile_ops = true; 
+ 
+   get_addr_dereference_operands (stmt, pptr, flags, full_ref,
+ 		 		 		 		  offset, size, recurse_on_base);
+ }
  
  /* A subroutine of get_expr_operands to handle TARGET_MEM_REF.  */
  
*************** get_expr_operands (tree stmt, tree *expr
*** 2325,2330 ****
--- 2340,2364 ----
  	return;
        }
  
+     case OMP_ATOMIC_LOAD:
+       {
+ 		 tree *addr = &TREE_OPERAND (expr, 1);
+ 		 get_expr_operands (stmt, &TREE_OPERAND (expr, 0), opf_def);
+ 
+ 		 if (TREE_CODE (*addr) == ADDR_EXPR)
+ 		   get_expr_operands (stmt, &TREE_OPERAND (*addr, 0), opf_def);
+ 		 else
+ 		   get_addr_dereference_operands (stmt, addr, opf_def,
+ 		 		 		 		 		  NULL_TREE, 0, -1, true);
+ 		 return;
+       }
+ 
+     case OMP_ATOMIC_STORE:
+       {
+ 		 get_expr_operands (stmt, &TREE_OPERAND (expr, 0), opf_use);
+ 		 return;
+       }
+ 
      case BLOCK:
      case FUNCTION_DECL:
      case EXC_PTR_EXPR:
