[patch] Generation of nontemporal stores
Zdenek Dvorak
rakdver@kam.mff.cuni.cz
Sun Jun 10 19:38:00 GMT 2007
Hello,
this patch makes us use nontemporal store instruction if it is
available, for stores in loops for that we are sure that their value is
not reused -- i.e., the stores are independent on all other memory references
in the loop, and the amount of memory accessed in the loop is greater than the
size of L2 cache.
We cannot safely weaken the restriction that the stores are completely
independent (e.g., replacing it with the condition that the dependence
distance is large enough, as we do in the detection of nontemporal
prefetchs), as ordering of nontemporal stores is not guaranteed. For
this reason, we also need to emit a fence insn after loops that use
nontemporal stores.
The patch includes
http://gcc.gnu.org/ml/gcc-patches/2007-04/msg01862.html (rtl part of
generation of nontemporal stores), only the new parts are mentioned in
the changelog.
For stream benchmark, we get (on athlon, with -O2 -msse2
-ftree-vectorize -fprefetch-loop-arrays):
without the patch:
Function Rate (MB/s) Avg time Min time Max time
Copy: 2114.3088 0.0152 0.0151 0.0153
Scale: 2172.4429 0.0148 0.0147 0.0149
Add: 2400.6119 0.0202 0.0200 0.0211
Triad: 2433.7177 0.0198 0.0197 0.0199
with the patch:
Function Rate (MB/s) Avg time Min time Max time
Copy: 3186.2644 0.0101 0.0100 0.0102
Scale: 3172.3991 0.0101 0.0101 0.0102
Add: 3375.2731 0.0143 0.0142 0.0144
Triad: 3377.4264 0.0143 0.0142 0.0144
i.e., 40% increase in rate. The patch is basically neutral on spec2006
(on amd64, all changes within noise).
Bootstrapped & regtested on i686 and ia64.
Zdenek
* tree-data-ref.c (dr_analyze_alias): Handle case smt is NULL.
* tree-predcom.c (mark_virtual_ops_for_renaming): Exported.
* tree-ssa-loop-prefetch.c: Include optabs.h.
(FENCE_FOLLOWING_MOVNT): New macro.
(struct mem_ref): Add independent_p and storent_p fields.
(record_ref): Initalize the new fields.
(gather_memory_references_ref): Return true if the reference
could be analysed.
(gather_memory_references): Check whether all memory accesses
in loop were recorded.
(should_issue_prefetch_p): Return false for nontemporal stores.
(nontemporal_store_p, mark_nontemporal_store, emit_mfence_after_loop,
may_use_storent_in_loop_p, mark_nontemporal_stores): New functions.
(determine_loop_nest_reuse): Detect independent memory references.
(loop_prefetch_arrays): Call mark_nontemporal_stores.
* tree-flow.h (mark_virtual_ops_for_renaming): Declare.
* Makefile.in (tree-ssa-loop-prefetch.o): Add OPTABS_H dependency.
* config/i386/i386.h (x86_mfence): Declare.
(FENCE_FOLLOWING_MOVNT): Return x86_mfence.
* config/i386/i386.c (x86_mfence): New variable.
(ix86_init_mmx_sse_builtins): Initialize x86_mfence.
* gcc.dg/tree-ssa/prefetch-7.c: New test.
Index: tree-pretty-print.c
===================================================================
*** tree-pretty-print.c (revision 125532)
--- tree-pretty-print.c (working copy)
*************** dump_generic_node (pretty_printer *buffe
*** 1063,1068 ****
--- 1063,1071 ----
false);
pp_space (buffer);
pp_character (buffer, '=');
+ if (TREE_CODE (node) == GIMPLE_MODIFY_STMT
+ && MOVE_NONTEMPORAL (node))
+ pp_string (buffer, "{nt}");
pp_space (buffer);
dump_generic_node (buffer, GENERIC_TREE_OPERAND (node, 1), spc, flags,
false);
Index: optabs.c
===================================================================
*** optabs.c (revision 125532)
--- optabs.c (working copy)
*************** init_optabs (void)
*** 5492,5497 ****
--- 5492,5499 ----
movstrict_optab = init_optab (STRICT_LOW_PART);
cmp_optab = init_optab (COMPARE);
+ storent_optab = init_optab (UNKNOWN);
+
ucmp_optab = init_optab (UNKNOWN);
tst_optab = init_optab (UNKNOWN);
Index: optabs.h
===================================================================
*** optabs.h (revision 125532)
--- optabs.h (working copy)
*************** enum optab_index
*** 151,156 ****
--- 151,158 ----
OTI_movstrict,
/* Move, with a misaligned memory. */
OTI_movmisalign,
+ /* Nontemporal store. */
+ OTI_storent,
/* Unary operations */
/* Negation */
*************** extern GTY(()) optab optab_table[OTI_MAX
*** 367,372 ****
--- 369,375 ----
#define mov_optab (optab_table[OTI_mov])
#define movstrict_optab (optab_table[OTI_movstrict])
#define movmisalign_optab (optab_table[OTI_movmisalign])
+ #define storent_optab (optab_table[OTI_storent])
#define neg_optab (optab_table[OTI_neg])
#define negv_optab (optab_table[OTI_negv])
Index: genopinit.c
===================================================================
*** genopinit.c (revision 125532)
--- genopinit.c (working copy)
*************** static const char * const optabs[] =
*** 164,169 ****
--- 164,170 ----
"mov_optab->handlers[$A].insn_code = CODE_FOR_$(mov$a$)",
"movstrict_optab->handlers[$A].insn_code = CODE_FOR_$(movstrict$a$)",
"movmisalign_optab->handlers[$A].insn_code = CODE_FOR_$(movmisalign$a$)",
+ "storent_optab->handlers[$A].insn_code = CODE_FOR_$(storent$a$)",
"cmp_optab->handlers[$A].insn_code = CODE_FOR_$(cmp$a$)",
"tst_optab->handlers[$A].insn_code = CODE_FOR_$(tst$a$)",
"addcc_optab->handlers[$A].insn_code = CODE_FOR_$(add$acc$)",
Index: tree.h
===================================================================
*** tree.h (revision 125532)
--- tree.h (working copy)
*************** struct gimple_stmt GTY(())
*** 447,452 ****
--- 447,454 ----
EH_FILTER_MUST_NOT_THROW in EH_FILTER_EXPR
TYPE_REF_CAN_ALIAS_ALL in
POINTER_TYPE, REFERENCE_TYPE
+ MOVE_NONTEMPORAL in
+ GIMPLE_MODIFY_STMT
CASE_HIGH_SEEN in CASE_LABEL_EXPR
public_flag:
*************** extern void omp_clause_range_check_faile
*** 1159,1164 ****
--- 1161,1170 ----
#define TYPE_REF_CAN_ALIAS_ALL(NODE) \
(PTR_OR_REF_CHECK (NODE)->base.static_flag)
+ /* In a MODIFY_EXPR, means that the store in the expression is nontemporal. */
+ #define MOVE_NONTEMPORAL(NODE) \
+ (GIMPLE_MODIFY_STMT_CHECK (NODE)->base.static_flag)
+
/* In an INTEGER_CST, REAL_CST, COMPLEX_CST, or VECTOR_CST, this means
there was an overflow in folding. */
Index: testsuite/gcc.dg/tree-ssa/prefetch-7.c
===================================================================
*** testsuite/gcc.dg/tree-ssa/prefetch-7.c (revision 0)
--- testsuite/gcc.dg/tree-ssa/prefetch-7.c (revision 0)
***************
*** 0 ****
--- 1,59 ----
+ /* { dg-do compile { target i?86-*-* x86_64-*-* } } */
+ /* { dg-require-effective-target ilp32 } */
+ /* { dg-options "-O2 -fprefetch-loop-arrays -march=athlon -msse2 -mfpmath=sse --param simultaneous-prefetches=100 --param max-unrolled-insns=1 -fdump-tree-aprefetch-details -fdump-tree-final_cleanup" } */
+
+ #define K 1000000
+ int a[K], b[K];
+
+ void test(int *p)
+ {
+ unsigned i;
+
+ /* Nontemporal store should be used for a. */
+ for (i = 0; i < K; i++)
+ a[i] = 0;
+
+ /* Nontemporal store should be used for a, nontemporal prefetch for b. */
+ for (i = 0; i < K; i++)
+ a[i] = b[i];
+
+ /* Nontemporal store should not be used here (only write and read temporal
+ prefetches). */
+ for (i = 0; i < K - 10000; i++)
+ a[i + 10000] = a[i];
+
+ /* Nontemporal store should not be used here (only write and read nontemporal
+ prefetches). */
+ for (i = 0; i < K - 100000; i++)
+ a[i + 100000] = a[i];
+
+ /* Nontemporal store should be used neither for a nor for p, as we do not know
+ whether they alias or not. */
+ for (i = 0; i < K; i++)
+ {
+ a[i] = 0;
+ *p++ = 1;
+ }
+
+ /* Nontemporal store should not be used for a, as we do not know whether its
+ value will be reused or not. */
+ for (i = 0; i < 1000; i++)
+ a[i] = 0;
+ }
+
+ /* { dg-final { scan-tree-dump-times "Issued prefetch" 5 "aprefetch" } } */
+ /* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 3 "aprefetch" } } */
+ /* { dg-final { scan-tree-dump-times "nontemporal store" 2 "aprefetch" } } */
+
+ /* { dg-final { scan-tree-dump-times "builtin_prefetch" 8 "final_cleanup" } } */
+ /* { dg-final { scan-tree-dump-times "=\\{nt\\}" 2 "final_cleanup" } } */
+ /* { dg-final { scan-tree-dump-times "__builtin_ia32_mfence" 2 "final_cleanup" } } */
+
+ /* { dg-final { scan-assembler-times "prefetchw" 5 } } */
+ /* { dg-final { scan-assembler-times "prefetcht" 1 } } */
+ /* { dg-final { scan-assembler-times "prefetchnta" 2 } } */
+ /* { dg-final { scan-assembler-times "movnti" 2 } } */
+ /* { dg-final { scan-assembler-times "mfence" 2 } } */
+
+ /* { dg-final { cleanup-tree-dump "aprefetch" } } */
+ /* { dg-final { cleanup-tree-dump "final_cleanup" } } */
Index: expr.c
===================================================================
*** expr.c (revision 125532)
--- expr.c (working copy)
*************** static void store_constructor_field (rtx
*** 142,148 ****
tree, tree, int, int);
static void store_constructor (tree, rtx, int, HOST_WIDE_INT);
static rtx store_field (rtx, HOST_WIDE_INT, HOST_WIDE_INT, enum machine_mode,
! tree, tree, int);
static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (tree, tree);
--- 142,148 ----
tree, tree, int, int);
static void store_constructor (tree, rtx, int, HOST_WIDE_INT);
static rtx store_field (rtx, HOST_WIDE_INT, HOST_WIDE_INT, enum machine_mode,
! tree, tree, int, bool);
static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (tree, tree);
*************** optimize_bitfield_assignment_op (unsigne
*** 4074,4083 ****
}
! /* Expand an assignment that stores the value of FROM into TO. */
void
! expand_assignment (tree to, tree from)
{
rtx to_rtx = 0;
rtx result;
--- 4074,4084 ----
}
! /* Expand an assignment that stores the value of FROM into TO. If NONTEMPORAL
! is true, try generating a nontemporal store. */
void
! expand_assignment (tree to, tree from, bool nontemporal)
{
rtx to_rtx = 0;
rtx result;
*************** expand_assignment (tree to, tree from)
*** 4164,4175 ****
if (TREE_CODE (TREE_TYPE (from)) == COMPLEX_TYPE)
{
gcc_assert (bitpos == 0);
! result = store_expr (from, to_rtx, false);
}
else
{
gcc_assert (bitpos == 0 || bitpos == GET_MODE_BITSIZE (mode1));
! result = store_expr (from, XEXP (to_rtx, bitpos != 0), false);
}
}
else
--- 4165,4177 ----
if (TREE_CODE (TREE_TYPE (from)) == COMPLEX_TYPE)
{
gcc_assert (bitpos == 0);
! result = store_expr (from, to_rtx, false, nontemporal);
}
else
{
gcc_assert (bitpos == 0 || bitpos == GET_MODE_BITSIZE (mode1));
! result = store_expr (from, XEXP (to_rtx, bitpos != 0), false,
! nontemporal);
}
}
else
*************** expand_assignment (tree to, tree from)
*** 4195,4201 ****
result = NULL;
else
result = store_field (to_rtx, bitsize, bitpos, mode1, from,
! TREE_TYPE (tem), get_alias_set (to));
}
if (result)
--- 4197,4204 ----
result = NULL;
else
result = store_field (to_rtx, bitsize, bitpos, mode1, from,
! TREE_TYPE (tem), get_alias_set (to),
! nontemporal);
}
if (result)
*************** expand_assignment (tree to, tree from)
*** 4302,4314 ****
/* Compute FROM and store the value in the rtx we got. */
push_temp_slots ();
! result = store_expr (from, to_rtx, 0);
preserve_temp_slots (result);
free_temp_slots ();
pop_temp_slots ();
return;
}
/* Generate code for computing expression EXP,
and storing the value into TARGET.
--- 4305,4350 ----
/* Compute FROM and store the value in the rtx we got. */
push_temp_slots ();
! result = store_expr (from, to_rtx, 0, nontemporal);
preserve_temp_slots (result);
free_temp_slots ();
pop_temp_slots ();
return;
}
+ /* Emits nontemporal store insn that moves FROM to TO. Returns true if this
+ succeeded, false otherwise. */
+
+ static bool
+ emit_storent_insn (rtx to, rtx from)
+ {
+ enum machine_mode mode = GET_MODE (to), imode;
+ enum insn_code code = storent_optab->handlers[mode].insn_code;
+ rtx pattern;
+
+ if (code == CODE_FOR_nothing)
+ return false;
+
+ imode = insn_data[code].operand[0].mode;
+ if (!insn_data[code].operand[0].predicate (to, imode))
+ return false;
+
+ imode = insn_data[code].operand[1].mode;
+ if (!insn_data[code].operand[1].predicate (from, imode))
+ {
+ from = copy_to_mode_reg (imode, from);
+ if (!insn_data[code].operand[1].predicate (from, imode))
+ return false;
+ }
+
+ pattern = GEN_FCN (code) (to, from);
+ if (pattern == NULL_RTX)
+ return false;
+
+ emit_insn (pattern);
+ return true;
+ }
+
/* Generate code for computing expression EXP,
and storing the value into TARGET.
*************** expand_assignment (tree to, tree from)
*** 4320,4329 ****
be more thorough?
If CALL_PARAM_P is nonzero, this is a store into a call param on the
! stack, and block moves may need to be treated specially. */
rtx
! store_expr (tree exp, rtx target, int call_param_p)
{
rtx temp;
rtx alt_rtl = NULL_RTX;
--- 4356,4367 ----
be more thorough?
If CALL_PARAM_P is nonzero, this is a store into a call param on the
! stack, and block moves may need to be treated specially.
!
! If NONTEMPORAL is true, try using a nontemporal store instruction. */
rtx
! store_expr (tree exp, rtx target, int call_param_p, bool nontemporal)
{
rtx temp;
rtx alt_rtl = NULL_RTX;
*************** store_expr (tree exp, rtx target, int ca
*** 4344,4350 ****
part. */
expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
! return store_expr (TREE_OPERAND (exp, 1), target, call_param_p);
}
else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
{
--- 4382,4389 ----
part. */
expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
! return store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
! nontemporal);
}
else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
{
*************** store_expr (tree exp, rtx target, int ca
*** 4358,4368 ****
do_pending_stack_adjust ();
NO_DEFER_POP;
jumpifnot (TREE_OPERAND (exp, 0), lab1);
! store_expr (TREE_OPERAND (exp, 1), target, call_param_p);
emit_jump_insn (gen_jump (lab2));
emit_barrier ();
emit_label (lab1);
! store_expr (TREE_OPERAND (exp, 2), target, call_param_p);
emit_label (lab2);
OK_DEFER_POP;
--- 4397,4409 ----
do_pending_stack_adjust ();
NO_DEFER_POP;
jumpifnot (TREE_OPERAND (exp, 0), lab1);
! store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
! nontemporal);
emit_jump_insn (gen_jump (lab2));
emit_barrier ();
emit_label (lab1);
! store_expr (TREE_OPERAND (exp, 2), target, call_param_p,
! nontemporal);
emit_label (lab2);
OK_DEFER_POP;
*************** store_expr (tree exp, rtx target, int ca
*** 4433,4439 ****
}
else
{
! temp = expand_expr_real (exp, target, GET_MODE (target),
(call_param_p
? EXPAND_STACK_PARM : EXPAND_NORMAL),
&alt_rtl);
--- 4474,4485 ----
}
else
{
! rtx tmp_target;
!
! /* If we want to use a nontemporal store, force the value to
! register first. */
! tmp_target = nontemporal ? NULL_RTX : target;
! temp = expand_expr_real (exp, tmp_target, GET_MODE (target),
(call_param_p
? EXPAND_STACK_PARM : EXPAND_NORMAL),
&alt_rtl);
*************** store_expr (tree exp, rtx target, int ca
*** 4591,4596 ****
--- 4637,4647 ----
emit_block_move (target, temp, expr_size (exp),
(call_param_p
? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
+ else if (nontemporal
+ && emit_storent_insn (target, temp))
+ /* If we managed to emit a nontemporal store, there is nothing else to
+ do. */
+ ;
else
{
temp = force_operand (temp, target);
*************** store_constructor_field (rtx target, uns
*** 4941,4947 ****
store_constructor (exp, target, cleared, bitsize / BITS_PER_UNIT);
}
else
! store_field (target, bitsize, bitpos, mode, exp, type, alias_set);
}
/* Store the value of constructor EXP into the rtx TARGET.
--- 4992,4998 ----
store_constructor (exp, target, cleared, bitsize / BITS_PER_UNIT);
}
else
! store_field (target, bitsize, bitpos, mode, exp, type, alias_set, false);
}
/* Store the value of constructor EXP into the rtx TARGET.
*************** store_constructor (tree exp, rtx target,
*** 5291,5297 ****
= gen_reg_rtx (promote_mode (domain, DECL_MODE (index),
&unsignedp, 0));
SET_DECL_RTL (index, index_r);
! store_expr (lo_index, index_r, 0);
/* Build the head of the loop. */
do_pending_stack_adjust ();
--- 5342,5348 ----
= gen_reg_rtx (promote_mode (domain, DECL_MODE (index),
&unsignedp, 0));
SET_DECL_RTL (index, index_r);
! store_expr (lo_index, index_r, 0, false);
/* Build the head of the loop. */
do_pending_stack_adjust ();
*************** store_constructor (tree exp, rtx target,
*** 5318,5324 ****
store_constructor (value, xtarget, cleared,
bitsize / BITS_PER_UNIT);
else
! store_expr (value, xtarget, 0);
/* Generate a conditional jump to exit the loop. */
exit_cond = build2 (LT_EXPR, integer_type_node,
--- 5369,5375 ----
store_constructor (value, xtarget, cleared,
bitsize / BITS_PER_UNIT);
else
! store_expr (value, xtarget, 0, false);
/* Generate a conditional jump to exit the loop. */
exit_cond = build2 (LT_EXPR, integer_type_node,
*************** store_constructor (tree exp, rtx target,
*** 5329,5335 ****
the loop. */
expand_assignment (index,
build2 (PLUS_EXPR, TREE_TYPE (index),
! index, integer_one_node));
emit_jump (loop_start);
--- 5380,5387 ----
the loop. */
expand_assignment (index,
build2 (PLUS_EXPR, TREE_TYPE (index),
! index, integer_one_node),
! false);
emit_jump (loop_start);
*************** store_constructor (tree exp, rtx target,
*** 5360,5366 ****
expand_normal (position),
highest_pow2_factor (position));
xtarget = adjust_address (xtarget, mode, 0);
! store_expr (value, xtarget, 0);
}
else
{
--- 5412,5418 ----
expand_normal (position),
highest_pow2_factor (position));
xtarget = adjust_address (xtarget, mode, 0);
! store_expr (value, xtarget, 0, false);
}
else
{
*************** store_constructor (tree exp, rtx target,
*** 5522,5532 ****
ALIAS_SET is the alias set for the destination. This value will
(in general) be different from that for TARGET, since TARGET is a
! reference to the containing structure. */
static rtx
store_field (rtx target, HOST_WIDE_INT bitsize, HOST_WIDE_INT bitpos,
! enum machine_mode mode, tree exp, tree type, int alias_set)
{
HOST_WIDE_INT width_mask = 0;
--- 5574,5587 ----
ALIAS_SET is the alias set for the destination. This value will
(in general) be different from that for TARGET, since TARGET is a
! reference to the containing structure.
!
! If NONTEMPORAL is true, try generating a nontemporal store. */
static rtx
store_field (rtx target, HOST_WIDE_INT bitsize, HOST_WIDE_INT bitpos,
! enum machine_mode mode, tree exp, tree type, int alias_set,
! bool nontemporal)
{
HOST_WIDE_INT width_mask = 0;
*************** store_field (rtx target, HOST_WIDE_INT b
*** 5561,5567 ****
if (bitsize != (HOST_WIDE_INT) GET_MODE_BITSIZE (GET_MODE (target)))
emit_move_insn (object, target);
! store_field (blk_object, bitsize, bitpos, mode, exp, type, alias_set);
emit_move_insn (target, object);
--- 5616,5623 ----
if (bitsize != (HOST_WIDE_INT) GET_MODE_BITSIZE (GET_MODE (target)))
emit_move_insn (object, target);
! store_field (blk_object, bitsize, bitpos, mode, exp, type, alias_set,
! nontemporal);
emit_move_insn (target, object);
*************** store_field (rtx target, HOST_WIDE_INT b
*** 5574,5580 ****
/* We're storing into a struct containing a single __complex. */
gcc_assert (!bitpos);
! return store_expr (exp, target, 0);
}
/* If the structure is in a register or if the component
--- 5630,5636 ----
/* We're storing into a struct containing a single __complex. */
gcc_assert (!bitpos);
! return store_expr (exp, target, 0, nontemporal);
}
/* If the structure is in a register or if the component
*************** store_field (rtx target, HOST_WIDE_INT b
*** 5675,5681 ****
if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
set_mem_alias_set (to_rtx, alias_set);
! return store_expr (exp, to_rtx, 0);
}
}
--- 5731,5737 ----
if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
set_mem_alias_set (to_rtx, alias_set);
! return store_expr (exp, to_rtx, 0, nontemporal);
}
}
*************** expand_expr_real_1 (tree exp, rtx target
*** 7831,7837 ****
/* Store data into beginning of memory target. */
store_expr (TREE_OPERAND (exp, 0),
adjust_address (target, TYPE_MODE (valtype), 0),
! modifier == EXPAND_STACK_PARM);
else
{
--- 7887,7894 ----
/* Store data into beginning of memory target. */
store_expr (TREE_OPERAND (exp, 0),
adjust_address (target, TYPE_MODE (valtype), 0),
! modifier == EXPAND_STACK_PARM,
! false);
else
{
*************** expand_expr_real_1 (tree exp, rtx target
*** 7844,7850 ****
* BITS_PER_UNIT),
(HOST_WIDE_INT) GET_MODE_BITSIZE (mode)),
0, TYPE_MODE (valtype), TREE_OPERAND (exp, 0),
! type, 0);
}
/* Return the entire union. */
--- 7901,7907 ----
* BITS_PER_UNIT),
(HOST_WIDE_INT) GET_MODE_BITSIZE (mode)),
0, TYPE_MODE (valtype), TREE_OPERAND (exp, 0),
! type, 0, false);
}
/* Return the entire union. */
*************** expand_expr_real_1 (tree exp, rtx target
*** 8760,8772 ****
op1 = gen_label_rtx ();
jumpifnot (TREE_OPERAND (exp, 0), op0);
store_expr (TREE_OPERAND (exp, 1), temp,
! modifier == EXPAND_STACK_PARM);
emit_jump_insn (gen_jump (op1));
emit_barrier ();
emit_label (op0);
store_expr (TREE_OPERAND (exp, 2), temp,
! modifier == EXPAND_STACK_PARM);
emit_label (op1);
OK_DEFER_POP;
--- 8817,8831 ----
op1 = gen_label_rtx ();
jumpifnot (TREE_OPERAND (exp, 0), op0);
store_expr (TREE_OPERAND (exp, 1), temp,
! modifier == EXPAND_STACK_PARM,
! false);
emit_jump_insn (gen_jump (op1));
emit_barrier ();
emit_label (op0);
store_expr (TREE_OPERAND (exp, 2), temp,
! modifier == EXPAND_STACK_PARM,
! false);
emit_label (op1);
OK_DEFER_POP;
*************** expand_expr_real_1 (tree exp, rtx target
*** 8781,8787 ****
tree lhs = TREE_OPERAND (exp, 0);
tree rhs = TREE_OPERAND (exp, 1);
gcc_assert (ignore);
! expand_assignment (lhs, rhs);
return const0_rtx;
}
--- 8840,8846 ----
tree lhs = TREE_OPERAND (exp, 0);
tree rhs = TREE_OPERAND (exp, 1);
gcc_assert (ignore);
! expand_assignment (lhs, rhs, false);
return const0_rtx;
}
*************** expand_expr_real_1 (tree exp, rtx target
*** 8813,8825 ****
do_jump (TREE_OPERAND (rhs, 1),
value ? label : 0,
value ? 0 : label);
! expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value));
do_pending_stack_adjust ();
emit_label (label);
return const0_rtx;
}
! expand_assignment (lhs, rhs);
return const0_rtx;
}
--- 8872,8885 ----
do_jump (TREE_OPERAND (rhs, 1),
value ? label : 0,
value ? 0 : label);
! expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
! MOVE_NONTEMPORAL (exp));
do_pending_stack_adjust ();
emit_label (label);
return const0_rtx;
}
! expand_assignment (lhs, rhs, MOVE_NONTEMPORAL (exp));
return const0_rtx;
}
Index: expr.h
===================================================================
*** expr.h (revision 125532)
--- expr.h (working copy)
*************** extern void emit_push_insn (rtx, enum ma
*** 477,489 ****
int, rtx, int, rtx, rtx, int, rtx);
/* Expand an assignment that stores the value of FROM into TO. */
! extern void expand_assignment (tree, tree);
/* Generate code for computing expression EXP,
and storing the value into TARGET.
If SUGGEST_REG is nonzero, copy the value through a register
and return that register, if that is possible. */
! extern rtx store_expr (tree, rtx, int);
/* Given an rtx that may include add and multiply operations,
generate them as insns and return a pseudo-reg containing the value.
--- 477,489 ----
int, rtx, int, rtx, rtx, int, rtx);
/* Expand an assignment that stores the value of FROM into TO. */
! extern void expand_assignment (tree, tree, bool);
/* Generate code for computing expression EXP,
and storing the value into TARGET.
If SUGGEST_REG is nonzero, copy the value through a register
and return that register, if that is possible. */
! extern rtx store_expr (tree, rtx, int, bool);
/* Given an rtx that may include add and multiply operations,
generate them as insns and return a pseudo-reg containing the value.
Index: function.c
===================================================================
*** function.c (revision 125532)
--- function.c (working copy)
*************** assign_parm_setup_reg (struct assign_par
*** 2723,2729 ****
/* TREE_USED gets set erroneously during expand_assignment. */
save_tree_used = TREE_USED (parm);
! expand_assignment (parm, make_tree (data->nominal_type, tempreg));
TREE_USED (parm) = save_tree_used;
all->first_conversion_insn = get_insns ();
all->last_conversion_insn = get_last_insn ();
--- 2723,2729 ----
/* TREE_USED gets set erroneously during expand_assignment. */
save_tree_used = TREE_USED (parm);
! expand_assignment (parm, make_tree (data->nominal_type, tempreg), false);
TREE_USED (parm) = save_tree_used;
all->first_conversion_insn = get_insns ();
all->last_conversion_insn = get_last_insn ();
Index: tree-data-ref.c
===================================================================
*** tree-data-ref.c (revision 125532)
--- tree-data-ref.c (working copy)
*************** dr_analyze_alias (struct data_reference
*** 729,735 ****
}
DR_SYMBOL_TAG (dr) = smt;
! if (var_can_have_subvars (smt))
DR_SUBVARS (dr) = get_subvars_for_var (smt);
vops = BITMAP_ALLOC (NULL);
--- 729,735 ----
}
DR_SYMBOL_TAG (dr) = smt;
! if (smt && var_can_have_subvars (smt))
DR_SUBVARS (dr) = get_subvars_for_var (smt);
vops = BITMAP_ALLOC (NULL);
Index: calls.c
===================================================================
*** calls.c (revision 125532)
--- calls.c (working copy)
*************** initialize_argument_information (int num
*** 1080,1086 ****
else
copy = assign_temp (type, 0, 1, 0);
! store_expr (args[i].tree_value, copy, 0);
if (callee_copies)
*ecf_flags &= ~(ECF_CONST | ECF_LIBCALL_BLOCK);
--- 1080,1086 ----
else
copy = assign_temp (type, 0, 1, 0);
! store_expr (args[i].tree_value, copy, 0, false);
if (callee_copies)
*ecf_flags &= ~(ECF_CONST | ECF_LIBCALL_BLOCK);
Index: tree-predcom.c
===================================================================
*** tree-predcom.c (revision 125532)
--- tree-predcom.c (working copy)
*************** get_init_expr (chain_p chain, unsigned i
*** 1378,1384 ****
/* Marks all virtual operands of statement STMT for renaming. */
! static void
mark_virtual_ops_for_renaming (tree stmt)
{
ssa_op_iter iter;
--- 1378,1384 ----
/* Marks all virtual operands of statement STMT for renaming. */
! void
mark_virtual_ops_for_renaming (tree stmt)
{
ssa_op_iter iter;
Index: tree-ssa-loop-prefetch.c
===================================================================
*** tree-ssa-loop-prefetch.c (revision 125532)
--- tree-ssa-loop-prefetch.c (working copy)
*************** Software Foundation, 59 Temple Place - S
*** 47,52 ****
--- 47,53 ----
#include "langhooks.h"
#include "tree-inline.h"
#include "tree-data-ref.h"
+ #include "optabs.h"
/* This pass inserts prefetch instructions to optimize cache usage during
accesses to arrays in loops. It processes loops sequentially and:
*************** Software Foundation, 59 Temple Place - S
*** 177,182 ****
--- 178,190 ----
is accessed several times in a single iteration of the loop. */
#define NONTEMPORAL_FRACTION 16
+ /* In case we have to emit a memory fence instruction after the loop that
+ uses nontemporal stores, this defines the builtin to use. */
+
+ #ifndef FENCE_FOLLOWING_MOVNT
+ #define FENCE_FOLLOWING_MOVNT NULL_TREE
+ #endif
+
/* The group of references between that reuse may occur. */
struct mem_ref_group
*************** struct mem_ref
*** 198,204 ****
tree stmt; /* Statement in that the reference appears. */
tree mem; /* The reference. */
HOST_WIDE_INT delta; /* Constant offset of the reference. */
- bool write_p; /* Is it a write? */
struct mem_ref_group *group; /* The group of references it belongs to. */
unsigned HOST_WIDE_INT prefetch_mod;
/* Prefetch only each PREFETCH_MOD-th
--- 206,211 ----
*************** struct mem_ref
*** 208,215 ****
iterations. */
unsigned reuse_distance; /* The amount of data accessed before the first
reuse of this value. */
- bool issue_prefetch_p; /* Should we really issue the prefetch? */
struct mem_ref *next; /* The next reference in the group. */
};
/* Dumps information about reference REF to FILE. */
--- 215,227 ----
iterations. */
unsigned reuse_distance; /* The amount of data accessed before the first
reuse of this value. */
struct mem_ref *next; /* The next reference in the group. */
+ unsigned write_p : 1; /* Is it a write? */
+ unsigned independent_p : 1; /* True if the reference is independent on
+ all other references inside the loop. */
+ unsigned issue_prefetch_p : 1; /* Should we really issue the prefetch? */
+ unsigned storent_p : 1; /* True if we changed the store to a
+ nontemporal one. */
};
/* Dumps information about reference REF to FILE. */
*************** record_ref (struct mem_ref_group *group,
*** 302,307 ****
--- 314,321 ----
(*aref)->issue_prefetch_p = false;
(*aref)->group = group;
(*aref)->next = NULL;
+ (*aref)->independent_p = false;
+ (*aref)->storent_p = false;
if (dump_file && (dump_flags & TDF_DETAILS))
dump_mem_ref (dump_file, *aref);
*************** analyze_ref (struct loop *loop, tree *re
*** 434,442 ****
}
/* Record a memory reference REF to the list REFS. The reference occurs in
! LOOP in statement STMT and it is write if WRITE_P. */
! static void
gather_memory_references_ref (struct loop *loop, struct mem_ref_group **refs,
tree ref, bool write_p, tree stmt)
{
--- 448,457 ----
}
/* Record a memory reference REF to the list REFS. The reference occurs in
! LOOP in statement STMT and it is write if WRITE_P. Returns true if the
! reference was recorded, false otherwise. */
! static bool
gather_memory_references_ref (struct loop *loop, struct mem_ref_group **refs,
tree ref, bool write_p, tree stmt)
{
*************** gather_memory_references_ref (struct loo
*** 445,470 ****
struct mem_ref_group *agrp;
if (!analyze_ref (loop, &ref, &base, &step, &delta, stmt))
! return;
/* Now we know that REF = &BASE + STEP * iter + DELTA, where DELTA and STEP
are integer constants. */
agrp = find_or_create_group (refs, base, step);
record_ref (agrp, stmt, ref, delta, write_p);
}
! /* Record the suitable memory references in LOOP. */
static struct mem_ref_group *
! gather_memory_references (struct loop *loop)
{
basic_block *body = get_loop_body_in_dom_order (loop);
basic_block bb;
unsigned i;
block_stmt_iterator bsi;
! tree stmt, lhs, rhs;
struct mem_ref_group *refs = NULL;
/* Scan the loop body in order, so that the former references precede the
later ones. */
for (i = 0; i < loop->num_nodes; i++)
--- 460,490 ----
struct mem_ref_group *agrp;
if (!analyze_ref (loop, &ref, &base, &step, &delta, stmt))
! return false;
/* Now we know that REF = &BASE + STEP * iter + DELTA, where DELTA and STEP
are integer constants. */
agrp = find_or_create_group (refs, base, step);
record_ref (agrp, stmt, ref, delta, write_p);
+
+ return true;
}
! /* Record the suitable memory references in LOOP. NO_OTHER_REFS is set to
! true if there are no other memory references inside the loop. */
static struct mem_ref_group *
! gather_memory_references (struct loop *loop, bool *no_other_refs)
{
basic_block *body = get_loop_body_in_dom_order (loop);
basic_block bb;
unsigned i;
block_stmt_iterator bsi;
! tree stmt, lhs, rhs, call;
struct mem_ref_group *refs = NULL;
+ *no_other_refs = true;
+
/* Scan the loop body in order, so that the former references precede the
later ones. */
for (i = 0; i < loop->num_nodes; i++)
*************** gather_memory_references (struct loop *l
*** 476,491 ****
for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
{
stmt = bsi_stmt (bsi);
if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
! continue;
lhs = GIMPLE_STMT_OPERAND (stmt, 0);
rhs = GIMPLE_STMT_OPERAND (stmt, 1);
if (REFERENCE_CLASS_P (rhs))
! gather_memory_references_ref (loop, &refs, rhs, false, stmt);
if (REFERENCE_CLASS_P (lhs))
! gather_memory_references_ref (loop, &refs, lhs, true, stmt);
}
}
free (body);
--- 496,521 ----
for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
{
stmt = bsi_stmt (bsi);
+ call = get_call_expr_in (stmt);
+ if (call && !(call_expr_flags (call) & ECF_CONST))
+ *no_other_refs = false;
+
if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
! {
! if (!ZERO_SSA_OPERANDS (stmt, SSA_OP_ALL_VIRTUALS))
! *no_other_refs = false;
! continue;
! }
lhs = GIMPLE_STMT_OPERAND (stmt, 0);
rhs = GIMPLE_STMT_OPERAND (stmt, 1);
if (REFERENCE_CLASS_P (rhs))
! *no_other_refs &= gather_memory_references_ref (loop, &refs,
! rhs, false, stmt);
if (REFERENCE_CLASS_P (lhs))
! *no_other_refs &= gather_memory_references_ref (loop, &refs,
! lhs, true, stmt);
}
}
free (body);
*************** should_issue_prefetch_p (struct mem_ref
*** 746,751 ****
--- 776,785 ----
if (ref->prefetch_before != PREFETCH_ALL)
return false;
+ /* Do not prefetch nontemporal stores. */
+ if (ref->storent_p)
+ return false;
+
return true;
}
*************** issue_prefetches (struct mem_ref_group *
*** 884,889 ****
--- 918,1047 ----
issue_prefetch_ref (ref, unroll_factor, ahead);
}
+ /* Returns true if REF is a memory write for that a nontemporal store insn
+ can be used. */
+
+ static bool
+ nontemporal_store_p (struct mem_ref *ref)
+ {
+ enum machine_mode mode;
+ enum insn_code code;
+
+ /* REF must be a write that is not reused. We require it to be independent
+ on all other memory references in the loop, as the nontemporal stores may
+ be reordered with respect to other memory references. */
+ if (!ref->write_p
+ || !ref->independent_p
+ || ref->reuse_distance < L2_CACHE_SIZE_BYTES)
+ return false;
+
+ /* Check that we have the storent instruction for the mode. */
+ mode = TYPE_MODE (TREE_TYPE (ref->mem));
+ if (mode == BLKmode)
+ return false;
+
+ code = storent_optab->handlers[mode].insn_code;
+ return code != CODE_FOR_nothing;
+ }
+
+ /* If REF is a nontemporal store, we mark the corresponding modify statement
+ and return true. Otherwise, we return false. */
+
+ static bool
+ mark_nontemporal_store (struct mem_ref *ref)
+ {
+ if (!nontemporal_store_p (ref))
+ return false;
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, "Marked reference %p as a nontemporal store.\n",
+ (void *) ref);
+
+ MOVE_NONTEMPORAL (ref->stmt) = true;
+ ref->storent_p = true;
+
+ return true;
+ }
+
+ /* Issue a memory fence instruction after LOOP. */
+
+ static void
+ emit_mfence_after_loop (struct loop *loop)
+ {
+ VEC (edge, heap) *exits = get_loop_exit_edges (loop);
+ edge exit;
+ tree call;
+ block_stmt_iterator bsi;
+ unsigned i;
+
+ for (i = 0; VEC_iterate (edge, exits, i, exit); i++)
+ {
+ call = build_function_call_expr (FENCE_FOLLOWING_MOVNT, NULL_TREE);
+
+ if (!single_pred_p (exit->dest)
+ /* If possible, we prefer not to insert the fence on other paths
+ in cfg. */
+ && !(exit->flags & EDGE_ABNORMAL))
+ split_loop_exit_edge (exit);
+ bsi = bsi_after_labels (exit->dest);
+
+ bsi_insert_before (&bsi, call, BSI_NEW_STMT);
+ mark_virtual_ops_for_renaming (call);
+ }
+
+ VEC_free (edge, heap, exits);
+ update_ssa (TODO_update_ssa_only_virtuals);
+ }
+
+ /* Returns true if we can use storent in loop, false otherwise. */
+
+ static bool
+ may_use_storent_in_loop_p (struct loop *loop)
+ {
+ bool ret = true;
+
+ if (loop->inner != NULL)
+ return false;
+
+ /* If we must issue a mfence insn after using storent, check that there
+ is a suitable place for it at each of the loop exits. */
+ if (FENCE_FOLLOWING_MOVNT != NULL_TREE)
+ {
+ VEC (edge, heap) *exits = get_loop_exit_edges (loop);
+ unsigned i;
+ edge exit;
+
+ for (i = 0; VEC_iterate (edge, exits, i, exit); i++)
+ if ((exit->flags & EDGE_ABNORMAL)
+ && exit->dest == EXIT_BLOCK_PTR)
+ ret = false;
+
+ VEC_free (edge, heap, exits);
+ }
+
+ return ret;
+ }
+
+ /* Marks nontemporal stores in LOOP. GROUPS contains the description of memory
+ references in the loop. */
+
+ static void
+ mark_nontemporal_stores (struct loop *loop, struct mem_ref_group *groups)
+ {
+ struct mem_ref *ref;
+ bool any = false;
+
+ if (!may_use_storent_in_loop_p (loop))
+ return;
+
+ for (; groups; groups = groups->next)
+ for (ref = groups->refs; ref; ref = ref->next)
+ any |= mark_nontemporal_store (ref);
+
+ if (any && FENCE_FOLLOWING_MOVNT != NULL_TREE)
+ emit_mfence_after_loop (loop);
+ }
+
/* Determines whether we can profitably unroll LOOP FACTOR times, and if
this is the case, fill in DESC by the description of number of
iterations. */
*************** self_reuse_distance (data_reference_p dr
*** 1115,1130 ****
}
/* Determines the distance till the first reuse of each reference in REFS
! in the loop nest of LOOP. */
static void
! determine_loop_nest_reuse (struct loop *loop, struct mem_ref_group *refs)
{
struct loop *nest, *aloop;
VEC (data_reference_p, heap) *datarefs = NULL;
VEC (ddr_p, heap) *dependences = NULL;
struct mem_ref_group *gr;
! struct mem_ref *ref;
VEC (loop_p, heap) *vloops = NULL;
unsigned *loop_data_size;
unsigned i, j, n;
--- 1273,1290 ----
}
/* Determines the distance till the first reuse of each reference in REFS
! in the loop nest of LOOP. NO_OTHER_REFS is true if there are no other
! memory references in the loop. */
static void
! determine_loop_nest_reuse (struct loop *loop, struct mem_ref_group *refs,
! bool no_other_refs)
{
struct loop *nest, *aloop;
VEC (data_reference_p, heap) *datarefs = NULL;
VEC (ddr_p, heap) *dependences = NULL;
struct mem_ref_group *gr;
! struct mem_ref *ref, *refb;
VEC (loop_p, heap) *vloops = NULL;
unsigned *loop_data_size;
unsigned i, j, n;
*************** determine_loop_nest_reuse (struct loop *
*** 1188,1193 ****
--- 1348,1355 ----
dr->aux = ref;
VEC_safe_push (data_reference_p, heap, datarefs, dr);
}
+ else
+ no_other_refs = false;
}
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
*************** determine_loop_nest_reuse (struct loop *
*** 1196,1201 ****
--- 1358,1366 ----
ref = dr->aux;
if (ref->reuse_distance > dist)
ref->reuse_distance = dist;
+
+ if (no_other_refs)
+ ref->independent_p = true;
}
compute_all_dependences (datarefs, &dependences, vloops, true);
*************** determine_loop_nest_reuse (struct loop *
*** 1205,1216 ****
--- 1370,1387 ----
if (DDR_ARE_DEPENDENT (dep) == chrec_known)
continue;
+ ref = DDR_A (dep)->aux;
+ refb = DDR_B (dep)->aux;
+
if (DDR_ARE_DEPENDENT (dep) == chrec_dont_know
|| DDR_NUM_DIST_VECTS (dep) == 0)
{
/* If the dependence cannot be analysed, assume that there might be
a reuse. */
dist = 0;
+
+ ref->independent_p = false;
+ refb->independent_p = false;
}
else
{
*************** determine_loop_nest_reuse (struct loop *
*** 1228,1233 ****
--- 1399,1416 ----
adist = volume_of_dist_vector (DDR_DIST_VECT (dep, j),
loop_data_size, n);
+ /* If this is a dependence in the innermost loop (i.e., the
+ distances in all superloops are zero) and it is not
+ the trivial self-dependence with distance zero, record that
+ the references are not completely independent. */
+ if (lambda_vector_zerop (DDR_DIST_VECT (dep, j), n - 1)
+ && (ref != refb
+ || DDR_DIST_VECT (dep, j)[n-1] != 0))
+ {
+ ref->independent_p = false;
+ refb->independent_p = false;
+ }
+
/* Ignore accesses closer than
L1_CACHE_SIZE_BYTES / NONTEMPORAL_FRACTION,
so that we use nontemporal prefetches e.g. if single memory
*************** determine_loop_nest_reuse (struct loop *
*** 1241,1252 ****
}
}
- ref = DDR_A (dep)->aux;
- if (ref->reuse_distance > dist)
- ref->reuse_distance = dist;
- ref = DDR_B (dep)->aux;
if (ref->reuse_distance > dist)
ref->reuse_distance = dist;
}
free_dependence_relations (dependences);
--- 1424,1433 ----
}
}
if (ref->reuse_distance > dist)
ref->reuse_distance = dist;
+ if (refb->reuse_distance > dist)
+ refb->reuse_distance = dist;
}
free_dependence_relations (dependences);
*************** loop_prefetch_arrays (struct loop *loop)
*** 1273,1279 ****
unsigned ahead, ninsns, time, unroll_factor;
HOST_WIDE_INT est_niter;
struct tree_niter_desc desc;
! bool unrolled = false;
if (!maybe_hot_bb_p (loop->header))
{
--- 1454,1460 ----
unsigned ahead, ninsns, time, unroll_factor;
HOST_WIDE_INT est_niter;
struct tree_niter_desc desc;
! bool unrolled = false, no_other_refs;
if (!maybe_hot_bb_p (loop->header))
{
*************** loop_prefetch_arrays (struct loop *loop)
*** 1283,1289 ****
}
/* Step 1: gather the memory references. */
! refs = gather_memory_references (loop);
/* Step 2: estimate the reuse effects. */
prune_by_reuse (refs);
--- 1464,1470 ----
}
/* Step 1: gather the memory references. */
! refs = gather_memory_references (loop, &no_other_refs);
/* Step 2: estimate the reuse effects. */
prune_by_reuse (refs);
*************** loop_prefetch_arrays (struct loop *loop)
*** 1291,1297 ****
if (!anything_to_prefetch_p (refs))
goto fail;
! determine_loop_nest_reuse (loop, refs);
/* Step 3: determine the ahead and unroll factor. */
--- 1472,1478 ----
if (!anything_to_prefetch_p (refs))
goto fail;
! determine_loop_nest_reuse (loop, refs, no_other_refs);
/* Step 3: determine the ahead and unroll factor. */
*************** loop_prefetch_arrays (struct loop *loop)
*** 1313,1318 ****
--- 1494,1501 ----
goto fail;
}
+ mark_nontemporal_stores (loop, refs);
+
ninsns = tree_num_loop_insns (loop, &eni_size_weights);
unroll_factor = determine_unroll_factor (loop, refs, ninsns, &desc,
est_niter);
Index: tree-flow.h
===================================================================
*** tree-flow.h (revision 125532)
--- tree-flow.h (working copy)
*************** void tree_transform_and_unroll_loop (str
*** 1019,1024 ****
--- 1019,1025 ----
transform_callback, void *);
bool contains_abnormal_ssa_name_p (tree);
bool stmt_dominates_stmt_p (tree, tree);
+ void mark_virtual_ops_for_renaming (tree);
/* In tree-ssa-threadedge.c */
extern bool potentially_threadable_block (basic_block);
Index: Makefile.in
===================================================================
*** Makefile.in (revision 125532)
--- Makefile.in (working copy)
*************** tree-ssa-loop-prefetch.o: tree-ssa-loop-
*** 2085,2091 ****
output.h $(DIAGNOSTIC_H) $(TIMEVAR_H) $(TM_H) coretypes.h $(TREE_DUMP_H) \
tree-pass.h $(GGC_H) $(RECOG_H) insn-config.h $(HASHTAB_H) $(SCEV_H) \
$(CFGLOOP_H) $(PARAMS_H) langhooks.h $(BASIC_BLOCK_H) hard-reg-set.h \
! tree-chrec.h toplev.h langhooks.h $(TREE_INLINE_H) $(TREE_DATA_REF_H)
tree-predcom.o: tree-predcom.c $(CONFIG_H) $(SYSTEM_H) $(TREE_H) $(TM_P_H) \
$(CFGLOOP_H) $(TREE_FLOW_H) $(GGC_H) $(TREE_DATA_REF_H) $(SCEV_H) \
$(PARAMS_H) $(DIAGNOSTIC_H) tree-pass.h $(TM_H) coretypes.h tree-affine.h \
--- 2085,2092 ----
output.h $(DIAGNOSTIC_H) $(TIMEVAR_H) $(TM_H) coretypes.h $(TREE_DUMP_H) \
tree-pass.h $(GGC_H) $(RECOG_H) insn-config.h $(HASHTAB_H) $(SCEV_H) \
$(CFGLOOP_H) $(PARAMS_H) langhooks.h $(BASIC_BLOCK_H) hard-reg-set.h \
! tree-chrec.h toplev.h langhooks.h $(TREE_INLINE_H) $(TREE_DATA_REF_H) \
! $(OPTABS_H)
tree-predcom.o: tree-predcom.c $(CONFIG_H) $(SYSTEM_H) $(TREE_H) $(TM_P_H) \
$(CFGLOOP_H) $(TREE_FLOW_H) $(GGC_H) $(TREE_DATA_REF_H) $(SCEV_H) \
$(PARAMS_H) $(DIAGNOSTIC_H) tree-pass.h $(TM_H) coretypes.h tree-affine.h \
Index: config/i386/i386.h
===================================================================
*** config/i386/i386.h (revision 125532)
--- config/i386/i386.h (working copy)
*************** extern int ix86_isa_flags;
*** 386,391 ****
--- 386,396 ----
#define TARGET_TLS_DIRECT_SEG_REFS_DEFAULT 0
#endif
+ /* Fence to use after loop using storent. */
+
+ extern tree x86_mfence;
+ #define FENCE_FOLLOWING_MOVNT x86_mfence
+
/* Once GDB has been enhanced to deal with functions without frame
pointers, we can change this to allow for elimination of
the frame pointer in leaf functions. */
Index: config/i386/sse.md
===================================================================
*** config/i386/sse.md (revision 125532)
--- config/i386/sse.md (working copy)
***************
*** 317,322 ****
--- 317,354 ----
(set_attr "prefix_rep" "1")
(set_attr "mode" "TI")])
+ ; Expand patterns for non-temporal stores. At the moment, only those
+ ; that directly map to insns are defined; it would be possible to
+ ; define patterns for other modes that would expand to several insns.
+
+ (define_expand "storentv4sf"
+ [(set (match_operand:V4SF 0 "memory_operand" "=m")
+ (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "x")]
+ UNSPEC_MOVNT))]
+ "TARGET_SSE"
+ "")
+
+ (define_expand "storentv2df"
+ [(set (match_operand:V2DF 0 "memory_operand" "=m")
+ (unspec:V2DF [(match_operand:V2DF 1 "register_operand" "x")]
+ UNSPEC_MOVNT))]
+ "TARGET_SSE2"
+ "")
+
+ (define_expand "storentv2di"
+ [(set (match_operand:V2DI 0 "memory_operand" "=m")
+ (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "x")]
+ UNSPEC_MOVNT))]
+ "TARGET_SSE2"
+ "")
+
+ (define_expand "storentsi"
+ [(set (match_operand:SI 0 "memory_operand" "=m")
+ (unspec:SI [(match_operand:SI 1 "register_operand" "r")]
+ UNSPEC_MOVNT))]
+ "TARGET_SSE2"
+ "")
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Parallel single-precision floating point arithmetic
Index: config/i386/i386.c
===================================================================
*** config/i386/i386.c (revision 125532)
--- config/i386/i386.c (working copy)
*************** int ix86_section_threshold = 65536;
*** 1500,1505 ****
--- 1500,1508 ----
char internal_label_prefix[16];
int internal_label_prefix_len;
+ /* Fence to use after loop using movnt. */
+ tree x86_mfence;
+
/* Register class used for passing given 64bit part of the argument.
These represent classes as documented by the PS ABI, with the exception
of SSESF, SSEDF classes, that are basically SSE class, just gcc will
*************** ix86_init_mmx_sse_builtins (void)
*** 18073,18079 ****
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush", void_ftype_pcvoid, IX86_BUILTIN_CLFLUSH);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_lfence", void_ftype_void, IX86_BUILTIN_LFENCE);
! def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence", void_ftype_void, IX86_BUILTIN_MFENCE);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_loaddqu", v16qi_ftype_pcchar, IX86_BUILTIN_LOADDQU);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_storedqu", void_ftype_pchar_v16qi, IX86_BUILTIN_STOREDQU);
--- 18076,18082 ----
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush", void_ftype_pcvoid, IX86_BUILTIN_CLFLUSH);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_lfence", void_ftype_void, IX86_BUILTIN_LFENCE);
! x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence", void_ftype_void, IX86_BUILTIN_MFENCE);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_loaddqu", v16qi_ftype_pcchar, IX86_BUILTIN_LOADDQU);
def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_storedqu", void_ftype_pchar_v16qi, IX86_BUILTIN_STOREDQU);
Index: stmt.c
===================================================================
*** stmt.c (revision 125532)
--- stmt.c (working copy)
*************** expand_asm_expr (tree exp)
*** 1114,1120 ****
{
if (o[i] != TREE_VALUE (tail))
{
! expand_assignment (o[i], TREE_VALUE (tail));
free_temp_slots ();
/* Restore the original value so that it's correct the next
--- 1114,1120 ----
{
if (o[i] != TREE_VALUE (tail))
{
! expand_assignment (o[i], TREE_VALUE (tail), false);
free_temp_slots ();
/* Restore the original value so that it's correct the next
More information about the Gcc-patches
mailing list