This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
[PATCH] Pass down branch probability through dojump.c functions (PR middle-end/42233)
- From: Jakub Jelinek <jakub at redhat dot com>
- To: gcc-patches at gcc dot gnu dot org
- Date: Thu, 18 Feb 2010 21:20:04 +0100
- Subject: [PATCH] Pass down branch probability through dojump.c functions (PR middle-end/42233)
- Reply-to: Jakub Jelinek <jakub at redhat dot com>
Hi!
While the gimple_boolify patch I've just sent fixes the regression
on the case where __builtin_expect's second argument is 0, when it is 1
there is another problem - add_reg_br_prob_note is a hack which can only
handle the most simple jumps, in particular it doesn't handle the case when
there is more than one jump and on the testcase from this PR we have two
jumps.
The following patch fixes it by passing down probability from
expand_gimple_cond (or other callers that know probabilities) down to
the myriad of dojump.c functions down to do_compare_rtx_and_jump
and inserts REG_BR_PROB note there. The passed PROB argument is
the probability of jump to the if_true_label in functions that
have 2 label arguments, or -1 if REG_BR_PROB note should not be inserted
(when probability is unknown).
Bootstrapped/regtested on x86_64-linux and i686-linux. Ok for trunk?
2010-02-18 Jakub Jelinek <jakub@redhat.com>
PR middle-end/42233
* expr.h (jumpifnot, jumpifnot_1, jumpif, jumpif_1, do_jump,
do_jump_1, do_compare_rtx_and_jump): Add PROB argument.
* dojump.c: Include output.h.
(inv): New inline function.
(jumpifnot, jumpifnot_1, jumpif, jumpif_1, do_jump_1, do_jump,
do_jump_by_parts_greater_rtx, do_jump_by_parts_greater,
do_jump_by_parts_zero_rtx, do_jump_by_parts_equality_rtx,
do_jump_by_parts_equality, do_compare_and_jump): Add PROB
argument, pass it down to other calls.
(do_compare_rtx_and_jump): Likewise. If PROB is not -1,
add REG_BR_PROB note to the conditional jump.
* cfgexpand.c (add_reg_br_prob_note): Removed.
(expand_gimple_cond): Don't call it, add the probability
as last argument to jumpif_1/jumpifnot_1.
* Makefile.in (dojump.o): Depend on output.h.
* builtins.c (expand_errno_check): Adjust do_compare_rtx_and_jump
callers.
* expmed.c (emit_store_flag_force, do_cmp_and_jump): Likewise.
* stmt.c (do_jump_if_equal): Likewise.
* cfgrtl.c (rtl_lv_add_condition_to_bb): Likewise.
* loop-unswitch.c (compare_and_jump_seq): Likewise.
* config/rs6000/rs6000.c (rs6000_aix_emit_builtin_unwind_init):
Likewise.
* optabs.c (expand_doubleword_shift, expand_abs): Likewise.
* expr.c (expand_expr_real_1): Adjust do_jump, jumpifnot and
jumpifnot_1 callers.
(expand_expr_real_2): Adjust jumpifnot_1 and do_compare_rtx_and_jump
callers.
(store_expr): Adjust jumpifnot caller.
(store_constructor): Adjust jumpif caller.
--- gcc/expr.h.jj 2009-12-14 14:00:58.000000000 +0100
+++ gcc/expr.h 2010-02-18 14:54:54.000000000 +0100
@@ -583,20 +583,20 @@ extern void do_pending_stack_adjust (voi
extern tree string_constant (tree, tree *);
/* Generate code to evaluate EXP and jump to LABEL if the value is zero. */
-extern void jumpifnot (tree, rtx);
-extern void jumpifnot_1 (enum tree_code, tree, tree, rtx);
+extern void jumpifnot (tree, rtx, int);
+extern void jumpifnot_1 (enum tree_code, tree, tree, rtx, int);
/* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */
-extern void jumpif (tree, rtx);
-extern void jumpif_1 (enum tree_code, tree, tree, rtx);
+extern void jumpif (tree, rtx, int);
+extern void jumpif_1 (enum tree_code, tree, tree, rtx, int);
/* Generate code to evaluate EXP and jump to IF_FALSE_LABEL if
the result is zero, or IF_TRUE_LABEL if the result is one. */
-extern void do_jump (tree, rtx, rtx);
-extern void do_jump_1 (enum tree_code, tree, tree, rtx, rtx);
+extern void do_jump (tree, rtx, rtx, int);
+extern void do_jump_1 (enum tree_code, tree, tree, rtx, rtx, int);
extern void do_compare_rtx_and_jump (rtx, rtx, enum rtx_code, int,
- enum machine_mode, rtx, rtx, rtx);
+ enum machine_mode, rtx, rtx, rtx, int);
/* Two different ways of generating switch statements. */
extern int try_casesi (tree, tree, tree, tree, rtx, rtx, rtx);
--- gcc/dojump.c.jj 2009-12-14 14:00:58.000000000 +0100
+++ gcc/dojump.c 2010-02-18 15:30:47.000000000 +0100
@@ -35,12 +35,21 @@ along with GCC; see the file COPYING3.
#include "langhooks.h"
#include "ggc.h"
#include "basic-block.h"
+#include "output.h"
static bool prefer_and_bit_test (enum machine_mode, int);
-static void do_jump_by_parts_greater (tree, tree, int, rtx, rtx);
-static void do_jump_by_parts_equality (tree, tree, rtx, rtx);
+static void do_jump_by_parts_greater (tree, tree, int, rtx, rtx, int);
+static void do_jump_by_parts_equality (tree, tree, rtx, rtx, int);
static void do_compare_and_jump (tree, tree, enum rtx_code, enum rtx_code, rtx,
- rtx);
+ rtx, int);
+
+/* Invert probability if there is any. -1 stands for unknown. */
+
+static inline int
+inv (int prob)
+{
+ return prob == -1 ? -1 : REG_BR_PROB_BASE - prob;
+}
/* At the start of a function, record that we have no previously-pushed
arguments waiting to be popped. */
@@ -96,29 +105,29 @@ do_pending_stack_adjust (void)
functions here. */
void
-jumpifnot (tree exp, rtx label)
+jumpifnot (tree exp, rtx label, int prob)
{
- do_jump (exp, label, NULL_RTX);
+ do_jump (exp, label, NULL_RTX, inv (prob));
}
void
-jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx label)
+jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx label, int prob)
{
- do_jump_1 (code, op0, op1, label, NULL_RTX);
+ do_jump_1 (code, op0, op1, label, NULL_RTX, inv (prob));
}
/* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */
void
-jumpif (tree exp, rtx label)
+jumpif (tree exp, rtx label, int prob)
{
- do_jump (exp, NULL_RTX, label);
+ do_jump (exp, NULL_RTX, label, prob);
}
void
-jumpif_1 (enum tree_code code, tree op0, tree op1, rtx label)
+jumpif_1 (enum tree_code code, tree op0, tree op1, rtx label, int prob)
{
- do_jump_1 (code, op0, op1, NULL_RTX, label);
+ do_jump_1 (code, op0, op1, NULL_RTX, label, prob);
}
/* Used internally by prefer_and_bit_test. */
@@ -162,11 +171,12 @@ prefer_and_bit_test (enum machine_mode m
}
/* Subroutine of do_jump, dealing with exploded comparisons of the type
- OP0 CODE OP1 . IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump. */
+ OP0 CODE OP1 . IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump.
+ PROB is probability of jump to if_true_label, or -1 if unknown. */
void
do_jump_1 (enum tree_code code, tree op0, tree op1,
- rtx if_false_label, rtx if_true_label)
+ rtx if_false_label, rtx if_true_label, int prob)
{
enum machine_mode mode;
rtx drop_through_label = 0;
@@ -183,12 +193,14 @@ do_jump_1 (enum tree_code code, tree op0
!= MODE_COMPLEX_INT);
if (integer_zerop (op1))
- do_jump (op0, if_true_label, if_false_label);
+ do_jump (op0, if_true_label, if_false_label, inv (prob));
else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
&& !can_compare_p (EQ, TYPE_MODE (inner_type), ccp_jump))
- do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label);
+ do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label,
+ prob);
else
- do_compare_and_jump (op0, op1, EQ, EQ, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, EQ, EQ, if_false_label, if_true_label,
+ prob);
break;
}
@@ -202,12 +214,14 @@ do_jump_1 (enum tree_code code, tree op0
!= MODE_COMPLEX_INT);
if (integer_zerop (op1))
- do_jump (op0, if_false_label, if_true_label);
+ do_jump (op0, if_false_label, if_true_label, prob);
else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
&& !can_compare_p (NE, TYPE_MODE (inner_type), ccp_jump))
- do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label);
+ do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label,
+ inv (prob));
else
- do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label,
+ prob);
break;
}
@@ -215,83 +229,97 @@ do_jump_1 (enum tree_code code, tree op0
mode = TYPE_MODE (TREE_TYPE (op0));
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (LT, mode, ccp_jump))
- do_jump_by_parts_greater (op0, op1, 1, if_false_label, if_true_label);
+ do_jump_by_parts_greater (op0, op1, 1, if_false_label, if_true_label,
+ prob);
else
- do_compare_and_jump (op0, op1, LT, LTU, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, LT, LTU, if_false_label, if_true_label,
+ prob);
break;
case LE_EXPR:
mode = TYPE_MODE (TREE_TYPE (op0));
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (LE, mode, ccp_jump))
- do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label);
+ do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label,
+ inv (prob));
else
- do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label,
+ prob);
break;
case GT_EXPR:
mode = TYPE_MODE (TREE_TYPE (op0));
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (GT, mode, ccp_jump))
- do_jump_by_parts_greater (op0, op1, 0, if_false_label, if_true_label);
+ do_jump_by_parts_greater (op0, op1, 0, if_false_label, if_true_label,
+ prob);
else
- do_compare_and_jump (op0, op1, GT, GTU, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, GT, GTU, if_false_label, if_true_label,
+ prob);
break;
case GE_EXPR:
mode = TYPE_MODE (TREE_TYPE (op0));
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (GE, mode, ccp_jump))
- do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label);
+ do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label,
+ inv (prob));
else
- do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label,
+ prob);
break;
case ORDERED_EXPR:
do_compare_and_jump (op0, op1, ORDERED, ORDERED,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case UNORDERED_EXPR:
do_compare_and_jump (op0, op1, UNORDERED, UNORDERED,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case UNLT_EXPR:
- do_compare_and_jump (op0, op1, UNLT, UNLT, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, UNLT, UNLT, if_false_label, if_true_label,
+ prob);
break;
case UNLE_EXPR:
- do_compare_and_jump (op0, op1, UNLE, UNLE, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, UNLE, UNLE, if_false_label, if_true_label,
+ prob);
break;
case UNGT_EXPR:
- do_compare_and_jump (op0, op1, UNGT, UNGT, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, UNGT, UNGT, if_false_label, if_true_label,
+ prob);
break;
case UNGE_EXPR:
- do_compare_and_jump (op0, op1, UNGE, UNGE, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, UNGE, UNGE, if_false_label, if_true_label,
+ prob);
break;
case UNEQ_EXPR:
- do_compare_and_jump (op0, op1, UNEQ, UNEQ, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, UNEQ, UNEQ, if_false_label, if_true_label,
+ prob);
break;
case LTGT_EXPR:
- do_compare_and_jump (op0, op1, LTGT, LTGT, if_false_label, if_true_label);
+ do_compare_and_jump (op0, op1, LTGT, LTGT, if_false_label, if_true_label,
+ prob);
break;
case TRUTH_ANDIF_EXPR:
if (if_false_label == NULL_RTX)
{
drop_through_label = gen_label_rtx ();
- do_jump (op0, drop_through_label, NULL_RTX);
- do_jump (op1, NULL_RTX, if_true_label);
+ do_jump (op0, drop_through_label, NULL_RTX, prob);
+ do_jump (op1, NULL_RTX, if_true_label, prob);
}
else
{
- do_jump (op0, if_false_label, NULL_RTX);
- do_jump (op1, if_false_label, if_true_label);
+ do_jump (op0, if_false_label, NULL_RTX, prob);
+ do_jump (op1, if_false_label, if_true_label, prob);
}
break;
@@ -299,13 +327,13 @@ do_jump_1 (enum tree_code code, tree op0
if (if_true_label == NULL_RTX)
{
drop_through_label = gen_label_rtx ();
- do_jump (op0, NULL_RTX, drop_through_label);
- do_jump (op1, if_false_label, NULL_RTX);
+ do_jump (op0, NULL_RTX, drop_through_label, prob);
+ do_jump (op1, if_false_label, NULL_RTX, prob);
}
else
{
- do_jump (op0, NULL_RTX, if_true_label);
- do_jump (op1, if_false_label, if_true_label);
+ do_jump (op0, NULL_RTX, if_true_label, prob);
+ do_jump (op1, if_false_label, if_true_label, prob);
}
break;
@@ -327,10 +355,12 @@ do_jump_1 (enum tree_code code, tree op0
do_jump always does any pending stack adjust except when it does not
actually perform a jump. An example where there is no jump
- is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null. */
+ is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null.
+
+ PROB is probability of jump to if_true_label, or -1 if unknown. */
void
-do_jump (tree exp, rtx if_false_label, rtx if_true_label)
+do_jump (tree exp, rtx if_false_label, rtx if_true_label, int prob)
{
enum tree_code code = TREE_CODE (exp);
rtx temp;
@@ -377,11 +407,11 @@ do_jump (tree exp, rtx if_false_label, r
case LROTATE_EXPR:
case RROTATE_EXPR:
/* These cannot change zero->nonzero or vice versa. */
- do_jump (TREE_OPERAND (exp, 0), if_false_label, if_true_label);
+ do_jump (TREE_OPERAND (exp, 0), if_false_label, if_true_label, prob);
break;
case TRUTH_NOT_EXPR:
- do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label);
+ do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label, prob);
break;
case COND_EXPR:
@@ -397,10 +427,10 @@ do_jump (tree exp, rtx if_false_label, r
}
do_pending_stack_adjust ();
- do_jump (TREE_OPERAND (exp, 0), label1, NULL_RTX);
- do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label);
+ do_jump (TREE_OPERAND (exp, 0), label1, NULL_RTX, -1);
+ do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label, prob);
emit_label (label1);
- do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label);
+ do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label, prob);
break;
}
@@ -431,7 +461,8 @@ do_jump (tree exp, rtx if_false_label, r
&& TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (exp))
&& have_insn_for (COMPARE, TYPE_MODE (type)))
{
- do_jump (fold_convert (type, exp), if_false_label, if_true_label);
+ do_jump (fold_convert (type, exp), if_false_label, if_true_label,
+ prob);
break;
}
goto normal;
@@ -460,7 +491,7 @@ do_jump (tree exp, rtx if_false_label, r
case TRUTH_ORIF_EXPR:
other_code:
do_jump_1 (code, TREE_OPERAND (exp, 0), TREE_OPERAND (exp, 1),
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case BIT_AND_EXPR:
@@ -471,6 +502,7 @@ do_jump (tree exp, rtx if_false_label, r
{
tree exp0 = TREE_OPERAND (exp, 0);
rtx set_label, clr_label;
+ int setclr_prob = prob;
/* Strip narrowing integral type conversions. */
while (CONVERT_EXPR_P (exp0)
@@ -486,6 +518,7 @@ do_jump (tree exp, rtx if_false_label, r
exp0 = TREE_OPERAND (exp0, 0);
clr_label = if_true_label;
set_label = if_false_label;
+ setclr_prob = inv (prob);
}
else
{
@@ -508,7 +541,7 @@ do_jump (tree exp, rtx if_false_label, r
= (unsigned HOST_WIDE_INT) 1 << TREE_INT_CST_LOW (shift);
do_jump (build2 (BIT_AND_EXPR, argtype, arg,
build_int_cst_wide_type (argtype, mask, 0)),
- clr_label, set_label);
+ clr_label, set_label, setclr_prob);
break;
}
}
@@ -530,7 +563,8 @@ do_jump (tree exp, rtx if_false_label, r
&& TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (exp))
&& have_insn_for (COMPARE, TYPE_MODE (type)))
{
- do_jump (fold_convert (type, exp), if_false_label, if_true_label);
+ do_jump (fold_convert (type, exp), if_false_label, if_true_label,
+ prob);
break;
}
@@ -556,7 +590,7 @@ do_jump (tree exp, rtx if_false_label, r
/* High branch cost, expand as the bitwise OR of the conditions.
Do the same if the RHS has side effects, because we're effectively
turning a TRUTH_OR_EXPR into a TRUTH_ORIF_EXPR. */
- if (BRANCH_COST (optimize_insn_for_speed_p (), false)>= 4
+ if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 4
|| TREE_SIDE_EFFECTS (TREE_OPERAND (exp, 1)))
goto normal;
code = TRUTH_ORIF_EXPR;
@@ -580,7 +614,7 @@ do_jump (tree exp, rtx if_false_label, r
do_compare_rtx_and_jump (temp, CONST0_RTX (GET_MODE (temp)),
NE, TYPE_UNSIGNED (TREE_TYPE (exp)),
GET_MODE (temp), NULL_RTX,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
}
if (drop_through_label)
@@ -596,7 +630,8 @@ do_jump (tree exp, rtx if_false_label, r
static void
do_jump_by_parts_greater_rtx (enum machine_mode mode, int unsignedp, rtx op0,
- rtx op1, rtx if_false_label, rtx if_true_label)
+ rtx op1, rtx if_false_label, rtx if_true_label,
+ int prob)
{
int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
rtx drop_through_label = 0;
@@ -628,11 +663,12 @@ do_jump_by_parts_greater_rtx (enum machi
/* All but high-order word must be compared as unsigned. */
do_compare_rtx_and_jump (op0_word, op1_word, GT,
(unsignedp || i > 0), word_mode, NULL_RTX,
- NULL_RTX, if_true_label);
+ NULL_RTX, if_true_label, prob);
/* Consider lower words only if these are equal. */
do_compare_rtx_and_jump (op0_word, op1_word, NE, unsignedp, word_mode,
- NULL_RTX, NULL_RTX, if_false_label);
+ NULL_RTX, NULL_RTX, if_false_label,
+ inv (prob));
}
if (if_false_label)
@@ -648,7 +684,7 @@ do_jump_by_parts_greater_rtx (enum machi
static void
do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap,
- rtx if_false_label, rtx if_true_label)
+ rtx if_false_label, rtx if_true_label, int prob)
{
rtx op0 = expand_normal (swap ? treeop1 : treeop0);
rtx op1 = expand_normal (swap ? treeop0 : treeop1);
@@ -656,7 +692,7 @@ do_jump_by_parts_greater (tree treeop0,
int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
do_jump_by_parts_greater_rtx (mode, unsignedp, op0, op1, if_false_label,
- if_true_label);
+ if_true_label, prob);
}
/* Jump according to whether OP0 is 0. We assume that OP0 has an integer
@@ -666,7 +702,7 @@ do_jump_by_parts_greater (tree treeop0,
static void
do_jump_by_parts_zero_rtx (enum machine_mode mode, rtx op0,
- rtx if_false_label, rtx if_true_label)
+ rtx if_false_label, rtx if_true_label, int prob)
{
int nwords = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
rtx part;
@@ -688,8 +724,7 @@ do_jump_by_parts_zero_rtx (enum machine_
if (part != 0)
{
do_compare_rtx_and_jump (part, const0_rtx, EQ, 1, word_mode,
- NULL_RTX, if_false_label, if_true_label);
-
+ NULL_RTX, if_false_label, if_true_label, prob);
return;
}
@@ -700,7 +735,7 @@ do_jump_by_parts_zero_rtx (enum machine_
for (i = 0; i < nwords; i++)
do_compare_rtx_and_jump (operand_subword_force (op0, i, mode),
const0_rtx, EQ, 1, word_mode, NULL_RTX,
- if_false_label, NULL_RTX);
+ if_false_label, NULL_RTX, prob);
if (if_true_label)
emit_jump (if_true_label);
@@ -716,7 +751,7 @@ do_jump_by_parts_zero_rtx (enum machine_
static void
do_jump_by_parts_equality_rtx (enum machine_mode mode, rtx op0, rtx op1,
- rtx if_false_label, rtx if_true_label)
+ rtx if_false_label, rtx if_true_label, int prob)
{
int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
rtx drop_through_label = 0;
@@ -724,12 +759,14 @@ do_jump_by_parts_equality_rtx (enum mach
if (op1 == const0_rtx)
{
- do_jump_by_parts_zero_rtx (mode, op0, if_false_label, if_true_label);
+ do_jump_by_parts_zero_rtx (mode, op0, if_false_label, if_true_label,
+ prob);
return;
}
else if (op0 == const0_rtx)
{
- do_jump_by_parts_zero_rtx (mode, op1, if_false_label, if_true_label);
+ do_jump_by_parts_zero_rtx (mode, op1, if_false_label, if_true_label,
+ prob);
return;
}
@@ -740,7 +777,7 @@ do_jump_by_parts_equality_rtx (enum mach
do_compare_rtx_and_jump (operand_subword_force (op0, i, mode),
operand_subword_force (op1, i, mode),
EQ, 0, word_mode, NULL_RTX,
- if_false_label, NULL_RTX);
+ if_false_label, NULL_RTX, prob);
if (if_true_label)
emit_jump (if_true_label);
@@ -753,13 +790,13 @@ do_jump_by_parts_equality_rtx (enum mach
static void
do_jump_by_parts_equality (tree treeop0, tree treeop1, rtx if_false_label,
- rtx if_true_label)
+ rtx if_true_label, int prob)
{
rtx op0 = expand_normal (treeop0);
rtx op1 = expand_normal (treeop1);
enum machine_mode mode = TYPE_MODE (TREE_TYPE (treeop0));
do_jump_by_parts_equality_rtx (mode, op0, op1, if_false_label,
- if_true_label);
+ if_true_label, prob);
}
/* Split a comparison into two others, the second of which has the other
@@ -849,10 +886,11 @@ split_comparison (enum rtx_code code, en
void
do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
enum machine_mode mode, rtx size, rtx if_false_label,
- rtx if_true_label)
+ rtx if_true_label, int prob)
{
rtx tem;
rtx dummy_label = NULL_RTX;
+ rtx last;
/* Reverse the comparison if that is safe and we want to jump if it is
false. Also convert to the reverse comparison if the target can
@@ -878,6 +916,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op
if_true_label = if_false_label;
if_false_label = tem;
code = rcode;
+ prob = inv (prob);
}
}
@@ -924,52 +963,56 @@ do_compare_rtx_and_jump (rtx op0, rtx op
{
case LTU:
do_jump_by_parts_greater_rtx (mode, 1, op1, op0,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case LEU:
do_jump_by_parts_greater_rtx (mode, 1, op0, op1,
- if_true_label, if_false_label);
+ if_true_label, if_false_label,
+ inv (prob));
break;
case GTU:
do_jump_by_parts_greater_rtx (mode, 1, op0, op1,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case GEU:
do_jump_by_parts_greater_rtx (mode, 1, op1, op0,
- if_true_label, if_false_label);
+ if_true_label, if_false_label,
+ inv (prob));
break;
case LT:
do_jump_by_parts_greater_rtx (mode, 0, op1, op0,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case LE:
do_jump_by_parts_greater_rtx (mode, 0, op0, op1,
- if_true_label, if_false_label);
+ if_true_label, if_false_label,
+ inv (prob));
break;
case GT:
do_jump_by_parts_greater_rtx (mode, 0, op0, op1,
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
break;
case GE:
do_jump_by_parts_greater_rtx (mode, 0, op1, op0,
- if_true_label, if_false_label);
+ if_true_label, if_false_label,
+ inv (prob));
break;
case EQ:
do_jump_by_parts_equality_rtx (mode, op0, op1, if_false_label,
- if_true_label);
+ if_true_label, prob);
break;
case NE:
do_jump_by_parts_equality_rtx (mode, op0, op1, if_true_label,
- if_false_label);
+ if_false_label, inv (prob));
break;
default:
@@ -1025,16 +1068,38 @@ do_compare_rtx_and_jump (rtx op0, rtx op
else
dest_label = if_false_label;
do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode,
- size, dest_label, NULL_RTX);
+ size, dest_label, NULL_RTX, prob);
}
else
do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode,
- size, NULL_RTX, if_true_label);
+ size, NULL_RTX, if_true_label, prob);
}
}
+ last = get_last_insn ();
emit_cmp_and_jump_insns (op0, op1, code, size, mode, unsignedp,
if_true_label);
+ if (prob != -1 && profile_status != PROFILE_ABSENT)
+ {
+ for (last = NEXT_INSN (last);
+ last && NEXT_INSN (last);
+ last = NEXT_INSN (last))
+ if (JUMP_P (last))
+ break;
+ if (!last
+ || !JUMP_P (last)
+ || NEXT_INSN (last)
+ || !any_condjump_p (last))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Failed to add probability note\n");
+ }
+ else
+ {
+ gcc_assert (!find_reg_note (last, REG_BR_PROB, 0));
+ add_reg_note (last, REG_BR_PROB, GEN_INT (prob));
+ }
+ }
}
if (if_false_label)
@@ -1056,7 +1121,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op
static void
do_compare_and_jump (tree treeop0, tree treeop1, enum rtx_code signed_code,
enum rtx_code unsigned_code, rtx if_false_label,
- rtx if_true_label)
+ rtx if_true_label, int prob)
{
rtx op0, op1;
tree type;
@@ -1116,7 +1181,7 @@ do_compare_and_jump (tree treeop0, tree
do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode,
((mode == BLKmode)
? expr_size (treeop0) : NULL_RTX),
- if_false_label, if_true_label);
+ if_false_label, if_true_label, prob);
}
#include "gt-dojump.h"
--- gcc/cfgexpand.c.jj 2010-02-12 13:25:23.000000000 +0100
+++ gcc/cfgexpand.c 2010-02-18 15:08:42.000000000 +0100
@@ -91,46 +91,6 @@ gimple_assign_rhs_to_tree (gimple stmt)
}
-/* Verify that there is exactly single jump instruction since last and attach
- REG_BR_PROB note specifying probability.
- ??? We really ought to pass the probability down to RTL expanders and let it
- re-distribute it when the conditional expands into multiple conditionals.
- This is however difficult to do. */
-void
-add_reg_br_prob_note (rtx last, int probability)
-{
- if (profile_status == PROFILE_ABSENT)
- return;
- for (last = NEXT_INSN (last); last && NEXT_INSN (last); last = NEXT_INSN (last))
- if (JUMP_P (last))
- {
- /* It is common to emit condjump-around-jump sequence when we don't know
- how to reverse the conditional. Special case this. */
- if (!any_condjump_p (last)
- || !JUMP_P (NEXT_INSN (last))
- || !simplejump_p (NEXT_INSN (last))
- || !NEXT_INSN (NEXT_INSN (last))
- || !BARRIER_P (NEXT_INSN (NEXT_INSN (last)))
- || !NEXT_INSN (NEXT_INSN (NEXT_INSN (last)))
- || !LABEL_P (NEXT_INSN (NEXT_INSN (NEXT_INSN (last))))
- || NEXT_INSN (NEXT_INSN (NEXT_INSN (NEXT_INSN (last)))))
- goto failed;
- gcc_assert (!find_reg_note (last, REG_BR_PROB, 0));
- add_reg_note (last, REG_BR_PROB,
- GEN_INT (REG_BR_PROB_BASE - probability));
- return;
- }
- if (!last || !JUMP_P (last) || !any_condjump_p (last))
- goto failed;
- gcc_assert (!find_reg_note (last, REG_BR_PROB, 0));
- add_reg_note (last, REG_BR_PROB, GEN_INT (probability));
- return;
-failed:
- if (dump_file)
- fprintf (dump_file, "Failed to add probability note\n");
-}
-
-
#ifndef STACK_ALIGNMENT_NEEDED
#define STACK_ALIGNMENT_NEEDED 1
#endif
@@ -1663,8 +1623,8 @@ expand_gimple_cond (basic_block bb, gimp
two-way jump that needs to be decomposed into two basic blocks. */
if (false_edge->dest == bb->next_bb)
{
- jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest));
- add_reg_br_prob_note (last, true_edge->probability);
+ jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
+ true_edge->probability);
maybe_dump_rtl_for_gimple_stmt (stmt, last);
if (true_edge->goto_locus)
{
@@ -1679,8 +1639,8 @@ expand_gimple_cond (basic_block bb, gimp
}
if (true_edge->dest == bb->next_bb)
{
- jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest));
- add_reg_br_prob_note (last, false_edge->probability);
+ jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
+ false_edge->probability);
maybe_dump_rtl_for_gimple_stmt (stmt, last);
if (false_edge->goto_locus)
{
@@ -1694,8 +1654,8 @@ expand_gimple_cond (basic_block bb, gimp
return NULL;
}
- jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest));
- add_reg_br_prob_note (last, true_edge->probability);
+ jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
+ true_edge->probability);
last = get_last_insn ();
if (false_edge->goto_locus)
{
--- gcc/Makefile.in.jj 2010-02-17 10:52:00.000000000 +0100
+++ gcc/Makefile.in 2010-02-17 10:52:00.000000000 +0100
@@ -2788,7 +2788,7 @@ expr.o : expr.c $(CONFIG_H) $(SYSTEM_H)
$(TREE_PASS_H) $(DF_H) $(DIAGNOSTIC_H) vecprim.h $(SSAEXPAND_H)
dojump.o : dojump.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(RTL_H) $(TREE_H) \
$(FLAGS_H) $(FUNCTION_H) $(EXPR_H) $(OPTABS_H) $(INSN_ATTR_H) insn-config.h \
- langhooks.h $(GGC_H) gt-dojump.h vecprim.h $(BASIC_BLOCK_H)
+ langhooks.h $(GGC_H) gt-dojump.h vecprim.h $(BASIC_BLOCK_H) output.h
builtins.o : builtins.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(RTL_H) \
$(TREE_H) $(GIMPLE_H) $(FLAGS_H) $(TARGET_H) $(FUNCTION_H) $(REGS_H) \
$(EXPR_H) $(OPTABS_H) insn-config.h $(RECOG_H) output.h typeclass.h \
--- gcc/builtins.c.jj 2010-02-10 15:57:23.000000000 +0100
+++ gcc/builtins.c 2010-02-18 15:06:25.000000000 +0100
@@ -1881,7 +1881,9 @@ expand_errno_check (tree exp, rtx target
/* Test the result; if it is NaN, set errno=EDOM because
the argument was not in the domain. */
do_compare_rtx_and_jump (target, target, EQ, 0, GET_MODE (target),
- NULL_RTX, NULL_RTX, lab);
+ NULL_RTX, NULL_RTX, lab,
+ /* The jump is very likely. */
+ REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1));
#ifdef TARGET_EDOM
/* If this built-in doesn't throw an exception, set errno directly. */
--- gcc/expmed.c.jj 2009-12-31 11:49:14.000000000 +0100
+++ gcc/expmed.c 2010-02-18 15:10:19.000000000 +0100
@@ -5792,7 +5792,7 @@ emit_store_flag_force (rtx target, enum
{
label = gen_label_rtx ();
do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp,
- mode, NULL_RTX, NULL_RTX, label);
+ mode, NULL_RTX, NULL_RTX, label, -1);
emit_move_insn (target, trueval);
emit_label (label);
return target;
@@ -5830,7 +5830,7 @@ emit_store_flag_force (rtx target, enum
emit_move_insn (target, trueval);
label = gen_label_rtx ();
do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX,
- NULL_RTX, label);
+ NULL_RTX, label, -1);
emit_move_insn (target, falseval);
emit_label (label);
@@ -5848,5 +5848,5 @@ do_cmp_and_jump (rtx arg1, rtx arg2, enu
{
int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode,
- NULL_RTX, NULL_RTX, label);
+ NULL_RTX, NULL_RTX, label, -1);
}
--- gcc/expr.c.jj 2010-02-09 19:17:08.000000000 +0100
+++ gcc/expr.c 2010-02-18 15:11:28.000000000 +0100
@@ -4551,7 +4551,7 @@ store_expr (tree exp, rtx target, int ca
do_pending_stack_adjust ();
NO_DEFER_POP;
- jumpifnot (TREE_OPERAND (exp, 0), lab1);
+ jumpifnot (TREE_OPERAND (exp, 0), lab1, -1);
store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
nontemporal);
emit_jump_insn (gen_jump (lab2));
@@ -5547,7 +5547,7 @@ store_constructor (tree exp, rtx target,
/* Generate a conditional jump to exit the loop. */
exit_cond = build2 (LT_EXPR, integer_type_node,
index, hi_index);
- jumpif (exit_cond, loop_end);
+ jumpif (exit_cond, loop_end, -1);
/* Update the loop counter, and jump to the head of
the loop. */
@@ -8013,7 +8013,8 @@ expand_expr_real_2 (sepops ops, rtx targ
temp = gen_label_rtx ();
do_compare_rtx_and_jump (target, cmpop1, comparison_code,
- unsignedp, mode, NULL_RTX, NULL_RTX, temp);
+ unsignedp, mode, NULL_RTX, NULL_RTX, temp,
+ -1);
}
emit_move_insn (target, op1);
emit_label (temp);
@@ -8121,7 +8122,7 @@ expand_expr_real_2 (sepops ops, rtx targ
emit_move_insn (target, const0_rtx);
op1 = gen_label_rtx ();
- jumpifnot_1 (code, treeop0, treeop1, op1);
+ jumpifnot_1 (code, treeop0, treeop1, op1, -1);
emit_move_insn (target, const1_rtx);
@@ -9418,7 +9419,7 @@ expand_expr_real_1 (tree exp, rtx target
emit_move_insn (target, const0_rtx);
op1 = gen_label_rtx ();
- jumpifnot_1 (code, treeop0, treeop1, op1);
+ jumpifnot_1 (code, treeop0, treeop1, op1, -1);
if (target)
emit_move_insn (target, const1_rtx);
@@ -9475,7 +9476,7 @@ expand_expr_real_1 (tree exp, rtx target
NO_DEFER_POP;
op0 = gen_label_rtx ();
op1 = gen_label_rtx ();
- jumpifnot (treeop0, op0);
+ jumpifnot (treeop0, op0, -1);
store_expr (treeop1, temp,
modifier == EXPAND_STACK_PARM,
false);
@@ -9521,7 +9522,7 @@ expand_expr_real_1 (tree exp, rtx target
int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
do_jump (TREE_OPERAND (rhs, 1),
value ? label : 0,
- value ? 0 : label);
+ value ? 0 : label, -1);
expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
MOVE_NONTEMPORAL (exp));
do_pending_stack_adjust ();
--- gcc/stmt.c.jj 2009-11-25 16:47:37.000000000 +0100
+++ gcc/stmt.c 2010-02-18 15:14:34.000000000 +0100
@@ -2535,7 +2535,7 @@ do_jump_if_equal (enum machine_mode mode
int unsignedp)
{
do_compare_rtx_and_jump (op0, op1, EQ, unsignedp, mode,
- NULL_RTX, NULL_RTX, label);
+ NULL_RTX, NULL_RTX, label, -1);
}
/* Not all case values are encountered equally. This function
--- gcc/cfgrtl.c.jj 2009-11-25 16:47:37.000000000 +0100
+++ gcc/cfgrtl.c 2010-02-18 15:01:53.000000000 +0100
@@ -3025,7 +3025,7 @@ rtl_lv_add_condition_to_bb (basic_block
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
do_compare_rtx_and_jump (op0, op1, comp, 0,
- mode, NULL_RTX, NULL_RTX, label);
+ mode, NULL_RTX, NULL_RTX, label, -1);
jump = get_last_insn ();
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
--- gcc/loop-unswitch.c.jj 2009-11-28 17:18:34.000000000 +0100
+++ gcc/loop-unswitch.c 2010-02-18 15:13:26.000000000 +0100
@@ -121,7 +121,7 @@ compare_and_jump_seq (rtx op0, rtx op1,
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
do_compare_rtx_and_jump (op0, op1, comp, 0,
- mode, NULL_RTX, NULL_RTX, label);
+ mode, NULL_RTX, NULL_RTX, label, -1);
jump = get_last_insn ();
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
--- gcc/config/rs6000/rs6000.c.jj 2010-02-07 11:53:21.000000000 +0100
+++ gcc/config/rs6000/rs6000.c 2010-02-18 15:16:34.000000000 +0100
@@ -17954,7 +17954,7 @@ rs6000_aix_emit_builtin_unwind_init (voi
do_compare_rtx_and_jump (opcode, tocompare, EQ, 1,
SImode, NULL_RTX, NULL_RTX,
- no_toc_save_needed);
+ no_toc_save_needed, -1);
mem = gen_frame_mem (Pmode,
gen_rtx_PLUS (Pmode, stack_top,
--- gcc/optabs.c.jj 2009-11-28 17:16:16.000000000 +0100
+++ gcc/optabs.c 2010-02-18 15:14:03.000000000 +0100
@@ -1127,7 +1127,7 @@ expand_doubleword_shift (enum machine_mo
NO_DEFER_POP;
do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
- 0, 0, subword_label);
+ 0, 0, subword_label, -1);
OK_DEFER_POP;
if (!expand_superword_shift (binoptab, outof_input, superword_op1,
@@ -3469,7 +3469,7 @@ expand_abs (enum machine_mode mode, rtx
NO_DEFER_POP;
do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
- NULL_RTX, NULL_RTX, op1);
+ NULL_RTX, NULL_RTX, op1, -1);
op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
target, target, 0);
Jakub