PR middle-end/42233

	* expr.h (jumpifnot, jumpifnot_1, jumpif, jumpif_1, do_jump,
	do_jump_1, do_compare_rtx_and_jump): Add PROB argument.
	* dojump.c: Include output.h.
	(inv): New inline function.
	(jumpifnot, jumpifnot_1, jumpif, jumpif_1, do_jump_1, do_jump,
	do_jump_by_parts_greater_rtx, do_jump_by_parts_greater,
	do_jump_by_parts_zero_rtx, do_jump_by_parts_equality_rtx,
	do_jump_by_parts_equality, do_compare_and_jump): Add PROB
	argument, pass it down to other calls.
	(do_compare_rtx_and_jump): Likewise.  If PROB is not -1,
	add REG_BR_PROB note to the conditional jump.
	* cfgexpand.c (add_reg_br_prob_note): Removed.
	(expand_gimple_cond): Don't call it, add the probability
	as last argument to jumpif_1/jumpifnot_1.
	* Makefile.in (dojump.o): Depend on output.h.
	* builtins.c (expand_errno_check): Adjust do_compare_rtx_and_jump
	callers.
	* expmed.c (emit_store_flag_force, do_cmp_and_jump): Likewise.
	* stmt.c (do_jump_if_equal): Likewise.
	* cfgrtl.c (rtl_lv_add_condition_to_bb): Likewise.
	* loop-unswitch.c (compare_and_jump_seq): Likewise.
	* config/rs6000/rs6000.c (rs6000_aix_emit_builtin_unwind_init):
	Likewise.
	* optabs.c (expand_doubleword_shift, expand_abs): Likewise.
	* expr.c (expand_expr_real_1): Adjust do_jump, jumpifnot and
	jumpifnot_1 callers.
	(expand_expr_real_2): Adjust jumpifnot_1 and do_compare_rtx_and_jump
	callers.
	(store_expr): Adjust jumpifnot caller.
	(store_constructor): Adjust jumpif caller.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@156889 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 152 insertions, 87 deletions

diff --git a/gcc/dojump.c b/gcc/dojump.c
index 20d52ea5a28..3d03b4c32d4 100644
--- a/gcc/dojump.c
+++ b/gcc/dojump.c
@@ -1,6 +1,6 @@
 /* Convert tree expression to rtl instructions, for GNU compiler.
    Copyright (C) 1988, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
+   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
    Free Software Foundation, Inc.
 
 This file is part of GCC.
@@ -35,12 +35,21 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 #include "ggc.h"
 #include "basic-block.h"
+#include "output.h"
 
 static bool prefer_and_bit_test (enum machine_mode, int);
-static void do_jump_by_parts_greater (tree, tree, int, rtx, rtx);
-static void do_jump_by_parts_equality (tree, tree, rtx, rtx);
+static void do_jump_by_parts_greater (tree, tree, int, rtx, rtx, int);
+static void do_jump_by_parts_equality (tree, tree, rtx, rtx, int);
 static void do_compare_and_jump	(tree, tree, enum rtx_code, enum rtx_code, rtx,
-				 rtx);
+				 rtx, int);
+
+/* Invert probability if there is any.  -1 stands for unknown.  */
+
+static inline int
+inv (int prob)
+{
+  return prob == -1 ? -1 : REG_BR_PROB_BASE - prob;
+}
 
 /* At the start of a function, record that we have no previously-pushed
    arguments waiting to be popped.  */
@@ -96,29 +105,29 @@ do_pending_stack_adjust (void)
    functions here.  */
 
 void
-jumpifnot (tree exp, rtx label)
+jumpifnot (tree exp, rtx label, int prob)
 {
-  do_jump (exp, label, NULL_RTX);
+  do_jump (exp, label, NULL_RTX, inv (prob));
 }
 
 void
-jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx label)
+jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx label, int prob)
 {
-  do_jump_1 (code, op0, op1, label, NULL_RTX);
+  do_jump_1 (code, op0, op1, label, NULL_RTX, inv (prob));
 }
 
 /* Generate code to evaluate EXP and jump to LABEL if the value is nonzero.  */
 
 void
-jumpif (tree exp, rtx label)
+jumpif (tree exp, rtx label, int prob)
 {
-  do_jump (exp, NULL_RTX, label);
+  do_jump (exp, NULL_RTX, label, prob);
 }
 
 void
-jumpif_1 (enum tree_code code, tree op0, tree op1, rtx label)
+jumpif_1 (enum tree_code code, tree op0, tree op1, rtx label, int prob)
 {
-  do_jump_1 (code, op0, op1, NULL_RTX, label);
+  do_jump_1 (code, op0, op1, NULL_RTX, label, prob);
 }
 
 /* Used internally by prefer_and_bit_test.  */
@@ -162,11 +171,12 @@ prefer_and_bit_test (enum machine_mode mode, int bitnum)
 }
 
 /* Subroutine of do_jump, dealing with exploded comparisons of the type
-   OP0 CODE OP1 .  IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump.  */
+   OP0 CODE OP1 .  IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump.
+   PROB is probability of jump to if_true_label, or -1 if unknown.  */
 
 void
 do_jump_1 (enum tree_code code, tree op0, tree op1,
-	   rtx if_false_label, rtx if_true_label)
+	   rtx if_false_label, rtx if_true_label, int prob)
 {
   enum machine_mode mode;
   rtx drop_through_label = 0;
@@ -183,12 +193,14 @@ do_jump_1 (enum tree_code code, tree op0, tree op1,
 		    != MODE_COMPLEX_INT);
 
         if (integer_zerop (op1))
-          do_jump (op0, if_true_label, if_false_label);
+	  do_jump (op0, if_true_label, if_false_label, inv (prob));
         else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
                  && !can_compare_p (EQ, TYPE_MODE (inner_type), ccp_jump))
-          do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label);
+	  do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label,
+				     prob);
         else
-          do_compare_and_jump (op0, op1, EQ, EQ, if_false_label, if_true_label);
+	  do_compare_and_jump (op0, op1, EQ, EQ, if_false_label, if_true_label,
+			       prob);
         break;
       }
 
@@ -202,12 +214,14 @@ do_jump_1 (enum tree_code code, tree op0, tree op1,
 		    != MODE_COMPLEX_INT);
 
         if (integer_zerop (op1))
-          do_jump (op0, if_false_label, if_true_label);
+	  do_jump (op0, if_false_label, if_true_label, prob);
         else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
            && !can_compare_p (NE, TYPE_MODE (inner_type), ccp_jump))
-          do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label);
+	  do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label,
+				     inv (prob));
         else
-          do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label);
+	  do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label,
+			       prob);
         break;
       }
 
@@ -215,83 +229,97 @@ do_jump_1 (enum tree_code code, tree op0, tree op1,
       mode = TYPE_MODE (TREE_TYPE (op0));
       if (GET_MODE_CLASS (mode) == MODE_INT
           && ! can_compare_p (LT, mode, ccp_jump))
-        do_jump_by_parts_greater (op0, op1, 1, if_false_label, if_true_label);
+	do_jump_by_parts_greater (op0, op1, 1, if_false_label, if_true_label,
+				  prob);
       else
-        do_compare_and_jump (op0, op1, LT, LTU, if_false_label, if_true_label);
+	do_compare_and_jump (op0, op1, LT, LTU, if_false_label, if_true_label,
+			     prob);
       break;
 
     case LE_EXPR:
       mode = TYPE_MODE (TREE_TYPE (op0));
       if (GET_MODE_CLASS (mode) == MODE_INT
           && ! can_compare_p (LE, mode, ccp_jump))
-        do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label);
+	do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label,
+				  inv (prob));
       else
-        do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label);
+	do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label,
+			     prob);
       break;
 
     case GT_EXPR:
       mode = TYPE_MODE (TREE_TYPE (op0));
       if (GET_MODE_CLASS (mode) == MODE_INT
           && ! can_compare_p (GT, mode, ccp_jump))
-        do_jump_by_parts_greater (op0, op1, 0, if_false_label, if_true_label);
+	do_jump_by_parts_greater (op0, op1, 0, if_false_label, if_true_label,
+				  prob);
       else
-        do_compare_and_jump (op0, op1, GT, GTU, if_false_label, if_true_label);
+	do_compare_and_jump (op0, op1, GT, GTU, if_false_label, if_true_label,
+			     prob);
       break;
 
     case GE_EXPR:
       mode = TYPE_MODE (TREE_TYPE (op0));
       if (GET_MODE_CLASS (mode) == MODE_INT
           && ! can_compare_p (GE, mode, ccp_jump))
-        do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label);
+	do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label,
+				  inv (prob));
       else
-        do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label);
+	do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label,
+			     prob);
       break;
 
     case ORDERED_EXPR:
       do_compare_and_jump (op0, op1, ORDERED, ORDERED,
-			   if_false_label, if_true_label);
+			   if_false_label, if_true_label, prob);
       break;
 
     case UNORDERED_EXPR:
       do_compare_and_jump (op0, op1, UNORDERED, UNORDERED,
-			   if_false_label, if_true_label);
+			   if_false_label, if_true_label, prob);
       break;
 
     case UNLT_EXPR:
-      do_compare_and_jump (op0, op1, UNLT, UNLT, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, UNLT, UNLT, if_false_label, if_true_label,
+			   prob);
       break;
 
     case UNLE_EXPR:
-      do_compare_and_jump (op0, op1, UNLE, UNLE, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, UNLE, UNLE, if_false_label, if_true_label,
+			   prob);
       break;
 
     case UNGT_EXPR:
-      do_compare_and_jump (op0, op1, UNGT, UNGT, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, UNGT, UNGT, if_false_label, if_true_label,
+			   prob);
       break;
 
     case UNGE_EXPR:
-      do_compare_and_jump (op0, op1, UNGE, UNGE, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, UNGE, UNGE, if_false_label, if_true_label,
+			   prob);
       break;
 
     case UNEQ_EXPR:
-      do_compare_and_jump (op0, op1, UNEQ, UNEQ, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, UNEQ, UNEQ, if_false_label, if_true_label,
+			   prob);
       break;
 
     case LTGT_EXPR:
-      do_compare_and_jump (op0, op1, LTGT, LTGT, if_false_label, if_true_label);
+      do_compare_and_jump (op0, op1, LTGT, LTGT, if_false_label, if_true_label,
+			   prob);
       break;
 
     case TRUTH_ANDIF_EXPR:
       if (if_false_label == NULL_RTX)
         {
 	  drop_through_label = gen_label_rtx ();
-          do_jump (op0, drop_through_label, NULL_RTX);
-          do_jump (op1, NULL_RTX, if_true_label);
+	  do_jump (op0, drop_through_label, NULL_RTX, prob);
+	  do_jump (op1, NULL_RTX, if_true_label, prob);
 	}
       else
 	{
-	  do_jump (op0, if_false_label, NULL_RTX);
-          do_jump (op1, if_false_label, if_true_label);
+	  do_jump (op0, if_false_label, NULL_RTX, prob);
+	  do_jump (op1, if_false_label, if_true_label, prob);
 	}
       break;
 
@@ -299,13 +327,13 @@ do_jump_1 (enum tree_code code, tree op0, tree op1,
       if (if_true_label == NULL_RTX)
 	{
           drop_through_label = gen_label_rtx ();
-          do_jump (op0, NULL_RTX, drop_through_label);
-          do_jump (op1, if_false_label, NULL_RTX);
+	  do_jump (op0, NULL_RTX, drop_through_label, prob);
+	  do_jump (op1, if_false_label, NULL_RTX, prob);
 	}
       else
 	{
-          do_jump (op0, NULL_RTX, if_true_label);
-          do_jump (op1, if_false_label, if_true_label);
+	  do_jump (op0, NULL_RTX, if_true_label, prob);
+	  do_jump (op1, if_false_label, if_true_label, prob);
 	}
       break;
 
@@ -327,10 +355,12 @@ do_jump_1 (enum tree_code code, tree op0, tree op1,
 
    do_jump always does any pending stack adjust except when it does not
    actually perform a jump.  An example where there is no jump
-   is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null.  */
+   is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null.
+
+   PROB is probability of jump to if_true_label, or -1 if unknown.  */
 
 void
-do_jump (tree exp, rtx if_false_label, rtx if_true_label)
+do_jump (tree exp, rtx if_false_label, rtx if_true_label, int prob)
 {
   enum tree_code code = TREE_CODE (exp);
   rtx temp;
@@ -377,11 +407,11 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
     case LROTATE_EXPR:
     case RROTATE_EXPR:
       /* These cannot change zero->nonzero or vice versa.  */
-      do_jump (TREE_OPERAND (exp, 0), if_false_label, if_true_label);
+      do_jump (TREE_OPERAND (exp, 0), if_false_label, if_true_label, prob);
       break;
 
     case TRUTH_NOT_EXPR:
-      do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label);
+      do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label, prob);
       break;
 
     case COND_EXPR:
@@ -397,10 +427,10 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
 	  }
 
         do_pending_stack_adjust ();
-        do_jump (TREE_OPERAND (exp, 0), label1, NULL_RTX);
-        do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label);
+	do_jump (TREE_OPERAND (exp, 0), label1, NULL_RTX, -1);
+	do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label, prob);
         emit_label (label1);
-        do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label);
+	do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label, prob);
 	break;
       }
 
@@ -431,7 +461,8 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
             && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (exp))
             && have_insn_for (COMPARE, TYPE_MODE (type)))
           {
-            do_jump (fold_convert (type, exp), if_false_label, if_true_label);
+	    do_jump (fold_convert (type, exp), if_false_label, if_true_label,
+		     prob);
             break;
           }
         goto normal;
@@ -460,7 +491,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
     case TRUTH_ORIF_EXPR:
     other_code:
       do_jump_1 (code, TREE_OPERAND (exp, 0), TREE_OPERAND (exp, 1),
-		 if_false_label, if_true_label);
+		 if_false_label, if_true_label, prob);
       break;
 
     case BIT_AND_EXPR:
@@ -471,6 +502,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
 	{
 	  tree exp0 = TREE_OPERAND (exp, 0);
 	  rtx set_label, clr_label;
+	  int setclr_prob = prob;
 
 	  /* Strip narrowing integral type conversions.  */
 	  while (CONVERT_EXPR_P (exp0)
@@ -486,6 +518,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
 	      exp0 = TREE_OPERAND (exp0, 0);
 	      clr_label = if_true_label;
 	      set_label = if_false_label;
+	      setclr_prob = inv (prob);
 	    }
 	  else
 	    {
@@ -508,7 +541,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
 		    = (unsigned HOST_WIDE_INT) 1 << TREE_INT_CST_LOW (shift);
 		  do_jump (build2 (BIT_AND_EXPR, argtype, arg,
 				   build_int_cst_wide_type (argtype, mask, 0)),
-			   clr_label, set_label);
+			   clr_label, set_label, setclr_prob);
 		  break;
 		}
 	    }
@@ -530,7 +563,8 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
           && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (exp))
           && have_insn_for (COMPARE, TYPE_MODE (type)))
         {
-          do_jump (fold_convert (type, exp), if_false_label, if_true_label);
+	  do_jump (fold_convert (type, exp), if_false_label, if_true_label,
+		   prob);
           break;
         }
 
@@ -556,7 +590,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
       /* High branch cost, expand as the bitwise OR of the conditions.
 	 Do the same if the RHS has side effects, because we're effectively
 	 turning a TRUTH_OR_EXPR into a TRUTH_ORIF_EXPR.  */
-      if (BRANCH_COST (optimize_insn_for_speed_p (), false)>= 4
+      if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 4
 	  || TREE_SIDE_EFFECTS (TREE_OPERAND (exp, 1)))
 	goto normal;
       code = TRUTH_ORIF_EXPR;
@@ -580,7 +614,7 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
       do_compare_rtx_and_jump (temp, CONST0_RTX (GET_MODE (temp)),
 			       NE, TYPE_UNSIGNED (TREE_TYPE (exp)),
 			       GET_MODE (temp), NULL_RTX,
-			       if_false_label, if_true_label);
+			       if_false_label, if_true_label, prob);
     }
 
   if (drop_through_label)
@@ -596,7 +630,8 @@ do_jump (tree exp, rtx if_false_label, rtx if_true_label)
 
 static void
 do_jump_by_parts_greater_rtx (enum machine_mode mode, int unsignedp, rtx op0,
-			      rtx op1, rtx if_false_label, rtx if_true_label)
+			      rtx op1, rtx if_false_label, rtx if_true_label,
+			      int prob)
 {
   int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
   rtx drop_through_label = 0;
@@ -628,11 +663,12 @@ do_jump_by_parts_greater_rtx (enum machine_mode mode, int unsignedp, rtx op0,
       /* All but high-order word must be compared as unsigned.  */
       do_compare_rtx_and_jump (op0_word, op1_word, GT,
                                (unsignedp || i > 0), word_mode, NULL_RTX,
-                               NULL_RTX, if_true_label);
+			       NULL_RTX, if_true_label, prob);
 
       /* Consider lower words only if these are equal.  */
       do_compare_rtx_and_jump (op0_word, op1_word, NE, unsignedp, word_mode,
-                               NULL_RTX, NULL_RTX, if_false_label);
+			       NULL_RTX, NULL_RTX, if_false_label,
+			       inv (prob));
     }
 
   if (if_false_label)
@@ -648,7 +684,7 @@ do_jump_by_parts_greater_rtx (enum machine_mode mode, int unsignedp, rtx op0,
 
 static void
 do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap,
-			  rtx if_false_label, rtx if_true_label)
+			  rtx if_false_label, rtx if_true_label, int prob)
 {
   rtx op0 = expand_normal (swap ? treeop1 : treeop0);
   rtx op1 = expand_normal (swap ? treeop0 : treeop1);
@@ -656,7 +692,7 @@ do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap,
   int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
 
   do_jump_by_parts_greater_rtx (mode, unsignedp, op0, op1, if_false_label,
-				if_true_label);
+				if_true_label, prob);
 }
 
 /* Jump according to whether OP0 is 0.  We assume that OP0 has an integer
@@ -666,7 +702,7 @@ do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap,
 
 static void
 do_jump_by_parts_zero_rtx (enum machine_mode mode, rtx op0,
-			   rtx if_false_label, rtx if_true_label)
+			   rtx if_false_label, rtx if_true_label, int prob)
 {
   int nwords = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
   rtx part;
@@ -688,8 +724,7 @@ do_jump_by_parts_zero_rtx (enum machine_mode mode, rtx op0,
   if (part != 0)
     {
       do_compare_rtx_and_jump (part, const0_rtx, EQ, 1, word_mode,
-                               NULL_RTX, if_false_label, if_true_label);
-
+			       NULL_RTX, if_false_label, if_true_label, prob);
       return;
     }
 
@@ -700,7 +735,7 @@ do_jump_by_parts_zero_rtx (enum machine_mode mode, rtx op0,
   for (i = 0; i < nwords; i++)
     do_compare_rtx_and_jump (operand_subword_force (op0, i, mode),
                              const0_rtx, EQ, 1, word_mode, NULL_RTX,
-                             if_false_label, NULL_RTX);
+			     if_false_label, NULL_RTX, prob);
 
   if (if_true_label)
     emit_jump (if_true_label);
@@ -716,7 +751,7 @@ do_jump_by_parts_zero_rtx (enum machine_mode mode, rtx op0,
 
 static void
 do_jump_by_parts_equality_rtx (enum machine_mode mode, rtx op0, rtx op1,
-			       rtx if_false_label, rtx if_true_label)
+			       rtx if_false_label, rtx if_true_label, int prob)
 {
   int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
   rtx drop_through_label = 0;
@@ -724,12 +759,14 @@ do_jump_by_parts_equality_rtx (enum machine_mode mode, rtx op0, rtx op1,
 
   if (op1 == const0_rtx)
     {
-      do_jump_by_parts_zero_rtx (mode, op0, if_false_label, if_true_label);
+      do_jump_by_parts_zero_rtx (mode, op0, if_false_label, if_true_label,
+				 prob);
       return;
     }
   else if (op0 == const0_rtx)
     {
-      do_jump_by_parts_zero_rtx (mode, op1, if_false_label, if_true_label);
+      do_jump_by_parts_zero_rtx (mode, op1, if_false_label, if_true_label,
+				 prob);
       return;
     }
 
@@ -740,7 +777,7 @@ do_jump_by_parts_equality_rtx (enum machine_mode mode, rtx op0, rtx op1,
     do_compare_rtx_and_jump (operand_subword_force (op0, i, mode),
                              operand_subword_force (op1, i, mode),
                              EQ, 0, word_mode, NULL_RTX,
-			     if_false_label, NULL_RTX);
+			     if_false_label, NULL_RTX, prob);
 
   if (if_true_label)
     emit_jump (if_true_label);
@@ -753,13 +790,13 @@ do_jump_by_parts_equality_rtx (enum machine_mode mode, rtx op0, rtx op1,
 
 static void
 do_jump_by_parts_equality (tree treeop0, tree treeop1, rtx if_false_label,
-			   rtx if_true_label)
+			   rtx if_true_label, int prob)
 {
   rtx op0 = expand_normal (treeop0);
   rtx op1 = expand_normal (treeop1);
   enum machine_mode mode = TYPE_MODE (TREE_TYPE (treeop0));
   do_jump_by_parts_equality_rtx (mode, op0, op1, if_false_label,
-				 if_true_label);
+				 if_true_label, prob);
 }
 
 /* Split a comparison into two others, the second of which has the other
@@ -849,10 +886,11 @@ split_comparison (enum rtx_code code, enum machine_mode mode,
 void
 do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
 			 enum machine_mode mode, rtx size, rtx if_false_label,
-			 rtx if_true_label)
+			 rtx if_true_label, int prob)
 {
   rtx tem;
   rtx dummy_label = NULL_RTX;
+  rtx last;
 
   /* Reverse the comparison if that is safe and we want to jump if it is
      false.  Also convert to the reverse comparison if the target can
@@ -878,6 +916,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
           if_true_label = if_false_label;
           if_false_label = tem;
 	  code = rcode;
+	  prob = inv (prob);
 	}
     }
 
@@ -924,52 +963,56 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
 	{
 	case LTU:
 	  do_jump_by_parts_greater_rtx (mode, 1, op1, op0,
-					if_false_label, if_true_label);
+					if_false_label, if_true_label, prob);
 	  break;
 
 	case LEU:
 	  do_jump_by_parts_greater_rtx (mode, 1, op0, op1,
-					if_true_label, if_false_label);
+					if_true_label, if_false_label,
+					inv (prob));
 	  break;
 
 	case GTU:
 	  do_jump_by_parts_greater_rtx (mode, 1, op0, op1,
-					if_false_label, if_true_label);
+					if_false_label, if_true_label, prob);
 	  break;
 
 	case GEU:
 	  do_jump_by_parts_greater_rtx (mode, 1, op1, op0,
-					if_true_label, if_false_label);
+					if_true_label, if_false_label,
+					inv (prob));
 	  break;
 
 	case LT:
 	  do_jump_by_parts_greater_rtx (mode, 0, op1, op0,
-					if_false_label, if_true_label);
+					if_false_label, if_true_label, prob);
 	  break;
 
 	case LE:
 	  do_jump_by_parts_greater_rtx (mode, 0, op0, op1,
-					if_true_label, if_false_label);
+					if_true_label, if_false_label,
+					inv (prob));
 	  break;
 
 	case GT:
 	  do_jump_by_parts_greater_rtx (mode, 0, op0, op1,
-					if_false_label, if_true_label);
+					if_false_label, if_true_label, prob);
 	  break;
 
 	case GE:
 	  do_jump_by_parts_greater_rtx (mode, 0, op1, op0,
-					if_true_label, if_false_label);
+					if_true_label, if_false_label,
+					inv (prob));
 	  break;
 
 	case EQ:
 	  do_jump_by_parts_equality_rtx (mode, op0, op1, if_false_label,
-					 if_true_label);
+					 if_true_label, prob);
 	  break;
 
 	case NE:
 	  do_jump_by_parts_equality_rtx (mode, op0, op1, if_true_label,
-					 if_false_label);
+					 if_false_label, inv (prob));
 	  break;
 
 	default:
@@ -1025,16 +1068,38 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
 		  else
 		    dest_label = if_false_label;
                   do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode,
-					   size, dest_label, NULL_RTX);
+					   size, dest_label, NULL_RTX, prob);
 		}
               else
                 do_compare_rtx_and_jump (op0, op1, first_code, unsignedp, mode,
-					 size, NULL_RTX, if_true_label);
+					 size, NULL_RTX, if_true_label, prob);
 	    }
 	}
 
+      last = get_last_insn ();
       emit_cmp_and_jump_insns (op0, op1, code, size, mode, unsignedp,
 			       if_true_label);
+      if (prob != -1 && profile_status != PROFILE_ABSENT)
+	{
+	  for (last = NEXT_INSN (last);
+	       last && NEXT_INSN (last);
+	       last = NEXT_INSN (last))
+	    if (JUMP_P (last))
+	      break;
+	  if (!last
+	      || !JUMP_P (last)
+	      || NEXT_INSN (last)
+	      || !any_condjump_p (last))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "Failed to add probability note\n");
+	    }
+	  else
+	    {
+	      gcc_assert (!find_reg_note (last, REG_BR_PROB, 0));
+	      add_reg_note (last, REG_BR_PROB, GEN_INT (prob));
+	    }
+	}
     }
 
   if (if_false_label)
@@ -1056,7 +1121,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
 static void
 do_compare_and_jump (tree treeop0, tree treeop1, enum rtx_code signed_code,
 		     enum rtx_code unsigned_code, rtx if_false_label,
-		     rtx if_true_label)
+		     rtx if_true_label, int prob)
 {
   rtx op0, op1;
   tree type;
@@ -1116,7 +1181,7 @@ do_compare_and_jump (tree treeop0, tree treeop1, enum rtx_code signed_code,
   do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode,
                            ((mode == BLKmode)
                             ? expr_size (treeop0) : NULL_RTX),
-                           if_false_label, if_true_label);
+			   if_false_label, if_true_label, prob);
 }
 
 #include "gt-dojump.h"