diff options
84 files changed, 1721 insertions, 811 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index de228dd1fa5..2f339ccb3b1 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,210 @@ +2017-06-29 Jan Hubicka <hubicka@ucw.cz> + + * asan.c (asan_emit_stack_protection): Update. + (create_cond_insert_point): Update. + * auto-profile.c (afdo_propagate_circuit): Update. + * basic-block.h (struct edge_def): Turn probability to + profile_probability. + (EDGE_FREQUENCY): Update. + * bb-reorder.c (find_traces_1_round): Update. + (better_edge_p): Update. + (sanitize_hot_paths): Update. + * cfg.c (unchecked_make_edge): Initialize probability to uninitialized. + (make_single_succ_edge): Update. + (check_bb_profile): Update. + (dump_edge_info): Update. + (update_bb_profile_for_threading): Update. + * cfganal.c (connect_infinite_loops_to_exit): Initialize new edge + probabilitycount to 0. + * cfgbuild.c (compute_outgoing_frequencies): Update. + * cfgcleanup.c (try_forward_edges): Update. + (outgoing_edges_match): Update. + (try_crossjump_to_edge): Update. + * cfgexpand.c (expand_gimple_cond): Update make_single_succ_edge. + (expand_gimple_tailcall): Update. + (construct_init_block): Use make_single_succ_edge. + (construct_exit_block): Use make_single_succ_edge. + * cfghooks.c (verify_flow_info): Update. + (redirect_edge_succ_nodup): Update. + (split_edge): Update. + (account_profile_record): Update. + * cfgloopanal.c (single_likely_exit): Update. + * cfgloopmanip.c (scale_loop_profile): Update. + (set_zero_probability): Remove. + (duplicate_loop_to_header_edge): Update. + * cfgloopmanip.h (loop_version): Update prototype. + * cfgrtl.c (try_redirect_by_replacing_jump): Update. + (force_nonfallthru_and_redirect): Update. + (update_br_prob_note): Update. + (rtl_verify_edges): Update. + (purge_dead_edges): Update. + (rtl_lv_add_condition_to_bb): Update. + * cgraph.c: (cgraph_edge::redirect_call_stmt_to_calle): Update. + * cgraphunit.c (init_lowered_empty_function): Update. + (cgraph_node::expand_thunk): Update. + * cilk-common.c: Include profile-count.h + * dojump.c (inv): Remove. + (jumpifnot): Update. + (jumpifnot_1): Update. + (do_jump_1): Update. + (do_jump): Update. + (do_jump_by_parts_greater_rtx): Update. + (do_compare_rtx_and_jump): Update. + * dojump.h (jumpifnot, jumpifnot_1, jumpif_1, jumpif, do_jump, + do_jump_1. do_compare_rtx_and_jump): Update prototype. + * dwarf2cfi.c: Include profile-count.h + * except.c (dw2_build_landing_pads): Use make_single_succ_edge. + (sjlj_emit_dispatch_table): Likewise. + * explow.c: Include profile-count.h + * expmed.c (emit_store_flag_force): Update. + (do_cmp_and_jump): Update. + * expr.c (compare_by_pieces_d::generate): Update. + (compare_by_pieces_d::finish_mode): Update. + (emit_block_move_via_loop): Update. + (store_expr_with_bounds): Update. + (store_constructor): Update. + (expand_expr_real_2): Update. + (expand_expr_real_1): Update. + * expr.h (try_casesi, try_tablejump): Update prototypes. + * gimple-pretty-print.c (dump_probability): Update. + (dump_profile): New. + (dump_gimple_label): Update. + (dump_gimple_bb_header): Update. + * graph.c (draw_cfg_node_succ_edges): Update. + * hsa-gen.c (convert_switch_statements): Update. + * ifcvt.c (cheap_bb_rtx_cost_p): Update. + (find_if_case_1): Update. + (find_if_case_2): Update. + * internal-fn.c (expand_arith_overflow_result_store): Update. + (expand_addsub_overflow): Update. + (expand_neg_overflow): Update. + (expand_mul_overflow): Update. + (expand_vector_ubsan_overflow): Update. + * ipa-cp.c (good_cloning_opportunity_p): Update. + * ipa-split.c (split_function): Use make_single_succ_edge. + * ipa-utils.c (ipa_merge_profiles): Update. + * loop-doloop.c (add_test): Update. + (doloop_modify): Update. + * loop-unroll.c (compare_and_jump_seq): Update. + (unroll_loop_runtime_iterations): Update. + * lra-constraints.c (lra_inheritance): Update. + * lto-streamer-in.c (input_cfg): Update. + * lto-streamer-out.c (output_cfg): Update. + * mcf.c (adjust_cfg_counts): Update. + * modulo-sched.c (sms_schedule): Update. + * omp-expand.c (expand_omp_for_init_counts): Update. + (extract_omp_for_update_vars): Update. + (expand_omp_ordered_sink): Update. + (expand_omp_for_ordered_loops): Update. + (expand_omp_for_generic): Update. + (expand_omp_for_static_nochunk): Update. + (expand_omp_for_static_chunk): Update. + (expand_cilk_for): Update. + (expand_omp_simd): Update. + (expand_omp_taskloop_for_outer): Update. + (expand_omp_taskloop_for_inner): Update. + * omp-simd-clone.c (simd_clone_adjust): Update. + * optabs.c (expand_doubleword_shift): Update. + (expand_abs): Update. + (emit_cmp_and_jump_insn_1): Update. + (expand_compare_and_swap_loop): Update. + * optabs.h (emit_cmp_and_jump_insns): Update prototype. + * predict.c (predictable_edge_p): Update. + (edge_probability_reliable_p): Update. + (set_even_probabilities): Update. + (combine_predictions_for_insn): Update. + (combine_predictions_for_bb): Update. + (propagate_freq): Update. + (estimate_bb_frequencies): Update. + (force_edge_cold): Update. + * profile-count.c (profile_count::dump): Add missing space into dump. + (profile_count::debug): Add newline. + (profile_count::differs_from_p): Explicitly convert to unsigned. + (profile_count::stream_in): Update. + (profile_probability::dump): New member function. + (profile_probability::debug): New member function. + (profile_probability::differs_from_p): New member function. + (profile_probability::differs_lot_from_p): New member function. + (profile_probability::stream_in): New member function. + (profile_probability::stream_out): New member function. + * profile-count.h (profile_count_quality): Rename to ... + (profile_quality): ... this one. + (profile_probability): New. + (profile_count): Update. + * profile.c (compute_branch_probabilities): Update. + * recog.c (peep2_attempt): Update. + * sched-ebb.c (schedule_ebbs): Update. + * sched-rgn.c (find_single_block_region): Update. + (compute_dom_prob_ps): Update. + (schedule_region): Update. + * sel-sched-ir.c (compute_succs_info): Update. + * stmt.c (struct case_node): Update. + (do_jump_if_equal): Update. + (get_outgoing_edge_probs): Update. + (conditional_probability): Update. + (emit_case_dispatch_table): Update. + (expand_case): Update. + (expand_sjlj_dispatch_table): Update. + (emit_case_nodes): Update. + * targhooks.c: Update. + * tracer.c (better_p): Update. + (find_best_successor): Update. + * trans-mem.c (expand_transaction): Update. + * tree-call-cdce.c: Update. + * tree-cfg.c (gimple_split_edge): Upate. + (move_sese_region_to_fn): Upate. + * tree-cfgcleanup.c (cleanup_control_expr_graph): Upate. + * tree-eh.c (lower_resx): Upate. + (cleanup_empty_eh_move_lp): Upate. + * tree-if-conv.c (version_loop_for_if_conversion): Update. + * tree-inline.c (copy_edges_for_bb): Update. + (copy_cfg_body): Update. + * tree-parloops.c (gen_parallel_loop): Update. + * tree-profile.c (gimple_gen_ic_func_profiler): Update. + (gimple_gen_time_profiler): Update. + * tree-ssa-dce.c (remove_dead_stmt): Update. + * tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update. + * tree-ssa-loop-im.c (execute_sm_if_changed): Update. + * tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update. + (unloop_loops): Update. + (try_peel_loop): Update. + * tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update. + * tree-ssa-loop-split.c (connect_loops): Update. + (split_loop): Update. + * tree-ssa-loop-unswitch.c (tree_unswitch_loop): Update. + (hoist_guard): Update. + * tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update. + * tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update. + (value_replacement): Update. + * tree-ssa-reassoc.c (branch_fixup): Update. + * tree-ssa-tail-merge.c (replace_block_by): Update. + * tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update. + (create_edge_and_update_destination_phis): Update. + (compute_path_counts): Update. + (recompute_probabilities): Update. + (update_joiner_offpath_counts): Update. + (freqs_to_counts_path): Update. + (duplicate_thread_path): Update. + * tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update. + (struct switch_conv_info): Update. + (gen_inbound_check): Update. + * tree-vect-loop-manip.c (slpeel_add_loop_guard): Update. + (vect_do_peeling): Update. + (vect_loop_versioning): Update. + * tree-vect-loop.c (scale_profile_for_vect_loop): Update. + (optimize_mask_stores): Update. + * ubsan.c (ubsan_expand_null_ifn): Update. + * value-prof.c (gimple_divmod_fixed_value): Update. + (gimple_divmod_fixed_value_transform): Update. + (gimple_mod_pow2): Update. + (gimple_mod_pow2_value_transform): Update. + (gimple_mod_subtract): Update. + (gimple_mod_subtract_transform): Update. + (gimple_ic): Update. + (gimple_stringop_fixed_value): Update. + (gimple_stringops_transform): Update. + * value-prof.h: Update. + 2017-06-29 Carl Love <cel@us.ibm.com> * config/rs6000/rs6000-c.c: Add support for built-in functions diff --git a/gcc/asan.c b/gcc/asan.c index 3f814819add..2de16402c51 100644 --- a/gcc/asan.c +++ b/gcc/asan.c @@ -1145,9 +1145,9 @@ asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb, emit_move_insn (orig_base, base); ret = expand_normal (asan_detect_stack_use_after_return); lab = gen_label_rtx (); - int very_likely = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX, - VOIDmode, 0, lab, very_likely); + VOIDmode, 0, lab, + profile_probability::very_likely ()); snprintf (buf, sizeof buf, "__asan_stack_malloc_%d", use_after_return_class); ret = init_one_libfunc (buf); @@ -1158,9 +1158,9 @@ asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb, /* __asan_stack_malloc_[n] returns a pointer to fake stack if succeeded and NULL otherwise. Check RET value is NULL here and jump over the BASE reassignment in this case. Otherwise, reassign BASE to RET. */ - int very_unlikely = REG_BR_PROB_BASE / 2000 - 1; emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX, - VOIDmode, 0, lab, very_unlikely); + VOIDmode, 0, lab, + profile_probability:: very_unlikely ()); ret = convert_memory_address (Pmode, ret); emit_move_insn (base, ret); emit_label (lab); @@ -1255,9 +1255,9 @@ asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb, { rtx_code_label *lab2 = gen_label_rtx (); char c = (char) ASAN_STACK_MAGIC_USE_AFTER_RET; - int very_likely = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); emit_cmp_and_jump_insns (orig_base, base, EQ, NULL_RTX, - VOIDmode, 0, lab2, very_likely); + VOIDmode, 0, lab2, + profile_probability::very_likely ()); shadow_mem = gen_rtx_MEM (BLKmode, shadow_base); set_mem_alias_set (shadow_mem, asan_shadow_set); mem = gen_rtx_MEM (ptr_mode, base); @@ -1588,7 +1588,8 @@ create_cond_insert_point (gimple_stmt_iterator *iter, = then_more_likely_p ? PROB_VERY_UNLIKELY : PROB_ALWAYS - PROB_VERY_UNLIKELY; - e->probability = PROB_ALWAYS - fallthrough_probability; + e->probability = profile_probability::from_reg_br_prob_base + (PROB_ALWAYS - fallthrough_probability); if (create_then_fallthru_edge) make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU); @@ -1596,7 +1597,8 @@ create_cond_insert_point (gimple_stmt_iterator *iter, e = find_edge (cond_bb, fallthru_bb); e->flags = EDGE_FALSE_VALUE; e->count = cond_bb->count; - e->probability = fallthrough_probability; + e->probability + = profile_probability::from_reg_br_prob_base (fallthrough_probability); /* Update dominance info for the newly created then_bb; note that fallthru_bb's dominance info has already been updated by diff --git a/gcc/auto-profile.c b/gcc/auto-profile.c index 973d7af37ff..521f4c0d3eb 100644 --- a/gcc/auto-profile.c +++ b/gcc/auto-profile.c @@ -1346,9 +1346,10 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge) continue; total++; only_one = ep; - if (e->probability == 0 && !is_edge_annotated (ep, *annotated_edge)) + if (!e->probability.initialized_p () + && !is_edge_annotated (ep, *annotated_edge)) { - ep->probability = 0; + ep->probability = profile_probability::never (); ep->count = profile_count::zero (); set_edge_annotated (ep, annotated_edge); } diff --git a/gcc/basic-block.h b/gcc/basic-block.h index b08a14a151b..c0c47784c02 100644 --- a/gcc/basic-block.h +++ b/gcc/basic-block.h @@ -45,7 +45,7 @@ struct GTY((user)) edge_def { unsigned int dest_idx; int flags; /* see cfg-flags.def */ - int probability; /* biased by REG_BR_PROB_BASE */ + profile_probability probability; profile_count count; /* Expected number of executions calculated in profile.c */ }; @@ -300,8 +300,7 @@ enum cfg_bb_flags ? EDGE_SUCC ((bb), 1) : EDGE_SUCC ((bb), 0)) /* Return expected execution frequency of the edge E. */ -#define EDGE_FREQUENCY(e) RDIV ((e)->src->frequency * (e)->probability, \ - REG_BR_PROB_BASE) +#define EDGE_FREQUENCY(e) e->probability.apply (e->src->frequency) /* Compute a scale factor (or probability) suitable for scaling of gcov_type values via apply_probability() and apply_scale(). */ diff --git a/gcc/bb-reorder.c b/gcc/bb-reorder.c index c0386f4b37a..276ab589427 100644 --- a/gcc/bb-reorder.c +++ b/gcc/bb-reorder.c @@ -206,8 +206,8 @@ static void find_traces_1_round (int, int, gcov_type, struct trace *, int *, int, bb_heap_t **, int); static basic_block copy_bb (basic_block, edge, basic_block, int); static long bb_to_key (basic_block); -static bool better_edge_p (const_basic_block, const_edge, int, int, int, int, - const_edge); +static bool better_edge_p (const_basic_block, const_edge, profile_probability, + int, profile_probability, int, const_edge); static bool connect_better_edge_p (const_edge, bool, int, const_edge, struct trace *); static void connect_traces (int, struct trace *); @@ -513,11 +513,12 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th, do { - int prob, freq; + profile_probability prob; + int freq; bool ends_in_call; /* The probability and frequency of the best edge. */ - int best_prob = INT_MIN / 2; + profile_probability best_prob = profile_probability::uninitialized (); int best_freq = INT_MIN / 2; best_edge = NULL; @@ -565,7 +566,9 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th, successor (i.e. it is unsuitable successor). When optimizing for size, ignore the probability and frequency. */ if (!(e->flags & EDGE_CAN_FALLTHRU) || (e->flags & EDGE_COMPLEX) - || ((prob < branch_th || EDGE_FREQUENCY (e) < exec_th + || !prob.initialized_p () + || ((prob.to_reg_br_prob_base () < branch_th + || EDGE_FREQUENCY (e) < exec_th || e->count < count_th) && (!for_size))) continue; @@ -648,7 +651,9 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th, if (!(e->flags & EDGE_CAN_FALLTHRU) || (e->flags & EDGE_COMPLEX) - || prob < branch_th || freq < exec_th + || !prob.initialized_p () + || prob.to_reg_br_prob_base () < branch_th + || freq < exec_th || e->count < count_th) { /* When partitioning hot/cold basic blocks, make sure @@ -936,14 +941,15 @@ bb_to_key (basic_block bb) BEST_PROB; similarly for frequency. */ static bool -better_edge_p (const_basic_block bb, const_edge e, int prob, int freq, - int best_prob, int best_freq, const_edge cur_best_edge) +better_edge_p (const_basic_block bb, const_edge e, profile_probability prob, + int freq, profile_probability best_prob, int best_freq, + const_edge cur_best_edge) { bool is_better_edge; /* The BEST_* values do not have to be best, but can be a bit smaller than maximum values. */ - int diff_prob = best_prob / 10; + profile_probability diff_prob = best_prob.apply_scale (1, 10); int diff_freq = best_freq / 10; /* The smaller one is better to keep the original order. */ @@ -1494,7 +1500,8 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count, vec<edge, va_gc> *edges = walk_up ? bb->preds : bb->succs; edge e; edge_iterator ei; - int highest_probability = 0; + profile_probability highest_probability + = profile_probability::uninitialized (); int highest_freq = 0; profile_count highest_count = profile_count::uninitialized (); bool found = false; @@ -1517,12 +1524,13 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count, /* The following loop will look for the hottest edge via the edge count, if it is non-zero, then fallback to the edge frequency and finally the edge probability. */ - if (e->count > highest_count) + if (!highest_count.initialized_p () || e->count > highest_count) highest_count = e->count; int edge_freq = EDGE_FREQUENCY (e); if (edge_freq > highest_freq) highest_freq = edge_freq; - if (e->probability > highest_probability) + if (!highest_probability.initialized_p () + || e->probability > highest_probability) highest_probability = e->probability; } diff --git a/gcc/cfg.c b/gcc/cfg.c index 3cbe684b743..5d31185c892 100644 --- a/gcc/cfg.c +++ b/gcc/cfg.c @@ -264,6 +264,7 @@ unchecked_make_edge (basic_block src, basic_block dst, int flags) n_edges_for_fn (cfun)++; e->count = profile_count::uninitialized (); + e->probability = profile_probability::uninitialized (); e->src = src; e->dest = dst; e->flags = flags; @@ -332,7 +333,7 @@ make_single_succ_edge (basic_block src, basic_block dest, int flags) { edge e = make_edge (src, dest, flags); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = src->count; return e; } @@ -400,7 +401,6 @@ static void check_bb_profile (basic_block bb, FILE * file, int indent) { edge e; - int sum = 0; edge_iterator ei; struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl); char *s_indent = (char *) alloca ((size_t) indent + 1); @@ -413,21 +413,38 @@ check_bb_profile (basic_block bb, FILE * file, int indent) if (bb != EXIT_BLOCK_PTR_FOR_FN (fun)) { bool found = false; + profile_probability sum = profile_probability::never (); + int isum = 0; + FOR_EACH_EDGE (e, ei, bb->succs) { - if (!(e->flags & EDGE_EH)) + if (!(e->flags & (EDGE_EH | EDGE_FAKE))) found = true; sum += e->probability; + if (e->probability.initialized_p ()) + isum += e->probability.to_reg_br_prob_base (); } /* Only report mismatches for non-EH control flow. If there are only EH edges it means that the BB ends by noreturn call. Here the control flow may just terminate. */ if (found) { - if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100) - fprintf (file, - ";; %sInvalid sum of outgoing probabilities %.1f%%\n", - s_indent, sum * 100.0 / REG_BR_PROB_BASE); + if (sum.differs_from_p (profile_probability::always ())) + { + fprintf (file, + ";; %sInvalid sum of outgoing probabilities ", + s_indent); + sum.dump (file); + fprintf (file, "\n"); + } + /* Probabilities caps to 100% and thus the previous test will never + fire if the sum of probabilities is too large. */ + else if (isum > REG_BR_PROB_BASE + 100) + { + fprintf (file, + ";; %sInvalid sum of outgoing probabilities %.1f%%\n", + s_indent, isum * 100.0 / REG_BR_PROB_BASE); + } profile_count lsum = profile_count::zero (); FOR_EACH_EDGE (e, ei, bb->succs) lsum += e->count; @@ -442,9 +459,9 @@ check_bb_profile (basic_block bb, FILE * file, int indent) } } } - if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun)) + if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun)) { - sum = 0; + int sum = 0; FOR_EACH_EDGE (e, ei, bb->preds) sum += EDGE_FREQUENCY (e); if (abs (sum - bb->frequency) > 100) @@ -498,8 +515,12 @@ dump_edge_info (FILE *file, edge e, dump_flags_t flags, int do_succ) else fprintf (file, " %d", side->index); - if (e->probability && do_details) - fprintf (file, " [%.1f%%] ", e->probability * 100.0 / REG_BR_PROB_BASE); + if (e->probability.initialized_p () && do_details) + { + fprintf (file, " ["); + e->probability.dump (file); + fprintf (file, "] "); + } if (e->count.initialized_p () && do_details) { @@ -860,7 +881,7 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency, profile_count count, edge taken_edge) { edge c; - int prob; + profile_probability prob; edge_iterator ei; if (bb->count < count) @@ -878,59 +899,51 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency, /* Compute the probability of TAKEN_EDGE being reached via threaded edge. Watch for overflows. */ if (bb->frequency) - prob = GCOV_COMPUTE_SCALE (edge_frequency, bb->frequency); + /* FIXME: We should get edge frequency as count. */ + prob = profile_probability::probability_in_gcov_type + (edge_frequency, bb->frequency); else - prob = 0; + prob = profile_probability::never (); if (prob > taken_edge->probability) { if (dump_file) - fprintf (dump_file, "Jump threading proved probability of edge " - "%i->%i too small (it is %i, should be %i).\n", - taken_edge->src->index, taken_edge->dest->index, - taken_edge->probability, prob); - prob = taken_edge->probability * 6 / 8; + { + fprintf (dump_file, "Jump threading proved probability of edge " + "%i->%i too small (it is ", + taken_edge->src->index, taken_edge->dest->index); + taken_edge->probability.dump (dump_file); + fprintf (dump_file, " should be "); + prob.dump (dump_file); + fprintf (dump_file, ")\n"); + } + prob = taken_edge->probability.apply_scale (6, 8); } /* Now rescale the probabilities. */ taken_edge->probability -= prob; - prob = REG_BR_PROB_BASE - prob; - if (prob <= 0) + prob = prob.invert (); + if (prob == profile_probability::never ()) { if (dump_file) fprintf (dump_file, "Edge frequencies of bb %i has been reset, " "frequency of block should end up being 0, it is %i\n", bb->index, bb->frequency); - EDGE_SUCC (bb, 0)->probability = REG_BR_PROB_BASE; + EDGE_SUCC (bb, 0)->probability = profile_probability::guessed_always (); ei = ei_start (bb->succs); ei_next (&ei); for (; (c = ei_safe_edge (ei)); ei_next (&ei)) - c->probability = 0; + c->probability = profile_probability::guessed_never (); } - else if (prob != REG_BR_PROB_BASE) + else if (!(prob == profile_probability::always ())) { - int scale = RDIV (65536 * REG_BR_PROB_BASE, prob); - FOR_EACH_EDGE (c, ei, bb->succs) - { - /* Protect from overflow due to additional scaling. */ - if (c->probability > prob) - c->probability = REG_BR_PROB_BASE; - else - { - c->probability = RDIV (c->probability * scale, 65536); - if (c->probability > REG_BR_PROB_BASE) - c->probability = REG_BR_PROB_BASE; - } - } + c->probability /= prob; } gcc_assert (bb == taken_edge->src); - if (taken_edge->count < count) - { - if (dump_file) - fprintf (dump_file, "edge %i->%i count became negative after threading", - taken_edge->src->index, taken_edge->dest->index); - } + if (dump_file && taken_edge->count < count) + fprintf (dump_file, "edge %i->%i count became negative after threading", + taken_edge->src->index, taken_edge->dest->index); taken_edge->count -= count; } diff --git a/gcc/cfganal.c b/gcc/cfganal.c index a3a6ea86994..792ea62cc56 100644 --- a/gcc/cfganal.c +++ b/gcc/cfganal.c @@ -610,7 +610,10 @@ connect_infinite_loops_to_exit (void) break; basic_block deadend_block = dfs_find_deadend (unvisited_block); - make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE); + edge e = make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun), + EDGE_FAKE); + e->count = profile_count::zero (); + e->probability = profile_probability::never (); dfs.add_bb (deadend_block); } } diff --git a/gcc/cfgbuild.c b/gcc/cfgbuild.c index af73a5e131c..56a2cb92860 100644 --- a/gcc/cfgbuild.c +++ b/gcc/cfgbuild.c @@ -545,10 +545,13 @@ compute_outgoing_frequencies (basic_block b) { probability = XINT (note, 0); e = BRANCH_EDGE (b); - e->probability = probability; + e->probability + = profile_probability::from_reg_br_prob_base (probability); e->count = b->count.apply_probability (probability); f = FALLTHRU_EDGE (b); - f->probability = REG_BR_PROB_BASE - probability; + f->probability + = profile_probability::from_reg_br_prob_base (REG_BR_PROB_BASE + - probability); f->count = b->count - e->count; return; } @@ -560,7 +563,7 @@ compute_outgoing_frequencies (basic_block b) else if (single_succ_p (b)) { e = single_succ_edge (b); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = b->count; return; } @@ -656,7 +659,8 @@ find_many_sub_basic_blocks (sbitmap blocks) } else uninitialized_src = true; - bb->frequency += EDGE_FREQUENCY (e); + if (e->probability.initialized_p ()) + bb->frequency += EDGE_FREQUENCY (e); } /* When some edges are missing with read profile, this is most likely because RTL expansion introduced loop. diff --git a/gcc/cfgcleanup.c b/gcc/cfgcleanup.c index 1201148fa88..5d00fafffa3 100644 --- a/gcc/cfgcleanup.c +++ b/gcc/cfgcleanup.c @@ -559,7 +559,7 @@ try_forward_edges (int mode, basic_block b) { /* Save the values now, as the edge may get removed. */ profile_count edge_count = e->count; - int edge_probability = e->probability; + profile_probability edge_probability = e->probability; int edge_frequency; int n = 0; @@ -585,7 +585,7 @@ try_forward_edges (int mode, basic_block b) /* We successfully forwarded the edge. Now update profile data: for each edge we traversed in the chain, remove the original edge's execution count. */ - edge_frequency = apply_probability (b->frequency, edge_probability); + edge_frequency = edge_probability.apply (b->frequency); do { @@ -1710,24 +1710,28 @@ outgoing_edges_match (int mode, basic_block bb1, basic_block bb2) && optimize_bb_for_speed_p (bb1) && optimize_bb_for_speed_p (bb2)) { - int prob2; + profile_probability prob2; if (b1->dest == b2->dest) prob2 = b2->probability; else /* Do not use f2 probability as f2 may be forwarded. */ - prob2 = REG_BR_PROB_BASE - b2->probability; + prob2 = b2->probability.invert (); /* Fail if the difference in probabilities is greater than 50%. This rules out two well-predicted branches with opposite outcomes. */ - if (abs (b1->probability - prob2) > REG_BR_PROB_BASE / 2) + if (b1->probability.differs_lot_from_p (prob2)) { if (dump_file) - fprintf (dump_file, - "Outcomes of branch in bb %i and %i differ too much (%i %i)\n", - bb1->index, bb2->index, b1->probability, prob2); - + { + fprintf (dump_file, + "Outcomes of branch in bb %i and %i differ too" + " much (", bb1->index, bb2->index); + b1->probability.dump (dump_file); + prob2.dump (dump_file); + fprintf (dump_file, ")\n"); + } return false; } } @@ -2149,12 +2153,9 @@ try_crossjump_to_edge (int mode, edge e1, edge e2, } if (!redirect_edges_to->frequency && !src1->frequency) - s->probability = (s->probability + s2->probability) / 2; - else - s->probability - = ((s->probability * redirect_edges_to->frequency + - s2->probability * src1->frequency) - / (redirect_edges_to->frequency + src1->frequency)); + s->probability = s->probability.combine_with_freq + (redirect_edges_to->frequency, + s2->probability, src1->frequency); } /* Adjust count and frequency for the block. An earlier jump diff --git a/gcc/cfgexpand.c b/gcc/cfgexpand.c index d61c261a172..71662f21350 100644 --- a/gcc/cfgexpand.c +++ b/gcc/cfgexpand.c @@ -2389,7 +2389,6 @@ static basic_block expand_gimple_cond (basic_block bb, gcond *stmt) { basic_block new_bb, dest; - edge new_edge; edge true_edge; edge false_edge; rtx_insn *last2, *last; @@ -2508,9 +2507,7 @@ expand_gimple_cond (basic_block bb, gcond *stmt) if (loop->latch == bb && loop->header == dest) loop->latch = new_bb; - new_edge = make_edge (new_bb, dest, 0); - new_edge->probability = REG_BR_PROB_BASE; - new_edge->count = new_bb->count; + make_single_succ_edge (new_bb, dest, 0); if (BARRIER_P (BB_END (new_bb))) BB_END (new_bb) = PREV_INSN (BB_END (new_bb)); update_bb_for_insn (new_bb); @@ -3788,7 +3785,7 @@ expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru) rtx_insn *last2, *last; edge e; edge_iterator ei; - int probability; + profile_probability probability; last2 = last = expand_gimple_stmt (stmt); @@ -3813,7 +3810,7 @@ expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru) all edges here, or redirecting the existing fallthru edge to the exit block. */ - probability = 0; + probability = profile_probability::never (); profile_count count = profile_count::zero (); for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); ) @@ -5839,12 +5836,11 @@ construct_init_block (void) { first_block = e->dest; redirect_edge_succ (e, init_block); - e = make_edge (init_block, first_block, flags); + e = make_single_succ_edge (init_block, first_block, flags); } else - e = make_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FALLTHRU); - e->probability = REG_BR_PROB_BASE; - e->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count; + e = make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun), + EDGE_FALLTHRU); update_bb_for_insn (init_block); return init_block; @@ -5924,9 +5920,8 @@ construct_exit_block (void) ix++; } - e = make_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FALLTHRU); - e->probability = REG_BR_PROB_BASE; - e->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count; + e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun), + EDGE_FALLTHRU); FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds) if (e2 != e) { diff --git a/gcc/cfghooks.c b/gcc/cfghooks.c index bcda422d89a..18dc49a035e 100644 --- a/gcc/cfghooks.c +++ b/gcc/cfghooks.c @@ -160,10 +160,10 @@ verify_flow_info (void) e->src->index, e->dest->index); err = 1; } - if (e->probability < 0 || e->probability > REG_BR_PROB_BASE) + if (!e->probability.verify ()) { - error ("verify_flow_info: Wrong probability of edge %i->%i %i", - e->src->index, e->dest->index, e->probability); + error ("verify_flow_info: Wrong probability of edge %i->%i", + e->src->index, e->dest->index); err = 1; } if (!e->count.verify ()) @@ -443,8 +443,6 @@ redirect_edge_succ_nodup (edge e, basic_block new_succ) { s->flags |= e->flags; s->probability += e->probability; - if (s->probability > REG_BR_PROB_BASE) - s->probability = REG_BR_PROB_BASE; s->count += e->count; /* FIXME: This should be called via a hook and only for IR_GIMPLE. */ redirect_edge_var_map_dup (s, e); @@ -640,7 +638,7 @@ split_edge (edge e) ret = cfg_hooks->split_edge (e); ret->count = count; ret->frequency = freq; - single_succ_edge (ret)->probability = REG_BR_PROB_BASE; + single_succ_edge (ret)->probability = profile_probability::always (); single_succ_edge (ret)->count = count; if (irr) @@ -1451,17 +1449,17 @@ account_profile_record (struct profile_record *record, int after_pass) basic_block bb; edge_iterator ei; edge e; - int sum; FOR_ALL_BB_FN (bb, cfun) { if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun) && profile_status_for_fn (cfun) != PROFILE_ABSENT) { - sum = 0; + profile_probability sum = profile_probability::never (); FOR_EACH_EDGE (e, ei, bb->succs) sum += e->probability; - if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100) + if (EDGE_COUNT (bb->succs) + && sum.differs_from_p (profile_probability::always ())) record->num_mismatched_freq_out[after_pass]++; profile_count lsum = profile_count::zero (); FOR_EACH_EDGE (e, ei, bb->succs) @@ -1472,7 +1470,7 @@ account_profile_record (struct profile_record *record, int after_pass) if (bb != ENTRY_BLOCK_PTR_FOR_FN (cfun) && profile_status_for_fn (cfun) != PROFILE_ABSENT) { - sum = 0; + int sum = 0; FOR_EACH_EDGE (e, ei, bb->preds) sum += EDGE_FREQUENCY (e); if (abs (sum - bb->frequency) > 100 diff --git a/gcc/cfgloopanal.c b/gcc/cfgloopanal.c index a9537637115..62e1c0084b8 100644 --- a/gcc/cfgloopanal.c +++ b/gcc/cfgloopanal.c @@ -477,7 +477,7 @@ single_likely_exit (struct loop *loop) reasons. FIXME: Turn to likely_never_executed */ if ((profile_status_for_fn (cfun) != PROFILE_ABSENT - && ex->probability < 5) + && ex->probability < profile_probability::from_reg_br_prob_base (5)) || ex->count == profile_count::zero ()) continue; if (!found) diff --git a/gcc/cfgloopmanip.c b/gcc/cfgloopmanip.c index d764ab9a8c7..f319026ae16 100644 --- a/gcc/cfgloopmanip.c +++ b/gcc/cfgloopmanip.c @@ -542,8 +542,9 @@ scale_loop_profile (struct loop *loop, int scale, gcov_type iteration_bound) /* Probability of exit must be 1/iterations. */ freq_delta = EDGE_FREQUENCY (e); - e->probability = REG_BR_PROB_BASE / iteration_bound; - other_e->probability = inverse_probability (e->probability); + e->probability = profile_probability::from_reg_br_prob_base + (REG_BR_PROB_BASE / iteration_bound); + other_e->probability = e->probability.invert (); freq_delta -= EDGE_FREQUENCY (e); /* Adjust counts accordingly. */ @@ -1101,43 +1102,6 @@ can_duplicate_loop_p (const struct loop *loop) return ret; } -/* Sets probability and count of edge E to zero. The probability and count - is redistributed evenly to the remaining edges coming from E->src. */ - -static void -set_zero_probability (edge e) -{ - basic_block bb = e->src; - edge_iterator ei; - edge ae, last = NULL; - unsigned n = EDGE_COUNT (bb->succs); - profile_count cnt = e->count, cnt1; - unsigned prob = e->probability, prob1; - - gcc_assert (n > 1); - cnt1 = cnt.apply_scale (1, (n - 1)); - prob1 = prob / (n - 1); - - FOR_EACH_EDGE (ae, ei, bb->succs) - { - if (ae == e) - continue; - - ae->probability += prob1; - ae->count += cnt1; - last = ae; - } - - /* Move the rest to one of the edges. */ - last->probability += prob % (n - 1); - /* TODO: Remove once we have fractional counts. */ - if (cnt.initialized_p ()) - last->count += profile_count::from_gcov_type (cnt.to_gcov_type () % (n - 1)); - - e->probability = 0; - e->count = profile_count::zero (); -} - /* Duplicates body of LOOP to given edge E NDUPL times. Takes care of updating loop structure and dominators. E's destination must be LOOP header for this to work, i.e. it must be entry or latch edge of this loop; these are @@ -1224,14 +1188,18 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, prob_pass_wont_exit = RDIV (REG_BR_PROB_BASE * (freq_le + freq_out_orig), freq_in); - if (orig - && REG_BR_PROB_BASE - orig->probability != 0) + if (orig && orig->probability.initialized_p () + && !(orig->probability == profile_probability::always ())) { /* The blocks that are dominated by a removed exit edge ORIG have frequencies scaled by this. */ - scale_after_exit - = GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE, - REG_BR_PROB_BASE - orig->probability); + if (orig->probability.initialized_p ()) + scale_after_exit + = GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE, + REG_BR_PROB_BASE + - orig->probability.to_reg_br_prob_base ()); + else + scale_after_exit = REG_BR_PROB_BASE; bbs_to_scale = BITMAP_ALLOC (NULL); for (i = 0; i < n; i++) { @@ -1387,7 +1355,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, { if (to_remove) to_remove->safe_push (new_spec_edges[SE_ORIG]); - set_zero_probability (new_spec_edges[SE_ORIG]); + force_edge_cold (new_spec_edges[SE_ORIG], true); /* Scale the frequencies of the blocks dominated by the exit. */ if (bbs_to_scale) @@ -1423,7 +1391,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, { if (to_remove) to_remove->safe_push (orig); - set_zero_probability (orig); + force_edge_cold (orig, true); /* Scale the frequencies of the blocks dominated by the exit. */ if (bbs_to_scale) @@ -1657,8 +1625,9 @@ force_single_succ_latches (void) static basic_block lv_adjust_loop_entry_edge (basic_block first_head, basic_block second_head, - edge e, void *cond_expr, unsigned then_prob, - unsigned else_prob) + edge e, void *cond_expr, + profile_probability then_prob, + profile_probability else_prob) { basic_block new_head = NULL; edge e1; @@ -1713,7 +1682,7 @@ lv_adjust_loop_entry_edge (basic_block first_head, basic_block second_head, struct loop * loop_version (struct loop *loop, void *cond_expr, basic_block *condition_bb, - unsigned then_prob, unsigned else_prob, + profile_probability then_prob, profile_probability else_prob, unsigned then_scale, unsigned else_scale, bool place_after) { diff --git a/gcc/cfgloopmanip.h b/gcc/cfgloopmanip.h index 7ad3bf5d7d4..3c9536c8fc5 100644 --- a/gcc/cfgloopmanip.h +++ b/gcc/cfgloopmanip.h @@ -58,7 +58,8 @@ basic_block create_preheader (struct loop *, int); extern void create_preheaders (int); extern void force_single_succ_latches (void); struct loop * loop_version (struct loop *, void *, - basic_block *, unsigned, unsigned, + basic_block *, + profile_probability, profile_probability, unsigned, unsigned, bool); #endif /* GCC_CFGLOOPMANIP_H */ diff --git a/gcc/cfgrtl.c b/gcc/cfgrtl.c index 623d04f4499..f2a238c1d5d 100644 --- a/gcc/cfgrtl.c +++ b/gcc/cfgrtl.c @@ -1155,7 +1155,7 @@ try_redirect_by_replacing_jump (edge e, basic_block target, bool in_cfglayout) else e->flags = 0; - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = src->count; if (e->dest != target) @@ -1504,12 +1504,10 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label) { int prob = XINT (note, 0); - b->probability = prob; + b->probability = profile_probability::from_reg_br_prob_base (prob); b->count = e->count.apply_probability (prob); e->probability -= e->probability; e->count -= b->count; - if (e->probability < 0) - e->probability = 0; } } @@ -1618,7 +1616,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label) { rtx_insn *new_head; profile_count count = e->count; - int probability = e->probability; + profile_probability probability = e->probability; /* Create the new structures. */ /* If the old block ended with a tablejump, skip its table @@ -1646,7 +1644,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label) /* Redirect old edge. */ redirect_edge_pred (e, jump_block); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); /* If e->src was previously region crossing, it no longer is and the reg crossing note should be removed. */ @@ -1656,7 +1654,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label) add also edge from asm goto bb to target. */ if (asm_goto_edge) { - new_edge->probability /= 2; + new_edge->probability = new_edge->probability.apply_scale (1, 2); new_edge->count = new_edge->count.apply_scale (1, 2); jump_block->count = jump_block->count.apply_scale (1, 2); jump_block->frequency /= 2; @@ -2251,12 +2249,13 @@ void update_br_prob_note (basic_block bb) { rtx note; - if (!JUMP_P (BB_END (bb))) + if (!JUMP_P (BB_END (bb)) || !BRANCH_EDGE (bb)->probability.initialized_p ()) return; note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX); - if (!note || XINT (note, 0) == BRANCH_EDGE (bb)->probability) + if (!note + || XINT (note, 0) == BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ()) return; - XINT (note, 0) = BRANCH_EDGE (bb)->probability; + XINT (note, 0) = BRANCH_EDGE (bb)->probability.to_reg_br_prob_base (); } /* Get the last insn associated with block BB (that includes barriers and @@ -2447,11 +2446,24 @@ rtl_verify_edges (void) && EDGE_COUNT (bb->succs) >= 2 && any_condjump_p (BB_END (bb))) { - if (XINT (note, 0) != BRANCH_EDGE (bb)->probability - && profile_status_for_fn (cfun) != PROFILE_ABSENT) + if (!BRANCH_EDGE (bb)->probability.initialized_p ()) + { + /* FIXME: sometimes we create BBs with only branch edge + probability defined. */ + if (0) + { + error ("verify_flow_info: " + "REG_BR_PROB is set but cfg probability is not"); + err = 1; + } + } + else if (XINT (note, 0) + != BRANCH_EDGE (bb)->probability.to_reg_br_prob_base () + && profile_status_for_fn (cfun) != PROFILE_ABSENT) { error ("verify_flow_info: REG_BR_PROB does not match cfg %i %i", - XINT (note, 0), BRANCH_EDGE (bb)->probability); + XINT (note, 0), + BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ()); err = 1; } } @@ -3143,7 +3155,7 @@ purge_dead_edges (basic_block bb) /* Redistribute probabilities. */ if (single_succ_p (bb)) { - single_succ_edge (bb)->probability = REG_BR_PROB_BASE; + single_succ_edge (bb)->probability = profile_probability::always (); single_succ_edge (bb)->count = bb->count; } else @@ -3154,8 +3166,9 @@ purge_dead_edges (basic_block bb) b = BRANCH_EDGE (bb); f = FALLTHRU_EDGE (bb); - b->probability = XINT (note, 0); - f->probability = REG_BR_PROB_BASE - b->probability; + b->probability = profile_probability::from_reg_br_prob_base + (XINT (note, 0)); + f->probability = profile_probability::always () - b->probability; b->count = bb->count.apply_probability (b->probability); f->count = bb->count.apply_probability (f->probability); } @@ -3208,7 +3221,7 @@ purge_dead_edges (basic_block bb) gcc_assert (single_succ_p (bb)); - single_succ_edge (bb)->probability = REG_BR_PROB_BASE; + single_succ_edge (bb)->probability = profile_probability::always (); single_succ_edge (bb)->count = bb->count; if (dump_file) @@ -4931,7 +4944,8 @@ rtl_lv_add_condition_to_bb (basic_block first_head , start_sequence (); op0 = force_operand (op0, NULL_RTX); op1 = force_operand (op1, NULL_RTX); - do_compare_rtx_and_jump (op0, op1, comp, 0, mode, NULL_RTX, NULL, label, -1); + do_compare_rtx_and_jump (op0, op1, comp, 0, mode, NULL_RTX, NULL, label, + profile_probability::uninitialized ()); jump = get_last_insn (); JUMP_LABEL (jump) = label; LABEL_NUSES (label)++; diff --git a/gcc/cgraph.c b/gcc/cgraph.c index 81aed5c357c..6711aeb828e 100644 --- a/gcc/cgraph.c +++ b/gcc/cgraph.c @@ -20,8 +20,9 @@ along with GCC; see the file COPYING3. If not see /* This file contains basic routines manipulating call graph - The call-graph is a data structure designed for intra-procedural optimization. - It represents a multi-graph where nodes are functions and edges are call sites. */ + The call-graph is a data structure designed for inter-procedural + optimization. It represents a multi-graph where nodes are functions + (symbols within symbol table) and edges are call sites. */ #include "config.h" #include "system.h" @@ -1316,13 +1317,16 @@ cgraph_edge::redirect_call_stmt_to_callee (void) push_cfun (DECL_STRUCT_FUNCTION (e->caller->decl)); new_stmt = gimple_ic (e->call_stmt, dyn_cast<cgraph_node *> (ref->referred), + /* FIXME: cleanup. */ + profile_probability::from_reg_br_prob_base ( e->count > profile_count::zero () || e2->count > profile_count::zero () - ? e->count.probability_in (e->count + e2->count) + ? e->count.probability_in + (e->count + e2->count).to_reg_br_prob_base () : e->frequency || e2->frequency ? RDIV (e->frequency * REG_BR_PROB_BASE, e->frequency + e2->frequency) - : REG_BR_PROB_BASE / 2, + : REG_BR_PROB_BASE / 2), e->count, e->count + e2->count); e->speculative = false; e->caller->set_call_stmt_including_clones (e->call_stmt, new_stmt, diff --git a/gcc/cgraphunit.c b/gcc/cgraphunit.c index 77209046d9a..a0d1f303b2a 100644 --- a/gcc/cgraphunit.c +++ b/gcc/cgraphunit.c @@ -1506,18 +1506,18 @@ init_lowered_empty_function (tree decl, bool in_ssa, profile_count count) /* Create BB for body of the function and connect it properly. */ ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = count; - ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = REG_BR_PROB_BASE; + ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX; EXIT_BLOCK_PTR_FOR_FN (cfun)->count = count; - EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency = REG_BR_PROB_BASE; + EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX; bb = create_basic_block (NULL, ENTRY_BLOCK_PTR_FOR_FN (cfun)); bb->count = count; bb->frequency = BB_FREQ_MAX; e = make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), bb, EDGE_FALLTHRU); e->count = count; - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); e->count = count; - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); add_bb_to_loop (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father); return bb; @@ -1891,19 +1891,18 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk) NULL_TREE, NULL_TREE); gsi_insert_after (&bsi, stmt, GSI_NEW_STMT); e = make_edge (bb, then_bb, EDGE_TRUE_VALUE); - e->probability = REG_BR_PROB_BASE - REG_BR_PROB_BASE / 16; + e->probability = profile_probability::guessed_always () + .apply_scale (1, 16); e->count = count - count.apply_scale (1, 16); e = make_edge (bb, else_bb, EDGE_FALSE_VALUE); - e->probability = REG_BR_PROB_BASE / 16; + e->probability = profile_probability::guessed_always () + .apply_scale (1, 16); e->count = count.apply_scale (1, 16); - e = make_edge (return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); - e->probability = REG_BR_PROB_BASE; - e->count = count; - e = make_edge (then_bb, return_bb, EDGE_FALLTHRU); - e->probability = REG_BR_PROB_BASE; - e->count = count - count.apply_scale (1, 16); + make_single_succ_edge (return_bb, + EXIT_BLOCK_PTR_FOR_FN (cfun), 0); + make_single_succ_edge (then_bb, return_bb, EDGE_FALLTHRU); e = make_edge (else_bb, return_bb, EDGE_FALLTHRU); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = count.apply_scale (1, 16); bsi = gsi_last_bb (then_bb); } diff --git a/gcc/cilk-common.c b/gcc/cilk-common.c index 9cbe03f8e02..edde4711869 100644 --- a/gcc/cilk-common.c +++ b/gcc/cilk-common.c @@ -38,6 +38,7 @@ along with GCC; see the file COPYING3. If not see #include "stor-layout.h" #include "langhooks.h" #include "explow.h" +#include "profile-count.h" #include "expr.h" #include "tree-iterator.h" #include "gimplify.h" diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi index d1e097b9ffc..772268084fa 100644 --- a/gcc/doc/invoke.texi +++ b/gcc/doc/invoke.texi @@ -8699,7 +8699,7 @@ into separate sections of the assembly and @file{.o} files, to improve paging and cache locality performance. This optimization is automatically turned off in the presence of -exception handling, for linkonce sections, for functions with a user-defined +exception handling or unwind tables (on targets using setjump/longjump or target specific scheme), for linkonce sections, for functions with a user-defined section attribute and on any architecture that does not support named sections. When @option{-fsplit-stack} is used this option is not enabled by default (to avoid linker errors), but may be enabled diff --git a/gcc/dojump.c b/gcc/dojump.c index b45bba3ac37..7a389663ab4 100644 --- a/gcc/dojump.c +++ b/gcc/dojump.c @@ -39,19 +39,13 @@ along with GCC; see the file COPYING3. If not see static bool prefer_and_bit_test (machine_mode, int); static void do_jump_by_parts_greater (tree, tree, int, - rtx_code_label *, rtx_code_label *, int); + rtx_code_label *, rtx_code_label *, + profile_probability); static void do_jump_by_parts_equality (tree, tree, rtx_code_label *, - rtx_code_label *, int); + rtx_code_label *, profile_probability); static void do_compare_and_jump (tree, tree, enum rtx_code, enum rtx_code, - rtx_code_label *, rtx_code_label *, int); - -/* Invert probability if there is any. -1 stands for unknown. */ - -static inline int -inv (int prob) -{ - return prob == -1 ? -1 : REG_BR_PROB_BASE - prob; -} + rtx_code_label *, rtx_code_label *, + profile_probability); /* At the start of a function, record that we have no previously-pushed arguments waiting to be popped. */ @@ -128,29 +122,29 @@ restore_pending_stack_adjust (saved_pending_stack_adjust *save) /* Generate code to evaluate EXP and jump to LABEL if the value is zero. */ void -jumpifnot (tree exp, rtx_code_label *label, int prob) +jumpifnot (tree exp, rtx_code_label *label, profile_probability prob) { - do_jump (exp, label, NULL, inv (prob)); + do_jump (exp, label, NULL, prob.invert ()); } void jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx_code_label *label, - int prob) + profile_probability prob) { - do_jump_1 (code, op0, op1, label, NULL, inv (prob)); + do_jump_1 (code, op0, op1, label, NULL, prob.invert ()); } /* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */ void -jumpif (tree exp, rtx_code_label *label, int prob) +jumpif (tree exp, rtx_code_label *label, profile_probability prob) { do_jump (exp, NULL, label, prob); } void jumpif_1 (enum tree_code code, tree op0, tree op1, - rtx_code_label *label, int prob) + rtx_code_label *label, profile_probability prob) { do_jump_1 (code, op0, op1, NULL, label, prob); } @@ -200,12 +194,12 @@ prefer_and_bit_test (machine_mode mode, int bitnum) /* Subroutine of do_jump, dealing with exploded comparisons of the type OP0 CODE OP1 . IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump. - PROB is probability of jump to if_true_label, or -1 if unknown. */ + PROB is probability of jump to if_true_label. */ void do_jump_1 (enum tree_code code, tree op0, tree op1, rtx_code_label *if_false_label, rtx_code_label *if_true_label, - int prob) + profile_probability prob) { machine_mode mode; rtx_code_label *drop_through_label = 0; @@ -222,7 +216,8 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, != MODE_COMPLEX_INT); if (integer_zerop (op1)) - do_jump (op0, if_true_label, if_false_label, inv (prob)); + do_jump (op0, if_true_label, if_false_label, + prob.invert ()); else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT && !can_compare_p (EQ, TYPE_MODE (inner_type), ccp_jump)) do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label, @@ -247,7 +242,7 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT && !can_compare_p (NE, TYPE_MODE (inner_type), ccp_jump)) do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); else do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label, prob); @@ -270,7 +265,7 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, if (GET_MODE_CLASS (mode) == MODE_INT && ! can_compare_p (LE, mode, ccp_jump)) do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); else do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label, prob); @@ -292,7 +287,7 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, if (GET_MODE_CLASS (mode) == MODE_INT && ! can_compare_p (GE, mode, ccp_jump)) do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); else do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label, prob); @@ -346,17 +341,17 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, half of the total probability of being false, so its jump has a false probability of half the total, relative to the probability we reached it (i.e. the first condition was true). */ - int op0_prob = -1; - int op1_prob = -1; - if (prob != -1) + profile_probability op0_prob = profile_probability::uninitialized (); + profile_probability op1_prob = profile_probability::uninitialized (); + if (prob.initialized_p ()) { - int false_prob = inv (prob); - int op0_false_prob = false_prob / 2; - int op1_false_prob = GCOV_COMPUTE_SCALE ((false_prob / 2), - inv (op0_false_prob)); + profile_probability false_prob = prob.invert (); + profile_probability op0_false_prob = false_prob.apply_scale (1, 2); + profile_probability op1_false_prob = false_prob.apply_scale (1, 2) + / op0_false_prob.invert (); /* Get the probability that each jump below is true. */ - op0_prob = inv (op0_false_prob); - op1_prob = inv (op1_false_prob); + op0_prob = op0_false_prob.invert (); + op1_prob = op1_false_prob.invert (); } if (if_false_label == NULL) { @@ -379,12 +374,12 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, The second condition has the other half of the total probability, so its jump has a probability of half the total, relative to the probability we reached it (i.e. the first condition was false). */ - int op0_prob = -1; - int op1_prob = -1; - if (prob != -1) + profile_probability op0_prob = profile_probability::uninitialized (); + profile_probability op1_prob = profile_probability::uninitialized (); + if (prob.initialized_p ()) { - op0_prob = prob / 2; - op1_prob = GCOV_COMPUTE_SCALE ((prob / 2), inv (op0_prob)); + op0_prob = prob.apply_scale (1, 2); + op1_prob = prob.apply_scale (1, 2) / op0_prob.invert (); } if (if_true_label == NULL) { @@ -420,11 +415,11 @@ do_jump_1 (enum tree_code code, tree op0, tree op1, actually perform a jump. An example where there is no jump is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null. - PROB is probability of jump to if_true_label, or -1 if unknown. */ + PROB is probability of jump to if_true_label. */ void do_jump (tree exp, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, profile_probability prob) { enum tree_code code = TREE_CODE (exp); rtx temp; @@ -481,7 +476,7 @@ do_jump (tree exp, rtx_code_label *if_false_label, case TRUTH_NOT_EXPR: do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label, - inv (prob)); + prob.invert ()); break; case COND_EXPR: @@ -497,7 +492,8 @@ do_jump (tree exp, rtx_code_label *if_false_label, } do_pending_stack_adjust (); - do_jump (TREE_OPERAND (exp, 0), label1, NULL, -1); + do_jump (TREE_OPERAND (exp, 0), label1, NULL, + profile_probability::uninitialized ()); do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label, prob); emit_label (label1); do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label, prob); @@ -542,7 +538,7 @@ do_jump (tree exp, rtx_code_label *if_false_label, { tree exp0 = TREE_OPERAND (exp, 0); rtx_code_label *set_label, *clr_label; - int setclr_prob = prob; + profile_probability setclr_prob = prob; /* Strip narrowing integral type conversions. */ while (CONVERT_EXPR_P (exp0) @@ -558,7 +554,7 @@ do_jump (tree exp, rtx_code_label *if_false_label, exp0 = TREE_OPERAND (exp0, 0); clr_label = if_true_label; set_label = if_false_label; - setclr_prob = inv (prob); + setclr_prob = prob.invert (); } else { @@ -673,7 +669,7 @@ static void do_jump_by_parts_greater_rtx (machine_mode mode, int unsignedp, rtx op0, rtx op1, rtx_code_label *if_false_label, rtx_code_label *if_true_label, - int prob) + profile_probability prob) { int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD); rtx_code_label *drop_through_label = 0; @@ -703,7 +699,7 @@ do_jump_by_parts_greater_rtx (machine_mode mode, int unsignedp, rtx op0, if_false_label = drop_through_label; drop_through_if_true = false; drop_through_if_false = true; - prob = inv (prob); + prob = prob.invert (); } /* Compare a word at a time, high order first. */ @@ -733,7 +729,8 @@ do_jump_by_parts_greater_rtx (machine_mode mode, int unsignedp, rtx op0, /* Consider lower words only if these are equal. */ do_compare_rtx_and_jump (op0_word, op1_word, NE, unsignedp, word_mode, - NULL_RTX, NULL, if_false_label, inv (prob)); + NULL_RTX, NULL, if_false_label, + prob.invert ()); } if (!drop_through_if_false) @@ -750,7 +747,8 @@ do_jump_by_parts_greater_rtx (machine_mode mode, int unsignedp, rtx op0, static void do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, + profile_probability prob) { rtx op0 = expand_normal (swap ? treeop1 : treeop0); rtx op1 = expand_normal (swap ? treeop0 : treeop1); @@ -769,7 +767,8 @@ do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap, static void do_jump_by_parts_zero_rtx (machine_mode mode, rtx op0, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, + profile_probability prob) { int nwords = GET_MODE_SIZE (mode) / UNITS_PER_WORD; rtx part; @@ -819,7 +818,8 @@ do_jump_by_parts_zero_rtx (machine_mode mode, rtx op0, static void do_jump_by_parts_equality_rtx (machine_mode mode, rtx op0, rtx op1, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, + profile_probability prob) { int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD); rtx_code_label *drop_through_label = NULL; @@ -859,7 +859,8 @@ do_jump_by_parts_equality_rtx (machine_mode mode, rtx op0, rtx op1, static void do_jump_by_parts_equality (tree treeop0, tree treeop1, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, + profile_probability prob) { rtx op0 = expand_normal (treeop0); rtx op1 = expand_normal (treeop1); @@ -956,7 +957,8 @@ void do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, machine_mode mode, rtx size, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, + profile_probability prob) { rtx tem; rtx_code_label *dummy_label = NULL; @@ -983,7 +985,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, { std::swap (if_true_label, if_false_label); code = rcode; - prob = inv (prob); + prob = prob.invert (); } } @@ -1035,7 +1037,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, case LEU: do_jump_by_parts_greater_rtx (mode, 1, op0, op1, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); break; case GTU: @@ -1046,7 +1048,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, case GEU: do_jump_by_parts_greater_rtx (mode, 1, op1, op0, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); break; case LT: @@ -1057,7 +1059,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, case LE: do_jump_by_parts_greater_rtx (mode, 0, op0, op1, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); break; case GT: @@ -1068,7 +1070,7 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, case GE: do_jump_by_parts_greater_rtx (mode, 0, op1, op0, if_true_label, if_false_label, - inv (prob)); + prob.invert ()); break; case EQ: @@ -1078,7 +1080,8 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, case NE: do_jump_by_parts_equality_rtx (mode, op0, op1, if_true_label, - if_false_label, inv (prob)); + if_false_label, + prob.invert ()); break; default: @@ -1115,11 +1118,13 @@ do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp, else { - int first_prob = prob; + profile_probability first_prob = prob; if (first_code == UNORDERED) - first_prob = REG_BR_PROB_BASE / 100; + first_prob = profile_probability::guessed_always ().apply_scale + (1, 100); else if (first_code == ORDERED) - first_prob = REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100; + first_prob = profile_probability::guessed_always ().apply_scale + (99, 100); if (and_them) { rtx_code_label *dest_label; @@ -1165,7 +1170,7 @@ static void do_compare_and_jump (tree treeop0, tree treeop1, enum rtx_code signed_code, enum rtx_code unsigned_code, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob) + rtx_code_label *if_true_label, profile_probability prob) { rtx op0, op1; tree type; diff --git a/gcc/dojump.h b/gcc/dojump.h index 00d66a7239d..ca6df449f74 100644 --- a/gcc/dojump.h +++ b/gcc/dojump.h @@ -57,23 +57,26 @@ extern void save_pending_stack_adjust (saved_pending_stack_adjust *); extern void restore_pending_stack_adjust (saved_pending_stack_adjust *); /* Generate code to evaluate EXP and jump to LABEL if the value is zero. */ -extern void jumpifnot (tree exp, rtx_code_label *label, int prob); -extern void jumpifnot_1 (enum tree_code, tree, tree, rtx_code_label *, int); +extern void jumpifnot (tree exp, rtx_code_label *label, + profile_probability prob); +extern void jumpifnot_1 (enum tree_code, tree, tree, rtx_code_label *, + profile_probability); /* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */ -extern void jumpif (tree exp, rtx_code_label *label, int prob); -extern void jumpif_1 (enum tree_code, tree, tree, rtx_code_label *, int); +extern void jumpif (tree exp, rtx_code_label *label, profile_probability prob); +extern void jumpif_1 (enum tree_code, tree, tree, rtx_code_label *, + profile_probability); /* Generate code to evaluate EXP and jump to IF_FALSE_LABEL if the result is zero, or IF_TRUE_LABEL if the result is one. */ extern void do_jump (tree exp, rtx_code_label *if_false_label, - rtx_code_label *if_true_label, int prob); + rtx_code_label *if_true_label, profile_probability prob); extern void do_jump_1 (enum tree_code, tree, tree, rtx_code_label *, - rtx_code_label *, int); + rtx_code_label *, profile_probability); extern void do_compare_rtx_and_jump (rtx, rtx, enum rtx_code, int, machine_mode, rtx, rtx_code_label *, - rtx_code_label *, int); + rtx_code_label *, profile_probability); extern bool split_comparison (enum rtx_code, machine_mode, enum rtx_code *, enum rtx_code *); diff --git a/gcc/dwarf2cfi.c b/gcc/dwarf2cfi.c index 2a527c9feca..a5f9832fc4a 100644 --- a/gcc/dwarf2cfi.c +++ b/gcc/dwarf2cfi.c @@ -35,6 +35,7 @@ along with GCC; see the file COPYING3. If not see #include "common/common-target.h" #include "except.h" /* expand_builtin_dwarf_sp_column */ +#include "profile-count.h" /* For expr.h */ #include "expr.h" /* init_return_column_size */ #include "output.h" /* asm_out_file */ #include "debug.h" /* dwarf2out_do_frame, dwarf2out_do_cfi_asm */ diff --git a/gcc/except.c b/gcc/except.c index 0bf2c2eac0e..4ec2f82eb1f 100644 --- a/gcc/except.c +++ b/gcc/except.c @@ -974,7 +974,6 @@ dw2_build_landing_pads (void) { basic_block bb; rtx_insn *seq; - edge e; if (lp == NULL || lp->post_landing_pad == NULL) continue; @@ -991,9 +990,7 @@ dw2_build_landing_pads (void) end_sequence (); bb = emit_to_new_bb_before (seq, label_rtx (lp->post_landing_pad)); - e = make_edge (bb, bb->next_bb, e_flags); - e->count = bb->count; - e->probability = REG_BR_PROB_BASE; + make_single_succ_edge (bb, bb->next_bb, e_flags); if (current_loops) { struct loop *loop = bb->next_bb->loop_father; @@ -1258,7 +1255,6 @@ sjlj_emit_dispatch_table (rtx_code_label *dispatch_label, int num_dispatch) rtx_insn *seq; basic_block bb; eh_region r; - edge e; int i, disp_index; vec<tree> dispatch_labels = vNULL; @@ -1346,9 +1342,7 @@ sjlj_emit_dispatch_table (rtx_code_label *dispatch_label, int num_dispatch) rtx_insn *before = label_rtx (lp->post_landing_pad); bb = emit_to_new_bb_before (seq2, before); - e = make_edge (bb, bb->next_bb, EDGE_FALLTHRU); - e->count = bb->count; - e->probability = REG_BR_PROB_BASE; + make_single_succ_edge (bb, bb->next_bb, EDGE_FALLTHRU); if (current_loops) { struct loop *loop = bb->next_bb->loop_father; @@ -1386,9 +1380,7 @@ sjlj_emit_dispatch_table (rtx_code_label *dispatch_label, int num_dispatch) bb = emit_to_new_bb_before (seq, first_reachable_label); if (num_dispatch == 1) { - e = make_edge (bb, bb->next_bb, EDGE_FALLTHRU); - e->count = bb->count; - e->probability = REG_BR_PROB_BASE; + make_single_succ_edge (bb, bb->next_bb, EDGE_FALLTHRU); if (current_loops) { struct loop *loop = bb->next_bb->loop_father; diff --git a/gcc/explow.c b/gcc/explow.c index 67cb6ff1513..50074e281ed 100644 --- a/gcc/explow.c +++ b/gcc/explow.c @@ -28,6 +28,7 @@ along with GCC; see the file COPYING3. If not see #include "memmodel.h" #include "tm_p.h" #include "expmed.h" +#include "profile-count.h" #include "optabs.h" #include "emit-rtl.h" #include "recog.h" diff --git a/gcc/expmed.c b/gcc/expmed.c index 0c812806f9b..fe23066fc26 100644 --- a/gcc/expmed.c +++ b/gcc/expmed.c @@ -5905,7 +5905,8 @@ emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1, { label = gen_label_rtx (); do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode, - NULL_RTX, NULL, label, -1); + NULL_RTX, NULL, label, + profile_probability::uninitialized ()); emit_move_insn (target, trueval); emit_label (label); return target; @@ -5943,7 +5944,7 @@ emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1, emit_move_insn (target, trueval); label = gen_label_rtx (); do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL, - label, -1); + label, profile_probability::uninitialized ()); emit_move_insn (target, falseval); emit_label (label); @@ -5961,5 +5962,5 @@ do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode, { int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU); do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX, - NULL, label, -1); + NULL, label, profile_probability::uninitialized ()); } diff --git a/gcc/expr.c b/gcc/expr.c index 5febf07929d..753f5efde6c 100644 --- a/gcc/expr.c +++ b/gcc/expr.c @@ -96,7 +96,8 @@ static rtx do_store_flag (sepops, rtx, machine_mode); #ifdef PUSH_ROUNDING static void emit_single_push_insn (machine_mode, rtx, tree); #endif -static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx, int); +static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx, + profile_probability); static rtx const_vector_from_tree (tree); static rtx const_scalar_mask_from_tree (tree); static tree tree_expr_size (const_tree); @@ -1452,7 +1453,7 @@ compare_by_pieces_d::generate (rtx op0, rtx op1, machine_mode mode) m_accumulator = NULL_RTX; } do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL, - m_fail_label, -1); + m_fail_label, profile_probability::uninitialized ()); } /* Return true if MODE can be used for a set of moves and comparisons, @@ -1484,7 +1485,8 @@ compare_by_pieces_d::finish_mode (machine_mode mode) { if (m_accumulator != NULL_RTX) do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode, - NULL_RTX, NULL, m_fail_label, -1); + NULL_RTX, NULL, m_fail_label, + profile_probability::uninitialized ()); } /* Generate several move instructions to compare LEN bytes from blocks @@ -1845,7 +1847,9 @@ emit_block_move_via_loop (rtx x, rtx y, rtx size, emit_label (cmp_label); emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode, - true, top_label, REG_BR_PROB_BASE * 90 / 100); + true, top_label, + profile_probability::guessed_always () + .apply_scale (9, 10)); } /* Expand a call to memcpy or memmove or memcmp, and return the result. @@ -5402,7 +5406,8 @@ store_expr_with_bounds (tree exp, rtx target, int call_param_p, do_pending_stack_adjust (); NO_DEFER_POP; - jumpifnot (TREE_OPERAND (exp, 0), lab1, -1); + jumpifnot (TREE_OPERAND (exp, 0), lab1, + profile_probability::uninitialized ()); store_expr_with_bounds (TREE_OPERAND (exp, 1), target, call_param_p, nontemporal, reverse, btarget); emit_jump_insn (targetm.gen_jump (lab2)); @@ -6504,7 +6509,8 @@ store_constructor (tree exp, rtx target, int cleared, HOST_WIDE_INT size, /* Generate a conditional jump to exit the loop. */ exit_cond = build2 (LT_EXPR, integer_type_node, index, hi_index); - jumpif (exit_cond, loop_end, -1); + jumpif (exit_cond, loop_end, + profile_probability::uninitialized ()); /* Update the loop counter, and jump to the head of the loop. */ @@ -9043,7 +9049,7 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode, lab = gen_label_rtx (); do_compare_rtx_and_jump (target, cmpop1, comparison_code, unsignedp, mode, NULL_RTX, NULL, lab, - -1); + profile_probability::uninitialized ()); } emit_move_insn (target, op1); emit_label (lab); @@ -9272,7 +9278,8 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode, emit_move_insn (target, const0_rtx); rtx_code_label *lab1 = gen_label_rtx (); - jumpifnot_1 (code, treeop0, treeop1, lab1, -1); + jumpifnot_1 (code, treeop0, treeop1, lab1, + profile_probability::uninitialized ()); if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type)) emit_move_insn (target, constm1_rtx); @@ -9523,7 +9530,8 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode, NO_DEFER_POP; rtx_code_label *lab0 = gen_label_rtx (); rtx_code_label *lab1 = gen_label_rtx (); - jumpifnot (treeop0, lab0, -1); + jumpifnot (treeop0, lab0, + profile_probability::uninitialized ()); store_expr (treeop1, temp, modifier == EXPAND_STACK_PARM, false, false); @@ -11042,7 +11050,8 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode, int value = TREE_CODE (rhs) == BIT_IOR_EXPR; do_jump (TREE_OPERAND (rhs, 1), value ? label : 0, - value ? 0 : label, -1); + value ? 0 : label, + profile_probability::uninitialized ()); expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value), false); do_pending_stack_adjust (); @@ -11512,7 +11521,7 @@ do_store_flag (sepops ops, rtx target, machine_mode mode) int try_casesi (tree index_type, tree index_expr, tree minval, tree range, rtx table_label, rtx default_label, rtx fallback_label, - int default_probability) + profile_probability default_probability) { struct expand_operand ops[5]; machine_mode index_mode = SImode; @@ -11582,7 +11591,7 @@ try_casesi (tree index_type, tree index_expr, tree minval, tree range, static void do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label, - rtx default_label, int default_probability) + rtx default_label, profile_probability default_probability) { rtx temp, vector; @@ -11645,7 +11654,8 @@ do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label, int try_tablejump (tree index_type, tree index_expr, tree minval, tree range, - rtx table_label, rtx default_label, int default_probability) + rtx table_label, rtx default_label, + profile_probability default_probability) { rtx index; diff --git a/gcc/expr.h b/gcc/expr.h index 84002abb025..b92ff3ce8a6 100644 --- a/gcc/expr.h +++ b/gcc/expr.h @@ -288,8 +288,8 @@ expand_normal (tree exp) extern tree string_constant (tree, tree *); /* Two different ways of generating switch statements. */ -extern int try_casesi (tree, tree, tree, tree, rtx, rtx, rtx, int); -extern int try_tablejump (tree, tree, tree, tree, rtx, rtx, int); +extern int try_casesi (tree, tree, tree, tree, rtx, rtx, rtx, profile_probability); +extern int try_tablejump (tree, tree, tree, tree, rtx, rtx, profile_probability); extern int safe_from_p (const_rtx, tree, int); diff --git a/gcc/gimple-pretty-print.c b/gcc/gimple-pretty-print.c index 447921be036..4012b3b9e2d 100644 --- a/gcc/gimple-pretty-print.c +++ b/gcc/gimple-pretty-print.c @@ -80,7 +80,7 @@ debug_gimple_stmt (gimple *gs) by xstrdup_for_dump. */ static const char * -dump_probability (int frequency, profile_count &count) +dump_profile (int frequency, profile_count &count) { float minimum = 0.01f; @@ -102,6 +102,38 @@ dump_probability (int frequency, profile_count &count) return ret; } +/* Return formatted string of a VALUE probability + (biased by REG_BR_PROB_BASE). Returned string is allocated + by xstrdup_for_dump. */ + +static const char * +dump_probability (profile_probability probability, profile_count &count) +{ + float minimum = 0.01f; + float fvalue = -1; + + if (probability.initialized_p ()) + { + fvalue = probability.to_reg_br_prob_base () * 100.0f / REG_BR_PROB_BASE; + if (fvalue < minimum && probability.to_reg_br_prob_base ()) + fvalue = minimum; + } + + char *buf; + if (count.initialized_p ()) + asprintf (&buf, "[%.2f%%] [count: %" PRId64 "]", fvalue, + count.to_gcov_type ()); + else if (probability.initialized_p ()) + asprintf (&buf, "[%.2f%%] [count: INV]", fvalue); + else + asprintf (&buf, "[INV] [count: INV]"); + + const char *ret = xstrdup_for_dump (buf); + free (buf); + + return ret; +} + /* Dump E probability to BUFFER. */ static void @@ -1090,7 +1122,7 @@ dump_gimple_label (pretty_printer *buffer, glabel *gs, int spc, dump_generic_node (buffer, label, spc, flags, false); basic_block bb = gimple_bb (gs); if (bb && !(flags & TDF_GIMPLE)) - pp_scalar (buffer, " %s", dump_probability (bb->frequency, bb->count)); + pp_scalar (buffer, " %s", dump_profile (bb->frequency, bb->count)); pp_colon (buffer); } if (flags & TDF_GIMPLE) @@ -2670,8 +2702,8 @@ dump_gimple_bb_header (FILE *outf, basic_block bb, int indent, fprintf (outf, "%*sbb_%d:\n", indent, "", bb->index); else fprintf (outf, "%*s<bb %d> %s:\n", - indent, "", bb->index, dump_probability (bb->frequency, - bb->count)); + indent, "", bb->index, dump_profile (bb->frequency, + bb->count)); } } } diff --git a/gcc/graph.c b/gcc/graph.c index 9261732d23e..2197a7e32af 100644 --- a/gcc/graph.c +++ b/gcc/graph.c @@ -136,12 +136,16 @@ draw_cfg_node_succ_edges (pretty_printer *pp, int funcdef_no, basic_block bb) pp_printf (pp, "\tfn_%d_basic_block_%d:s -> fn_%d_basic_block_%d:n " - "[style=%s,color=%s,weight=%d,constraint=%s, label=\"[%i%%]\"];\n", + "[style=%s,color=%s,weight=%d,constraint=%s];\n", funcdef_no, e->src->index, funcdef_no, e->dest->index, style, color, weight, - (e->flags & (EDGE_FAKE | EDGE_DFS_BACK)) ? "false" : "true", - e->probability * 100 / REG_BR_PROB_BASE); + (e->flags & (EDGE_FAKE | EDGE_DFS_BACK)) ? "false" : "true"); + if (e->probability.initialized_p ()) + pp_printf (pp, ", label=\"[%i%%]\"", + e->probability.to_reg_br_prob_base () + * 100 / REG_BR_PROB_BASE); + pp_printf (pp, "];\n"); } pp_flush (pp); } diff --git a/gcc/hsa-gen.c b/gcc/hsa-gen.c index 85f053cbed9..7d628179ea6 100644 --- a/gcc/hsa-gen.c +++ b/gcc/hsa-gen.c @@ -6139,7 +6139,7 @@ convert_switch_statements (void) auto_vec <edge> new_edges; auto_vec <phi_definition *> phi_todo_list; auto_vec <profile_count> edge_counts; - auto_vec <int> edge_probabilities; + auto_vec <profile_probability> edge_probabilities; /* Investigate all labels that and PHI nodes in these edges which should be fixed after we add new collection of edges. */ @@ -6231,12 +6231,12 @@ convert_switch_statements (void) basic_block label_bb = label_to_block_fn (func, CASE_LABEL (label)); edge new_edge = make_edge (cur_bb, label_bb, EDGE_TRUE_VALUE); - int prob_sum = sum_slice <int> (edge_probabilities, i, labels, 0) + - edge_probabilities[0]; + profile_probability prob_sum = sum_slice <profile_probability> + (edge_probabilities, i, labels, profile_probability::never ()) + + edge_probabilities[0]; - if (prob_sum) - new_edge->probability - = RDIV (REG_BR_PROB_BASE * edge_probabilities[i], prob_sum); + if (prob_sum.initialized_p ()) + new_edge->probability = edge_probabilities[i] / prob_sum; new_edge->count = edge_counts[i]; new_edges.safe_push (new_edge); @@ -6253,8 +6253,7 @@ convert_switch_statements (void) } edge next_edge = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE); - next_edge->probability - = inverse_probability (new_edge->probability); + next_edge->probability = new_edge->probability.invert (); next_edge->count = edge_counts[0] + sum_slice <profile_count> (edge_counts, i, labels, profile_count::zero ()); @@ -6265,7 +6264,7 @@ convert_switch_statements (void) of the switch. */ { edge e = make_edge (cur_bb, default_label_bb, EDGE_FALSE_VALUE); - e->probability = inverse_probability (new_edge->probability); + e->probability = new_edge->probability.invert (); e->count = edge_counts[0]; new_edges.safe_insert (0, e); } diff --git a/gcc/ifcvt.c b/gcc/ifcvt.c index 75458589468..fd682a4cf5a 100644 --- a/gcc/ifcvt.c +++ b/gcc/ifcvt.c @@ -78,7 +78,7 @@ static int cond_exec_changed_p; /* Forward references. */ static int count_bb_insns (const_basic_block); -static bool cheap_bb_rtx_cost_p (const_basic_block, int, int); +static bool cheap_bb_rtx_cost_p (const_basic_block, profile_probability, int); static rtx_insn *first_active_insn (basic_block); static rtx_insn *last_active_insn (basic_block, int); static rtx_insn *find_active_insn_before (basic_block, rtx_insn *); @@ -132,11 +132,14 @@ count_bb_insns (const_basic_block bb) plus a small fudge factor. */ static bool -cheap_bb_rtx_cost_p (const_basic_block bb, int scale, int max_cost) +cheap_bb_rtx_cost_p (const_basic_block bb, + profile_probability prob, int max_cost) { int count = 0; rtx_insn *insn = BB_HEAD (bb); bool speed = optimize_bb_for_speed_p (bb); + int scale = prob.initialized_p () ? prob.to_reg_br_prob_base () + : REG_BR_PROB_BASE; /* Set scale to REG_BR_PROB_BASE to void the identical scaling applied to insn_rtx_cost when optimizing for size. Only do @@ -4807,7 +4810,8 @@ find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge) basic_block then_bb = then_edge->dest; basic_block else_bb = else_edge->dest; basic_block new_bb; - int then_bb_index, then_prob; + int then_bb_index; + profile_probability then_prob; rtx else_target = NULL_RTX; /* If we are partitioning hot/cold basic blocks, we don't want to @@ -4853,10 +4857,7 @@ find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge) "\nIF-CASE-1 found, start %d, then %d\n", test_bb->index, then_bb->index); - if (then_edge->probability) - then_prob = REG_BR_PROB_BASE - then_edge->probability; - else - then_prob = REG_BR_PROB_BASE / 2; + then_prob = then_edge->probability.invert (); /* We're speculating from the THEN path, we want to make sure the cost of speculation is within reason. */ @@ -4927,7 +4928,7 @@ find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge) basic_block then_bb = then_edge->dest; basic_block else_bb = else_edge->dest; edge else_succ; - int then_prob, else_prob; + profile_probability then_prob, else_prob; /* We do not want to speculate (empty) loop latches. */ if (current_loops @@ -4973,16 +4974,8 @@ find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge) if (then_bb->index < NUM_FIXED_BLOCKS) return FALSE; - if (else_edge->probability) - { - else_prob = else_edge->probability; - then_prob = REG_BR_PROB_BASE - else_prob; - } - else - { - else_prob = REG_BR_PROB_BASE / 2; - then_prob = REG_BR_PROB_BASE / 2; - } + else_prob = else_edge->probability; + then_prob = else_prob.invert (); /* ELSE is predicted or SUCC(ELSE) postdominates THEN. */ if (else_prob > then_prob) diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c index 75fe027f7b2..d276dfecf9a 100644 --- a/gcc/internal-fn.c +++ b/gcc/internal-fn.c @@ -569,7 +569,7 @@ expand_arith_overflow_result_store (tree lhs, rtx target, gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode)); do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns), EQ, true, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); emit_label (done_label); } @@ -597,7 +597,7 @@ expand_arith_overflow_result_store (tree lhs, rtx target, } do_compare_rtx_and_jump (res, lres, EQ, true, tgtmode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); emit_label (done_label); } @@ -767,7 +767,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, tem = op1; do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU, true, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); goto do_error_label; } @@ -782,7 +782,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, code == PLUS_EXPR ? res : op0, sgn, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL, - done_label, PROB_VERY_LIKELY); + done_label, profile_probability::very_likely ()); goto do_error_label; } @@ -824,9 +824,9 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, else if (pos_neg == 3) /* If ARG0 is not known to be always positive, check at runtime. */ do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX, - NULL, do_error, PROB_VERY_UNLIKELY); + NULL, do_error, profile_probability::very_unlikely ()); do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL, - done_label, PROB_VERY_LIKELY); + done_label, profile_probability::very_likely ()); goto do_error_label; } @@ -840,7 +840,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL, - done_label, PROB_VERY_LIKELY); + done_label, profile_probability::very_likely ()); goto do_error_label; } @@ -852,7 +852,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX, - NULL, do_error, PROB_VERY_UNLIKELY); + NULL, do_error, profile_probability::very_unlikely ()); rtx tem = op1; /* The operation is commutative, so we can pick operand to compare against. For prec <= BITS_PER_WORD, I think preferring REG operand @@ -866,7 +866,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, : CONST_SCALAR_INT_P (op0)) tem = op0; do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL, - done_label, PROB_VERY_LIKELY); + done_label, profile_probability::very_likely ()); goto do_error_label; } @@ -894,7 +894,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, ? and_optab : ior_optab, op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); } else { @@ -902,17 +902,17 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, do_compare_rtx_and_jump (op1, const0_rtx, code == MINUS_EXPR ? GE : LT, false, mode, NULL_RTX, NULL, do_ior_label, - PROB_EVEN); + profile_probability::even ()); tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); emit_jump (do_error); emit_label (do_ior_label); tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); } goto do_error_label; } @@ -926,13 +926,13 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, OPTAB_LIB_WIDEN); rtx_code_label *op0_geu_op1 = gen_label_rtx (); do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL, - op0_geu_op1, PROB_EVEN); + op0_geu_op1, profile_probability::even ()); do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); emit_jump (do_error); emit_label (op0_geu_op1); do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); goto do_error_label; } @@ -960,7 +960,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) - add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + add_int_reg_note (last, REG_BR_PROB, PROB_UNLIKELY); emit_jump (done_label); goto do_error_label; } @@ -1020,7 +1020,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, /* No overflow if the result has bit sign cleared. */ do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); } /* Compare the result of the operation with the first operand. @@ -1031,7 +1031,7 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, do_compare_rtx_and_jump (res, op0, (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE, false, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); } do_error_label: @@ -1128,7 +1128,7 @@ expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan, /* Compare the operand with the most negative value. */ rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1))); do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL, - done_label, PROB_VERY_LIKELY); + done_label, profile_probability::very_likely ()); } emit_label (do_error); @@ -1261,15 +1261,15 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, ops.location = loc; res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); goto do_error_label; case 3: rtx_code_label *do_main_label; do_main_label = gen_label_rtx (); do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX, - NULL, do_main_label, PROB_VERY_LIKELY); + NULL, do_main_label, profile_probability::very_likely ()); do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX, - NULL, do_main_label, PROB_VERY_LIKELY); + NULL, do_main_label, profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); emit_label (do_main_label); goto do_main; @@ -1306,15 +1306,15 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, ops.location = loc; res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX, - NULL, do_error, PROB_VERY_UNLIKELY); + NULL, do_error, profile_probability::very_unlikely ()); int prec; prec = GET_MODE_PRECISION (mode); rtx sgn; sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode); do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX, - NULL, done_label, PROB_VERY_LIKELY); + NULL, done_label, profile_probability::very_likely ()); goto do_error_label; case 3: /* Rest of handling of this case after res is computed. */ @@ -1361,7 +1361,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); goto do_error_label; } /* The general case, do all the needed comparisons at runtime. */ @@ -1378,7 +1378,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX, - NULL, after_negate_label, PROB_VERY_LIKELY); + NULL, after_negate_label, profile_probability::very_likely ()); /* Both arguments negative here, negate them and continue with normal unsigned overflow checking multiplication. */ emit_move_insn (op0, expand_unop (mode, neg_optab, op0, @@ -1394,13 +1394,13 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN); do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX, - NULL, do_main_label, PROB_VERY_LIKELY); + NULL, do_main_label, profile_probability::very_likely ()); /* One argument is negative here, the other positive. This overflows always, unless one of the arguments is 0. But if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1 is, thus we can keep do_main code oring in overflow as is. */ do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX, - NULL, do_main_label, PROB_VERY_LIKELY); + NULL, do_main_label, profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); emit_label (do_main_label); goto do_main; @@ -1467,7 +1467,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, HIPART is non-zero. */ do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); else { rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1, @@ -1477,7 +1477,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, HIPART is different from RES < 0 ? -1 : 0. */ do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode, NULL_RTX, NULL, done_label, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); } } else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec) @@ -1570,12 +1570,12 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, if (!op0_small_p) do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode, NULL_RTX, NULL, large_op0, - PROB_UNLIKELY); + profile_probability::unlikely ()); if (!op1_small_p) do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode, NULL_RTX, NULL, small_op0_large_op1, - PROB_UNLIKELY); + profile_probability::unlikely ()); /* If both op0 and op1 are sign (!uns) or zero (uns) extended from hmode to mode, the multiplication will never overflow. We can @@ -1621,7 +1621,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, if (!op1_small_p) do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode, NULL_RTX, NULL, both_ops_large, - PROB_UNLIKELY); + profile_probability::unlikely ()); /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode, but op0 is not, prepare larger, hipart and lopart pseudos and @@ -1658,7 +1658,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, else if (larger_sign != -1) do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode, NULL_RTX, NULL, after_hipart_neg, - PROB_EVEN); + profile_probability::even ()); tem = convert_modes (mode, hmode, lopart, 1); tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1); @@ -1674,7 +1674,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, else if (smaller_sign != -1) do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode, NULL_RTX, NULL, after_lopart_neg, - PROB_EVEN); + profile_probability::even ()); tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX, 1, OPTAB_DIRECT); @@ -1704,7 +1704,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode, NULL_RTX, NULL, do_overflow, - PROB_VERY_UNLIKELY); + profile_probability::very_unlikely ()); /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */ rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec, @@ -1741,7 +1741,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, NULL_RTX, 1, OPTAB_DIRECT); do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode, NULL_RTX, NULL, do_error, - PROB_VERY_UNLIKELY); + profile_probability::very_unlikely ()); } if (!op1_medium_p) @@ -1750,7 +1750,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, NULL_RTX, 1, OPTAB_DIRECT); do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode, NULL_RTX, NULL, do_error, - PROB_VERY_UNLIKELY); + profile_probability::very_unlikely ()); } /* At this point hipart{0,1} are both in [-1, 0]. If they are @@ -1761,18 +1761,18 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, else if (op0_sign == 1 || op1_sign == 1) do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode, NULL_RTX, NULL, hipart_different, - PROB_EVEN); + profile_probability::even ()); do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX, NULL, do_error, - PROB_VERY_UNLIKELY); + profile_probability::very_unlikely ()); emit_jump (done_label); emit_label (hipart_different); do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX, NULL, do_error, - PROB_VERY_UNLIKELY); + profile_probability::very_unlikely ()); emit_jump (done_label); } @@ -1817,7 +1817,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, { rtx_code_label *all_done_label = gen_label_rtx (); do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX, - NULL, all_done_label, PROB_VERY_LIKELY); + NULL, all_done_label, profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); emit_label (all_done_label); } @@ -1828,14 +1828,14 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, rtx_code_label *all_done_label = gen_label_rtx (); rtx_code_label *set_noovf = gen_label_rtx (); do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX, - NULL, all_done_label, PROB_VERY_LIKELY); + NULL, all_done_label, profile_probability::very_likely ()); expand_arith_set_overflow (lhs, target); do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX, - NULL, set_noovf, PROB_VERY_LIKELY); + NULL, set_noovf, profile_probability::very_likely ()); do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX, - NULL, all_done_label, PROB_VERY_UNLIKELY); + NULL, all_done_label, profile_probability::very_unlikely ()); do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL, - all_done_label, PROB_VERY_UNLIKELY); + all_done_label, profile_probability::very_unlikely ()); emit_label (set_noovf); write_complex_part (target, const0_rtx, true); emit_label (all_done_label); @@ -1977,7 +1977,7 @@ expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs, emit_move_insn (cntvar, ret); do_compare_rtx_and_jump (cntvar, GEN_INT (cnt), NE, false, TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab, - PROB_VERY_LIKELY); + profile_probability::very_likely ()); } if (lhs && resv == NULL_TREE) { diff --git a/gcc/ipa-cp.c b/gcc/ipa-cp.c index ee0aa9f9c6c..3b9eab41672 100644 --- a/gcc/ipa-cp.c +++ b/gcc/ipa-cp.c @@ -2631,7 +2631,8 @@ good_cloning_opportunity_p (struct cgraph_node *node, int time_benefit, struct ipa_node_params *info = IPA_NODE_REF (node); if (max_count > profile_count::zero ()) { - int factor = RDIV (count_sum.probability_in (max_count) + int factor = RDIV (count_sum.probability_in + (max_count).to_reg_br_prob_base () * 1000, REG_BR_PROB_BASE); int64_t evaluation = (((int64_t) time_benefit * factor) / size_cost); diff --git a/gcc/ipa-split.c b/gcc/ipa-split.c index 0af31d68353..e3759d6c50e 100644 --- a/gcc/ipa-split.c +++ b/gcc/ipa-split.c @@ -1292,9 +1292,7 @@ split_function (basic_block return_bb, struct split_point *split_point, break; } } - e = make_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); - e->probability = REG_BR_PROB_BASE; - e->count = new_return_bb->count; + e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); add_bb_to_loop (new_return_bb, current_loops->tree_root); bitmap_set_bit (split_point->split_bbs, new_return_bb->index); retbnd = find_retbnd (return_bb); @@ -1527,11 +1525,9 @@ split_function (basic_block return_bb, struct split_point *split_point, } else { - e = make_edge (call_bb, return_bb, - return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun) - ? 0 : EDGE_FALLTHRU); - e->count = call_bb->count; - e->probability = REG_BR_PROB_BASE; + e = make_single_succ_edge (call_bb, return_bb, + return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun) + ? 0 : EDGE_FALLTHRU); /* If there is return basic block, see what value we need to store return value into and put call just before it. */ diff --git a/gcc/ipa-utils.c b/gcc/ipa-utils.c index 73151248234..65eed77c930 100644 --- a/gcc/ipa-utils.c +++ b/gcc/ipa-utils.c @@ -598,7 +598,8 @@ ipa_merge_profiles (struct cgraph_node *dst, } } int prob = direct->count.probability_in (direct->count - + indirect->count); + + indirect->count). + to_reg_br_prob_base (); direct->frequency = RDIV (freq * prob, REG_BR_PROB_BASE); indirect->frequency = RDIV (freq * (REG_BR_PROB_BASE - prob), REG_BR_PROB_BASE); @@ -616,7 +617,8 @@ ipa_merge_profiles (struct cgraph_node *dst, e2->speculative_call_info (direct, indirect, ref); e->count = count; e->frequency = freq; - int prob = direct->count.probability_in (e->count); + int prob = direct->count.probability_in (e->count) + .to_reg_br_prob_base (); e->make_speculative (direct->callee, direct->count, RDIV (freq * prob, REG_BR_PROB_BASE)); } diff --git a/gcc/loop-doloop.c b/gcc/loop-doloop.c index 05daabb22fb..13bf8cf2ecf 100644 --- a/gcc/loop-doloop.c +++ b/gcc/loop-doloop.c @@ -356,7 +356,8 @@ add_test (rtx cond, edge *e, basic_block dest) op0 = force_operand (op0, NULL_RTX); op1 = force_operand (op1, NULL_RTX); label = block_label (dest); - do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL, label, -1); + do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL, label, + profile_probability::uninitialized ()); jump = get_last_insn (); if (!jump || !JUMP_P (jump)) @@ -575,10 +576,11 @@ doloop_modify (struct loop *loop, struct niter_desc *desc, add_reg_note (jump_insn, REG_NONNEG, NULL_RTX); /* Update the REG_BR_PROB note. */ - if (true_prob_val) + if (true_prob_val && desc->in_edge->probability.initialized_p ()) { /* Seems safer to use the branch probability. */ - add_int_reg_note (jump_insn, REG_BR_PROB, desc->in_edge->probability); + add_int_reg_note (jump_insn, REG_BR_PROB, + desc->in_edge->probability.to_reg_br_prob_base ()); } } diff --git a/gcc/loop-unroll.c b/gcc/loop-unroll.c index e39766c2a79..5e2e77a2327 100644 --- a/gcc/loop-unroll.c +++ b/gcc/loop-unroll.c @@ -774,7 +774,8 @@ split_edge_and_insert (edge e, rtx_insn *insns) static rtx_insn * compare_and_jump_seq (rtx op0, rtx op1, enum rtx_code comp, - rtx_code_label *label, int prob, rtx_insn *cinsn) + rtx_code_label *label, profile_probability prob, + rtx_insn *cinsn) { rtx_insn *seq; rtx_jump_insn *jump; @@ -808,12 +809,14 @@ compare_and_jump_seq (rtx op0, rtx op1, enum rtx_code comp, op0 = force_operand (op0, NULL_RTX); op1 = force_operand (op1, NULL_RTX); do_compare_rtx_and_jump (op0, op1, comp, 0, - mode, NULL_RTX, NULL, label, -1); + mode, NULL_RTX, NULL, label, + profile_probability::uninitialized ()); jump = as_a <rtx_jump_insn *> (get_last_insn ()); jump->set_jump_target (label); LABEL_NUSES (label)++; } - add_int_reg_note (jump, REG_BR_PROB, prob); + if (prob.initialized_p ()) + add_int_reg_note (jump, REG_BR_PROB, prob.to_reg_br_prob_base ()); seq = get_insns (); end_sequence (); @@ -857,7 +860,8 @@ unroll_loop_runtime_iterations (struct loop *loop) { rtx old_niter, niter, tmp; rtx_insn *init_code, *branch_code; - unsigned i, j, p; + unsigned i, j; + profile_probability p; basic_block preheader, *body, swtch, ezc_swtch = NULL; int may_exit_copy, iter_freq, new_freq; profile_count iter_count, new_count; @@ -989,7 +993,7 @@ unroll_loop_runtime_iterations (struct loop *loop) /* Create item for switch. */ j = n_peel - i - (extra_zero_check ? 0 : 1); - p = REG_BR_PROB_BASE / (i + 2); + p = profile_probability::always ().apply_scale (1, i + 2); preheader = split_edge (loop_preheader_edge (loop)); /* Add in frequency/count of edge from switch block. */ @@ -1006,7 +1010,7 @@ unroll_loop_runtime_iterations (struct loop *loop) swtch = split_edge_and_insert (single_pred_edge (swtch), branch_code); set_immediate_dominator (CDI_DOMINATORS, preheader, swtch); - single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p; + single_succ_edge (swtch)->probability = p.invert (); single_succ_edge (swtch)->count = new_count; new_freq += iter_freq; new_count += iter_count; @@ -1021,7 +1025,7 @@ unroll_loop_runtime_iterations (struct loop *loop) if (extra_zero_check) { /* Add branch for zero iterations. */ - p = REG_BR_PROB_BASE / (max_unroll + 1); + p = profile_probability::always ().apply_scale (1, max_unroll + 1); swtch = ezc_swtch; preheader = split_edge (loop_preheader_edge (loop)); /* Recompute frequency/count adjustments since initial peel copy may @@ -1039,7 +1043,7 @@ unroll_loop_runtime_iterations (struct loop *loop) swtch = split_edge_and_insert (single_succ_edge (swtch), branch_code); set_immediate_dominator (CDI_DOMINATORS, preheader, swtch); - single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p; + single_succ_edge (swtch)->probability = p.invert (); single_succ_edge (swtch)->count -= iter_count; e = make_edge (swtch, preheader, single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP); diff --git a/gcc/lra-constraints.c b/gcc/lra-constraints.c index c6465c75523..2bb68261bb9 100644 --- a/gcc/lra-constraints.c +++ b/gcc/lra-constraints.c @@ -6482,7 +6482,8 @@ lra_inheritance (void) e = find_fallthru_edge (bb->succs); if (! e) break; - if (e->probability < EBB_PROBABILITY_CUTOFF) + if (e->probability.initialized_p () + && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF) break; bb = bb->next_bb; } diff --git a/gcc/lto-streamer-in.c b/gcc/lto-streamer-in.c index ccc8cf602df..ec47fe4f165 100644 --- a/gcc/lto-streamer-in.c +++ b/gcc/lto-streamer-in.c @@ -754,12 +754,12 @@ input_cfg (struct lto_input_block *ib, struct data_in *data_in, unsigned int dest_index; unsigned int edge_flags; basic_block dest; - int probability; + profile_probability probability; profile_count count; edge e; dest_index = streamer_read_uhwi (ib); - probability = (int) streamer_read_hwi (ib); + probability = profile_probability::stream_in (ib); count = profile_count::stream_in (ib).apply_scale (count_materialization_scale, REG_BR_PROB_BASE); edge_flags = streamer_read_uhwi (ib); diff --git a/gcc/lto-streamer-out.c b/gcc/lto-streamer-out.c index 9378d5cf7b4..f35dd020a51 100644 --- a/gcc/lto-streamer-out.c +++ b/gcc/lto-streamer-out.c @@ -1860,7 +1860,7 @@ output_cfg (struct output_block *ob, struct function *fn) FOR_EACH_EDGE (e, ei, bb->succs) { streamer_write_uhwi (ob, e->dest->index); - streamer_write_hwi (ob, e->probability); + e->probability.stream_out (ob); e->count.stream_out (ob); streamer_write_uhwi (ob, e->flags); } diff --git a/gcc/mcf.c b/gcc/mcf.c index bb4b4f9ef76..67088506dac 100644 --- a/gcc/mcf.c +++ b/gcc/mcf.c @@ -1231,12 +1231,15 @@ adjust_cfg_counts (fixup_graph_type *fixup_graph) } if (bb_gcov_count (bb)) - e->probability = RDIV (REG_BR_PROB_BASE * edge_gcov_count (e), - bb_gcov_count (bb)); + e->probability = profile_probability::probability_in_gcov_type + (edge_gcov_count (e), bb_gcov_count (bb)); if (dump_file) - fprintf (dump_file, " = %" PRId64 "\t(%.1f%%)\n", - edge_gcov_count (e), - e->probability * 100.0 / REG_BR_PROB_BASE); + { + fprintf (dump_file, " = %" PRId64 "\t", + edge_gcov_count (e)); + e->probability.dump (dump_file); + fprintf (dump_file, "\n"); + } } } @@ -1251,31 +1254,8 @@ adjust_cfg_counts (fixup_graph_type *fixup_graph) if (bb_gcov_count (bb)) { FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = RDIV (REG_BR_PROB_BASE * edge_gcov_count (e), - bb_gcov_count (bb)); - } - else - { - int total = 0; - FOR_EACH_EDGE (e, ei, bb->succs) - if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE))) - total++; - if (total) - { - FOR_EACH_EDGE (e, ei, bb->succs) - { - if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE))) - e->probability = REG_BR_PROB_BASE / total; - else - e->probability = 0; - } - } - else - { - total += EDGE_COUNT (bb->succs); - FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = REG_BR_PROB_BASE / total; - } + e->probability = profile_probability::probability_in_gcov_type + (edge_gcov_count (e), bb_gcov_count (bb)); } } diff --git a/gcc/modulo-sched.c b/gcc/modulo-sched.c index 005c6e596ad..4289738e27d 100644 --- a/gcc/modulo-sched.c +++ b/gcc/modulo-sched.c @@ -1713,12 +1713,13 @@ sms_schedule (void) rtx comp_rtx = gen_rtx_GT (VOIDmode, count_reg, gen_int_mode (stage_count, GET_MODE (count_reg))); - unsigned prob = (PROB_SMS_ENOUGH_ITERATIONS - * REG_BR_PROB_BASE) / 100; + profile_probability prob = profile_probability::guessed_always () + .apply_scale (PROB_SMS_ENOUGH_ITERATIONS, 100); loop_version (loop, comp_rtx, &condition_bb, - prob, REG_BR_PROB_BASE - prob, - prob, REG_BR_PROB_BASE - prob, + prob, prob.invert (), + prob.to_reg_br_prob_base (), + prob.invert ().to_reg_br_prob_base (), true); } diff --git a/gcc/omp-expand.c b/gcc/omp-expand.c index 53d3d3f84fd..929c53078d5 100644 --- a/gcc/omp-expand.c +++ b/gcc/omp-expand.c @@ -1741,9 +1741,9 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi, entry_bb); } ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE); - ne->probability = REG_BR_PROB_BASE / 2000 - 1; + ne->probability = profile_probability::very_unlikely (); e->flags = EDGE_TRUE_VALUE; - e->probability = REG_BR_PROB_BASE - ne->probability; + e->probability = ne->probability.invert (); if (l2_dom_bb == NULL) l2_dom_bb = entry_bb; entry_bb = e->dest; @@ -1920,7 +1920,7 @@ extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb, if (i < fd->collapse - 1) { e = make_edge (last_bb, bb, EDGE_FALSE_VALUE); - e->probability = REG_BR_PROB_BASE / 8; + e->probability = profile_probability::guessed_always ().apply_scale (1, 8); t = fd->loops[i + 1].n1; t = force_gimple_operand_gsi (&gsi, t, @@ -1961,7 +1961,7 @@ extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb, stmt = gimple_build_cond_empty (t); gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); e = make_edge (bb, body_bb, EDGE_TRUE_VALUE); - e->probability = REG_BR_PROB_BASE * 7 / 8; + e->probability = profile_probability::guessed_always ().apply_scale (7, 8); } else make_edge (bb, body_bb, EDGE_FALLTHRU); @@ -2219,8 +2219,8 @@ expand_omp_ordered_sink (gimple_stmt_iterator *gsi, struct omp_for_data *fd, GSI_CONTINUE_LINKING); gsi_insert_after (gsi, gimple_build_cond_empty (cond), GSI_NEW_STMT); edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE); - e3->probability = REG_BR_PROB_BASE / 8; - e1->probability = REG_BR_PROB_BASE - e3->probability; + e3->probability = profile_probability::guessed_always ().apply_scale (1, 8); + e1->probability = e3->probability.invert (); e1->flags = EDGE_TRUE_VALUE; set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src); @@ -2373,9 +2373,9 @@ expand_omp_for_ordered_loops (struct omp_for_data *fd, tree *counts, remove_edge (e1); make_edge (body_bb, new_header, EDGE_FALLTHRU); e3->flags = EDGE_FALSE_VALUE; - e3->probability = REG_BR_PROB_BASE / 8; + e3->probability = profile_probability::guessed_always ().apply_scale (1, 8); e1 = make_edge (new_header, new_body, EDGE_TRUE_VALUE); - e1->probability = REG_BR_PROB_BASE - e3->probability; + e1->probability = e3->probability.invert (); set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb); set_immediate_dominator (CDI_DOMINATORS, new_body, new_header); @@ -3149,8 +3149,8 @@ expand_omp_for_generic (struct omp_region *region, e->flags = EDGE_TRUE_VALUE; if (e) { - e->probability = REG_BR_PROB_BASE * 7 / 8; - find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8; + e->probability = profile_probability::guessed_always ().apply_scale (7, 8); + find_edge (cont_bb, l2_bb)->probability = e->probability.invert (); } else { @@ -3351,9 +3351,9 @@ expand_omp_for_static_nochunk (struct omp_region *region, ep = split_block (entry_bb, cond_stmt); ep->flags = EDGE_TRUE_VALUE; entry_bb = ep->dest; - ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); + ep->probability = profile_probability::very_likely (); ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE); - ep->probability = REG_BR_PROB_BASE / 2000 - 1; + ep->probability = profile_probability::very_unlikely (); if (gimple_in_ssa_p (cfun)) { int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; @@ -3634,10 +3634,10 @@ expand_omp_for_static_nochunk (struct omp_region *region, /* Connect all the blocks. */ ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE); - ep->probability = REG_BR_PROB_BASE / 4 * 3; + ep->probability = profile_probability::guessed_always ().apply_scale (3, 4); ep = find_edge (entry_bb, second_bb); ep->flags = EDGE_TRUE_VALUE; - ep->probability = REG_BR_PROB_BASE / 4; + ep->probability = profile_probability::guessed_always ().apply_scale (1, 4); find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE; find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE; @@ -3835,9 +3835,9 @@ expand_omp_for_static_chunk (struct omp_region *region, se = split_block (entry_bb, cond_stmt); se->flags = EDGE_TRUE_VALUE; entry_bb = se->dest; - se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); + se->probability = profile_probability::very_likely (); se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE); - se->probability = REG_BR_PROB_BASE / 2000 - 1; + se->probability = profile_probability::very_unlikely (); if (gimple_in_ssa_p (cfun)) { int dest_idx = find_edge (iter_part_bb, fin_bb)->dest_idx; @@ -4448,8 +4448,8 @@ expand_cilk_for (struct omp_region *region, struct omp_for_data *fd) } ne->flags = EDGE_FALSE_VALUE; - e->probability = REG_BR_PROB_BASE * 7 / 8; - ne->probability = REG_BR_PROB_BASE / 8; + e->probability = profile_probability::guessed_always ().apply_scale (7, 8); + ne->probability = e->probability.invert (); set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); @@ -4810,8 +4810,8 @@ expand_omp_simd (struct omp_region *region, struct omp_for_data *fd) } ne->flags = EDGE_FALSE_VALUE; - e->probability = REG_BR_PROB_BASE * 7 / 8; - ne->probability = REG_BR_PROB_BASE / 8; + e->probability = profile_probability::guessed_always ().apply_scale (7, 8); + ne->probability = e->probability.invert (); set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); @@ -4824,8 +4824,10 @@ expand_omp_simd (struct omp_region *region, struct omp_for_data *fd) gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT); make_edge (entry_bb, l2_bb, EDGE_FALSE_VALUE); FALLTHRU_EDGE (entry_bb)->flags = EDGE_TRUE_VALUE; - FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE * 7 / 8; - BRANCH_EDGE (entry_bb)->probability = REG_BR_PROB_BASE / 8; + FALLTHRU_EDGE (entry_bb)->probability + = profile_probability::guessed_always ().apply_scale (7, 8); + BRANCH_EDGE (entry_bb)->probability + = FALLTHRU_EDGE (entry_bb)->probability.invert (); l2_dom_bb = entry_bb; } set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); @@ -5018,9 +5020,9 @@ expand_omp_taskloop_for_outer (struct omp_region *region, gsi = gsi_last_bb (exit_bb); gsi_remove (&gsi, true); - FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE; + FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always (); remove_edge (BRANCH_EDGE (entry_bb)); - FALLTHRU_EDGE (cont_bb)->probability = REG_BR_PROB_BASE; + FALLTHRU_EDGE (cont_bb)->probability = profile_probability::always (); remove_edge (BRANCH_EDGE (cont_bb)); set_immediate_dominator (CDI_DOMINATORS, exit_bb, cont_bb); set_immediate_dominator (CDI_DOMINATORS, region->entry, @@ -5208,7 +5210,7 @@ expand_omp_taskloop_for_inner (struct omp_region *region, gsi = gsi_last_bb (exit_bb); gsi_remove (&gsi, true); - FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE; + FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always (); if (!broken_loop) remove_edge (BRANCH_EDGE (entry_bb)); else @@ -6604,8 +6606,11 @@ expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb, e = single_succ_edge (store_bb); e->flags &= ~EDGE_FALLTHRU; e->flags |= EDGE_FALSE_VALUE; + /* Expect no looping. */ + e->probability = profile_probability::guessed_always (); e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE); + e->probability = profile_probability::guessed_never (); /* Copy the new value to loadedi (we already did that before the condition if we are not in SSA). */ diff --git a/gcc/omp-simd-clone.c b/gcc/omp-simd-clone.c index 3aa8b131565..a1a563e8094 100644 --- a/gcc/omp-simd-clone.c +++ b/gcc/omp-simd-clone.c @@ -1151,8 +1151,7 @@ simd_clone_adjust (struct cgraph_node *node) if (incr_bb) { - edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); - e->probability = REG_BR_PROB_BASE; + make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); gsi = gsi_last_bb (incr_bb); iter2 = make_ssa_name (iter); g = gimple_build_assign (iter2, PLUS_EXPR, iter1, @@ -1264,7 +1263,10 @@ simd_clone_adjust (struct cgraph_node *node) redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); - make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); + edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); + + /* FIXME: Do we need to distribute probabilities for the conditional? */ + new_e->probability = profile_probability::guessed_never (); /* The successor of incr_bb is already pointing to latch_bb; just change the flags. make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ diff --git a/gcc/optabs.c b/gcc/optabs.c index b69e75943cc..9598adf84bf 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -693,7 +693,8 @@ expand_doubleword_shift (machine_mode op1_mode, optab binoptab, NO_DEFER_POP; do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode, - 0, 0, subword_label, -1); + 0, 0, subword_label, + profile_probability::uninitialized ()); OK_DEFER_POP; if (!expand_superword_shift (binoptab, outof_input, superword_op1, @@ -3187,7 +3188,8 @@ expand_abs (machine_mode mode, rtx op0, rtx target, NO_DEFER_POP; do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode, - NULL_RTX, NULL, op1, -1); + NULL_RTX, NULL, op1, + profile_probability::uninitialized ()); op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab, target, target, 0); @@ -3979,7 +3981,8 @@ prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode, we can do the branch. */ static void -emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob) +emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, + profile_probability prob) { machine_mode optab_mode; enum mode_class mclass; @@ -3994,13 +3997,13 @@ emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob) gcc_assert (insn_operand_matches (icode, 0, test)); insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0), XEXP (test, 1), label)); - if (prob != -1 + if (prob.initialized_p () && profile_status_for_fn (cfun) != PROFILE_ABSENT && insn && JUMP_P (insn) && any_condjump_p (insn) && !find_reg_note (insn, REG_BR_PROB, 0)) - add_int_reg_note (insn, REG_BR_PROB, prob); + add_int_reg_note (insn, REG_BR_PROB, prob.to_reg_br_prob_base ()); } /* Generate code to compare X with Y so that the condition codes are @@ -4025,7 +4028,7 @@ emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob) void emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size, machine_mode mode, int unsignedp, rtx label, - int prob) + profile_probability prob) { rtx op0 = x, op1 = y; rtx test; @@ -5856,7 +5859,8 @@ expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) /* Mark this jump predicted not taken. */ emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx, - GET_MODE (success), 1, label, 0); + GET_MODE (success), 1, label, + profile_probability::guessed_never ()); return true; } diff --git a/gcc/optabs.h b/gcc/optabs.h index 728b866f08d..07d07fe8366 100644 --- a/gcc/optabs.h +++ b/gcc/optabs.h @@ -247,7 +247,9 @@ extern rtx prepare_operand (enum insn_code, rtx, int, machine_mode, /* Emit a pair of rtl insns to compare two rtx's and to jump to a label if the comparison is true. */ extern void emit_cmp_and_jump_insns (rtx, rtx, enum rtx_code, rtx, - machine_mode, int, rtx, int prob=-1); + machine_mode, int, rtx, + profile_probability prob + = profile_probability::uninitialized ()); /* Generate code to indirectly jump to a location given in the rtx LOC. */ extern void emit_indirect_jump (rtx); diff --git a/gcc/predict.c b/gcc/predict.c index 71d6d0a5bb1..4ee36ffc8c3 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -58,6 +58,7 @@ along with GCC; see the file COPYING3. If not see #include "ipa-utils.h" #include "gimple-pretty-print.h" #include "selftest.h" +#include "cfgrtl.h" /* Enum with reasons why a predictor is ignored. */ @@ -404,11 +405,11 @@ optimize_loop_nest_for_size_p (struct loop *loop) bool predictable_edge_p (edge e) { - if (profile_status_for_fn (cfun) == PROFILE_ABSENT) + if (!e->probability.initialized_p ()) return false; - if ((e->probability + if ((e->probability.to_reg_br_prob_base () <= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100) - || (REG_BR_PROB_BASE - e->probability + || (REG_BR_PROB_BASE - e->probability.to_reg_br_prob_base () <= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100)) return true; return false; @@ -539,7 +540,7 @@ probability_reliable_p (int prob) bool edge_probability_reliable_p (const_edge e) { - return probability_reliable_p (e->probability); + return e->probability.reliable_p (); } /* Same predicate as edge_probability_reliable_p, working on notes. */ @@ -859,12 +860,13 @@ set_even_probabilities (basic_block bb, if (!unlikely_executed_edge_p (e)) { if (unlikely_edges != NULL && unlikely_edges->contains (e)) - e->probability = PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_unlikely (); else - e->probability = (REG_BR_PROB_BASE + c / 2) / c; + e->probability = profile_probability::guessed_always () + .apply_scale (1, c); } else - e->probability = 0; + e->probability = profile_probability::never (); } /* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB @@ -971,20 +973,23 @@ combine_predictions_for_insn (rtx_insn *insn, basic_block bb) conditional jump. */ if (!single_succ_p (bb)) { - BRANCH_EDGE (bb)->probability = combined_probability; + BRANCH_EDGE (bb)->probability + = profile_probability::from_reg_br_prob_base (combined_probability); FALLTHRU_EDGE (bb)->probability - = REG_BR_PROB_BASE - combined_probability; + = BRANCH_EDGE (bb)->probability.invert (); } } else if (!single_succ_p (bb)) { int prob = XINT (prob_note, 0); - BRANCH_EDGE (bb)->probability = prob; - FALLTHRU_EDGE (bb)->probability = REG_BR_PROB_BASE - prob; + BRANCH_EDGE (bb)->probability + = profile_probability::from_reg_br_prob_base (prob); + FALLTHRU_EDGE (bb)->probability + = BRANCH_EDGE (bb)->probability.invert (); } else - single_succ_edge (bb)->probability = REG_BR_PROB_BASE; + single_succ_edge (bb)->probability = profile_probability::always (); } /* Edge prediction hash traits. */ @@ -1129,6 +1134,8 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) if (!first) first = e; } + else if (!e->probability.initialized_p ()) + e->probability = profile_probability::never (); /* When there is no successor or only one choice, prediction is easy. @@ -1173,8 +1180,8 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) nedges, bb->index); FOR_EACH_EDGE (e, ei, bb->succs) if (!unlikely_executed_edge_p (e)) - dump_prediction (dump_file, PRED_COMBINED, e->probability, - bb, REASON_NONE, e); + dump_prediction (dump_file, PRED_COMBINED, + e->probability.to_reg_br_prob_base (), bb, REASON_NONE, e); } } return; @@ -1284,8 +1291,9 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) if (!bb->count.initialized_p () && !dry_run) { - first->probability = combined_probability; - second->probability = REG_BR_PROB_BASE - combined_probability; + first->probability + = profile_probability::from_reg_br_prob_base (combined_probability); + second->probability = first->probability.invert (); } } @@ -3042,7 +3050,7 @@ propagate_freq (basic_block head, bitmap tovisit) * BLOCK_INFO (e->src)->frequency / REG_BR_PROB_BASE); */ - sreal tmp = e->probability; + sreal tmp = e->probability.to_reg_br_prob_base (); tmp *= BLOCK_INFO (e->src)->frequency; tmp *= real_inv_br_prob_base; frequency += tmp; @@ -3074,7 +3082,7 @@ propagate_freq (basic_block head, bitmap tovisit) = ((e->probability * BLOCK_INFO (bb)->frequency) / REG_BR_PROB_BASE); */ - sreal tmp = e->probability; + sreal tmp = e->probability.to_reg_br_prob_base (); tmp *= BLOCK_INFO (bb)->frequency; EDGE_INFO (e)->back_edge_prob = tmp * real_inv_br_prob_base; } @@ -3534,7 +3542,7 @@ estimate_bb_frequencies (bool force) mark_dfs_back_edges (); single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))->probability = - REG_BR_PROB_BASE; + profile_probability::always (); /* Set up block info for each basic block. */ alloc_aux_for_blocks (sizeof (block_info)); @@ -3546,7 +3554,8 @@ estimate_bb_frequencies (bool force) FOR_EACH_EDGE (e, ei, bb->succs) { - EDGE_INFO (e)->back_edge_prob = e->probability; + EDGE_INFO (e)->back_edge_prob + = e->probability.to_reg_br_prob_base (); EDGE_INFO (e)->back_edge_prob *= real_inv_br_prob_base; } } @@ -3898,16 +3907,18 @@ void force_edge_cold (edge e, bool impossible) { profile_count count_sum = profile_count::zero (); - int prob_sum = 0; + profile_probability prob_sum = profile_probability::never (); edge_iterator ei; edge e2; profile_count old_count = e->count; - int old_probability = e->probability; - int prob_scale = REG_BR_PROB_BASE; + profile_probability old_probability = e->probability; bool uninitialized_exit = false; + profile_probability goal = (impossible ? profile_probability::never () + : profile_probability::very_unlikely ()); + /* If edge is already improbably or cold, just return. */ - if (e->probability <= (impossible ? PROB_VERY_UNLIKELY : 0) + if (e->probability <= goal && (!impossible || e->count == profile_count::zero ())) return; FOR_EACH_EDGE (e2, ei, e->src->succs) @@ -3917,24 +3928,26 @@ force_edge_cold (edge e, bool impossible) count_sum += e2->count; else uninitialized_exit = true; - prob_sum += e2->probability; + if (e2->probability.initialized_p ()) + prob_sum += e2->probability; } /* If there are other edges out of e->src, redistribute probabilitity there. */ - if (prob_sum) + if (prob_sum > profile_probability::never ()) { - e->probability - = MIN (e->probability, impossible ? 0 : PROB_VERY_UNLIKELY); + if (!(e->probability < goal)) + e->probability = goal; if (impossible) e->count = profile_count::zero (); - else if (old_probability) - e->count = e->count.apply_scale (e->probability, old_probability); + else if (old_probability > profile_probability::never ()) + e->count = e->count.apply_probability (e->probability + / old_probability); else e->count = e->count.apply_scale (1, REG_BR_PROB_BASE); - prob_scale = RDIV ((REG_BR_PROB_BASE - e->probability) * REG_BR_PROB_BASE, - prob_sum); + profile_probability prob_comp = prob_sum / e->probability.invert (); + if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, "Making edge %i->%i %s by redistributing " "probability to other edges.\n", @@ -3946,30 +3959,39 @@ force_edge_cold (edge e, bool impossible) { if (count_sum > 0) e2->count.apply_scale (count_sum2, count_sum); - e2->probability = RDIV (e2->probability * prob_scale, - REG_BR_PROB_BASE); + e2->probability /= prob_comp; } + if (current_ir_type () != IR_GIMPLE) + update_br_prob_note (e->src); } /* If all edges out of e->src are unlikely, the basic block itself is unlikely. */ else { - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); + if (current_ir_type () != IR_GIMPLE) + update_br_prob_note (e->src); if (e->src->count == profile_count::zero ()) return; if (count_sum == profile_count::zero () && !uninitialized_exit && impossible) { bool found = false; - for (gimple_stmt_iterator gsi = gsi_start_bb (e->src); - !gsi_end_p (gsi); gsi_next (&gsi)) - { - if (stmt_can_terminate_bb_p (gsi_stmt (gsi))) - { - found = true; - break; - } - } + if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun)) + ; + else if (current_ir_type () == IR_GIMPLE) + for (gimple_stmt_iterator gsi = gsi_start_bb (e->src); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + if (stmt_can_terminate_bb_p (gsi_stmt (gsi))) + { + found = true; + break; + } + } + /* FIXME: Implement RTL path. */ + else + found = true; if (!found) { if (dump_file && (dump_flags & TDF_DETAILS)) @@ -3989,7 +4011,8 @@ force_edge_cold (edge e, bool impossible) This in general is difficult task to do, but handle special case when BB has only one predecestor. This is common case when we are updating after loop transforms. */ - if (!prob_sum && count_sum == profile_count::zero () + if (!(prob_sum > profile_probability::never ()) + && count_sum == profile_count::zero () && single_pred_p (e->src) && e->src->frequency > (impossible ? 0 : 1)) { int old_frequency = e->src->frequency; diff --git a/gcc/profile-count.c b/gcc/profile-count.c index e64d1b56421..4d22428a195 100644 --- a/gcc/profile-count.c +++ b/gcc/profile-count.c @@ -31,6 +31,8 @@ along with GCC; see the file COPYING3. If not see #include "data-streamer.h" #include "cgraph.h" +/* Dump THIS to F. */ + void profile_count::dump (FILE *f) const { @@ -39,43 +41,53 @@ profile_count::dump (FILE *f) const else { fprintf (f, "%" PRId64, m_val); - if (m_quality == count_adjusted) - fprintf (f, "(adjusted)"); - else if (m_quality == count_afdo) - fprintf (f, "(auto FDO)"); - else if (m_quality == count_guessed) - fprintf (f, "(guessed)"); + if (m_quality == profile_adjusted) + fprintf (f, " (adjusted)"); + else if (m_quality == profile_afdo) + fprintf (f, " (auto FDO)"); + else if (m_quality == profile_guessed) + fprintf (f, " (guessed)"); } } +/* Dump THIS to stderr. */ + void profile_count::debug () const { dump (stderr); + fprintf (stderr, "\n"); } +/* Return true if THIS differs from OTHER; tolerate small diferences. */ + bool profile_count::differs_from_p (profile_count other) const { if (!initialized_p () || !other.initialized_p ()) return false; - if (m_val - other.m_val < 100 || other.m_val - m_val < 100) + if ((uint64_t)m_val - (uint64_t)other.m_val < 100 + || (uint64_t)other.m_val - (uint64_t)m_val < 100) return false; if (!other.m_val) return true; - int64_t ratio = m_val * 100 / other.m_val; + int64_t ratio = (int64_t)m_val * 100 / other.m_val; return ratio < 99 || ratio > 101; } +/* Stream THIS from IB. */ + profile_count profile_count::stream_in (struct lto_input_block *ib) { profile_count ret; ret.m_val = streamer_read_gcov_count (ib); - ret.m_quality = (profile_count_quality) streamer_read_uhwi (ib); + ret.m_quality = (profile_quality) streamer_read_uhwi (ib); return ret; } +/* Stream THIS to OB. */ + void profile_count::stream_out (struct output_block *ob) { @@ -83,9 +95,102 @@ profile_count::stream_out (struct output_block *ob) streamer_write_uhwi (ob, m_quality); } +/* Stream THIS to OB. */ + void profile_count::stream_out (struct lto_output_stream *ob) { streamer_write_gcov_count_stream (ob, m_val); streamer_write_uhwi_stream (ob, m_quality); } + +/* Dump THIS to F. */ + +void +profile_probability::dump (FILE *f) const +{ + if (!initialized_p ()) + fprintf (f, "uninitialized"); + else + { + /* Make difference between 0.00 as a roundoff error and actual 0. + Similarly for 1. */ + if (m_val == 0) + fprintf (f, "never"); + else if (m_val == max_probability) + fprintf (f, "always"); + else + fprintf (f, "%3.1f%%", (double)m_val * 100 / max_probability); + if (m_quality == profile_adjusted) + fprintf (f, " (adjusted)"); + else if (m_quality == profile_afdo) + fprintf (f, " (auto FDO)"); + else if (m_quality == profile_guessed) + fprintf (f, " (guessed)"); + } +} + +/* Dump THIS to stderr. */ + +void +profile_probability::debug () const +{ + dump (stderr); + fprintf (stderr, "\n"); +} + +/* Return true if THIS differs from OTHER; tolerate small diferences. */ + +bool +profile_probability::differs_from_p (profile_probability other) const +{ + if (!initialized_p () || !other.initialized_p ()) + return false; + if ((uint64_t)m_val - (uint64_t)other.m_val < 10 + || (uint64_t)other.m_val - (uint64_t)m_val < 10) + return false; + if (!other.m_val) + return true; + int64_t ratio = m_val * 100 / other.m_val; + return ratio < 99 || ratio > 101; +} + +/* Return true if THIS differs significantly from OTHER. */ + +bool +profile_probability::differs_lot_from_p (profile_probability other) const +{ + if (!initialized_p () || !other.initialized_p ()) + return false; + uint32_t d = m_val > other.m_val ? m_val - other.m_val : other.m_val - m_val; + return d > max_probability / 2; +} + +/* Stream THIS from IB. */ + +profile_probability +profile_probability::stream_in (struct lto_input_block *ib) +{ + profile_probability ret; + ret.m_val = streamer_read_uhwi (ib); + ret.m_quality = (profile_quality) streamer_read_uhwi (ib); + return ret; +} + +/* Stream THIS to OB. */ + +void +profile_probability::stream_out (struct output_block *ob) +{ + streamer_write_uhwi (ob, m_val); + streamer_write_uhwi (ob, m_quality); +} + +/* Stream THIS to OB. */ + +void +profile_probability::stream_out (struct lto_output_stream *ob) +{ + streamer_write_uhwi_stream (ob, m_val); + streamer_write_uhwi_stream (ob, m_quality); +} diff --git a/gcc/profile-count.h b/gcc/profile-count.h index 0f77e4efc36..461dac6c495 100644 --- a/gcc/profile-count.h +++ b/gcc/profile-count.h @@ -23,19 +23,19 @@ along with GCC; see the file COPYING3. If not see /* Quality of the proflie count. Because gengtype does not support enums inside of clases, this is in global namespace. */ -enum profile_count_quality { +enum profile_quality { /* Profile is based on static branch prediction heuristics. It may or may not reflect the reality. */ - count_guessed = 0, + profile_guessed = 0, /* Profile was determined by autofdo. */ - count_afdo = 1, + profile_afdo = 1, /* Profile was originally based on feedback but it was adjusted by code duplicating optimization. It may not precisely reflect the particular code path. */ - count_adjusted = 2, + profile_adjusted = 2, /* Profile was read from profile feedback or determined by accurate static method. */ - count_read = 3 + profile_precise = 3 }; /* The base value for branch probability notes and edge probabilities. */ @@ -43,6 +43,422 @@ enum profile_count_quality { #define RDIV(X,Y) (((X) + (Y) / 2) / (Y)) +/* Data type to hold probabilities. It implement fixed point arithmetics + with capping so probability is always in range [0,1] and scaling requiring + values greater than 1 needs to be represented otherwise. + + In addition to actual value the quality of profile is tracked and propagated + through all operations. Special value UNINITIALIZED is used for probabilities + that has not been detemrined yet (for example bacause of + -fno-guess-branch-probability) + + Typically probabilities are derived from profile feedback (via + probability_in_gcov_type), autoFDO or guessed statically and then propagated + thorough the compilation. + + Named probabilities are available: + - never (0 probability) + - guessed_never + - very_unlikely (1/2000 probability) + - unlikely (1/5 probablity) + - even (1/2 probability) + - likely (4/5 probability) + - very_likely (1999/2000 probability) + - guessed_always + - always + + Named probabilities except for never/always are assumed to be statically + guessed and thus not necessarily acurate. The difference between never + and guessedn never is that the first one should be used only in case that + well behaving program will very likely not execute the "never" path. + For example if the path is going to abort () call or it exception handling. + + Alawyas and guessted_always probabilities are symmetric. + + For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint + integer arithmetics. Once the code is converted to branch probabiitlies, + these conversions will probably go away because they are lossy. +*/ + +class GTY((user)) profile_probability +{ + /* For now use values in range 0...REG_BR_PROB_BASE. Later we can use full + precision of 30 bits available. */ + + static const int n_bits = 30; + static const uint32_t max_probability = REG_BR_PROB_BASE; + static const uint32_t uninitialized_probability = ((uint32_t) 1 << n_bits) - 1; + + uint32_t m_val : 30; + enum profile_quality m_quality : 2; + + friend class profile_count; +public: + + /* Named probabilities. */ + static profile_probability never () + { + profile_probability ret; + ret.m_val = 0; + ret.m_quality = profile_precise; + return ret; + } + static profile_probability guessed_never () + { + profile_probability ret; + ret.m_val = 0; + ret.m_quality = profile_guessed; + return ret; + } + static profile_probability very_unlikely () + { + /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */ + profile_probability r + = profile_probability::always ().apply_scale (1, 2000); + r.m_val--; + return r; + } + static profile_probability unlikely () + { + /* Be consistent with PROB_VERY_LIKELY in predict.h. */ + profile_probability r + = profile_probability::always ().apply_scale (1, 5); + r.m_val--; + return r; + } + static profile_probability even () + { + return profile_probability::always ().apply_scale (1, 2); + } + static profile_probability very_likely () + { + return profile_probability::always () - very_unlikely (); + } + static profile_probability likely () + { + return profile_probability::always () - unlikely (); + } + static profile_probability guessed_always () + { + profile_probability ret; + ret.m_val = max_probability; + ret.m_quality = profile_guessed; + return ret; + } + static profile_probability always () + { + profile_probability ret; + ret.m_val = max_probability; + ret.m_quality = profile_precise; + return ret; + } + /* Probabilities which has not been initialized. Either because + initialization did not happen yet or because profile is unknown. */ + static profile_probability uninitialized () + { + profile_probability c; + c.m_val = uninitialized_probability; + c.m_quality = profile_guessed; + return c; + } + + + /* Return true if value has been initialized. */ + bool initialized_p () const + { + return m_val != uninitialized_probability; + } + /* Return true if value can be trusted. */ + bool reliable_p () const + { + return initialized_p (); + } + + /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics. + this is mostly to support legacy code and hsould go away. */ + static profile_probability from_reg_br_prob_base (int v) + { + profile_probability ret; + gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE); + ret.m_val = RDIV (v * max_probability, REG_BR_PROB_BASE); + ret.m_quality = profile_guessed; + return ret; + } + int to_reg_br_prob_base () const + { + gcc_checking_assert (initialized_p ()); + return RDIV (m_val * REG_BR_PROB_BASE, max_probability); + } + + /* Return VAL1/VAL2. */ + static profile_probability probability_in_gcov_type + (gcov_type val1, gcov_type val2) + { + profile_probability ret; + gcc_checking_assert (val1 >= 0 && val2 > 0); + if (val1 > val2) + ret.m_val = max_probability; + else + ret.m_val = RDIV (val1 * max_probability, val2); + ret.m_quality = profile_precise; + return ret; + } + + /* Basic operations. */ + bool operator== (const profile_probability &other) const + { + return m_val == other.m_val && m_quality == other.m_quality; + } + profile_probability operator+ (const profile_probability &other) const + { + if (other == profile_probability::never ()) + return *this; + if (*this == profile_probability::never ()) + return other; + if (!initialized_p () || !other.initialized_p ()) + return profile_probability::uninitialized (); + + profile_probability ret; + ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability); + ret.m_quality = MIN (m_quality, other.m_quality); + return ret; + } + profile_probability &operator+= (const profile_probability &other) + { + if (other == profile_probability::never ()) + return *this; + if (*this == profile_probability::never ()) + { + *this = other; + return *this; + } + if (!initialized_p () || !other.initialized_p ()) + return *this = profile_probability::uninitialized (); + else + { + m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability); + m_quality = MIN (m_quality, other.m_quality); + } + return *this; + } + profile_probability operator- (const profile_probability &other) const + { + if (*this == profile_probability::never () + || other == profile_probability::never ()) + return *this; + if (!initialized_p () || !other.initialized_p ()) + return profile_probability::uninitialized (); + profile_probability ret; + ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0; + ret.m_quality = MIN (m_quality, other.m_quality); + return ret; + } + profile_probability &operator-= (const profile_probability &other) + { + if (*this == profile_probability::never () + || other == profile_probability::never ()) + return *this; + if (!initialized_p () || !other.initialized_p ()) + return *this = profile_probability::uninitialized (); + else + { + m_val = m_val >= other.m_val ? m_val - other.m_val : 0; + m_quality = MIN (m_quality, other.m_quality); + } + return *this; + } + profile_probability operator* (const profile_probability &other) const + { + if (*this == profile_probability::never () + || other == profile_probability::never ()) + return profile_probability::never (); + if (!initialized_p () || !other.initialized_p ()) + return profile_probability::uninitialized (); + profile_probability ret; + ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); + ret.m_quality = MIN (m_quality, other.m_quality); + return ret; + } + profile_probability &operator*= (const profile_probability &other) + { + if (*this == profile_probability::never () + || other == profile_probability::never ()) + return *this = profile_probability::never (); + if (!initialized_p () || !other.initialized_p ()) + return *this = profile_probability::uninitialized (); + else + { + m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); + m_quality = MIN (m_quality, other.m_quality); + } + return *this; + } + profile_probability operator/ (const profile_probability &other) const + { + if (*this == profile_probability::never ()) + return profile_probability::never (); + if (!initialized_p () || !other.initialized_p ()) + return profile_probability::uninitialized (); + profile_probability ret; + if (m_val >= other.m_val) + ret.m_val = max_probability; + else if (!m_val) + ret.m_val = 0; + else + { + gcc_checking_assert (other.m_val); + ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability, + other.m_val), + max_probability); + } + ret.m_quality = MIN (m_quality, other.m_quality); + return ret; + } + profile_probability &operator/= (const profile_probability &other) + { + if (*this == profile_probability::never ()) + return *this = profile_probability::never (); + if (!initialized_p () || !other.initialized_p ()) + return *this = profile_probability::uninitialized (); + else + { + if (m_val > other.m_val) + m_val = max_probability; + else if (!m_val) + ; + else + { + gcc_checking_assert (other.m_val); + m_val = MIN (RDIV ((uint64_t)m_val * max_probability, + other.m_val), + max_probability); + } + m_quality = MIN (m_quality, other.m_quality); + } + return *this; + } + + gcov_type apply (gcov_type val) const + { + if (*this == profile_probability::uninitialized ()) + return val / 2; + return RDIV (val * m_val, max_probability); + } + + /* Return 1-*THIS. */ + profile_probability invert () const + { + return profile_probability::always() - *this; + } + + profile_probability combine_with_freq (int freq1, profile_probability other, + int freq2) const + { + profile_probability ret; + + if (*this == profile_probability::uninitialized () + || other == profile_probability::uninitialized ()) + return profile_probability::uninitialized (); + + gcc_checking_assert (freq1 >= 0 && freq2 >= 0); + if (!freq1 && !freq2) + { + ret.m_val = (m_val + other.m_val) / 2; + } + else + ret.m_val = RDIV (m_val * (uint64_t) freq1 + + other.m_val * (uint64_t) freq2, freq1 + freq2); + ret.m_quality = MIN (m_quality, other.m_quality); + return ret; + } + + /* Return *THIS * NUM / DEN. */ + profile_probability apply_scale (int64_t num, int64_t den) const + { + if (*this == profile_probability::never ()) + return *this; + if (!initialized_p ()) + return profile_probability::uninitialized (); + profile_probability ret; + ret.m_val = MIN (RDIV (m_val * num, den), + max_probability); + ret.m_quality = MIN (m_quality, profile_adjusted); + return ret; + } + + /* Return true when the probability of edge is reliable. + + The profile guessing code is good at predicting branch outcome (ie. + taken/not taken), that is predicted right slightly over 75% of time. + It is however notoriously poor on predicting the probability itself. + In general the profile appear a lot flatter (with probabilities closer + to 50%) than the reality so it is bad idea to use it to drive optimization + such as those disabling dynamic branch prediction for well predictable + branches. + + There are two exceptions - edges leading to noreturn edges and edges + predicted by number of iterations heuristics are predicted well. This macro + should be able to distinguish those, but at the moment it simply check for + noreturn heuristic that is only one giving probability over 99% or bellow + 1%. In future we might want to propagate reliability information across the + CFG if we find this information useful on multiple places. */ + + bool probably_reliable_p () const + { + if (m_quality >= profile_adjusted) + return true; + if (!initialized_p ()) + return false; + return m_val < max_probability / 100 + || m_val > max_probability - max_probability / 100; + } + + /* Return false if profile_probability is bogus. */ + bool verify () const + { + if (m_val == uninitialized_probability) + return m_quality == profile_guessed; + else + return m_val <= REG_BR_PROB_BASE; + } + + /* Comparsions are three-state and conservative. False is returned if + the inequality can not be decided. */ + bool operator< (const profile_probability &other) const + { + return initialized_p () && other.initialized_p () && m_val < other.m_val; + } + bool operator> (const profile_probability &other) const + { + return initialized_p () && other.initialized_p () && m_val > other.m_val; + } + + bool operator<= (const profile_probability &other) const + { + return initialized_p () && other.initialized_p () && m_val <= other.m_val; + } + bool operator>= (const profile_probability &other) const + { + return initialized_p () && other.initialized_p () && m_val >= other.m_val; + } + + /* Output THIS to F. */ + void dump (FILE *f) const; + + /* Print THIS to stderr. */ + void debug () const; + + /* Return true if THIS is known to differ significantly from OTHER. */ + bool differs_from_p (profile_probability other) const; + /* Return if difference is greater than 50%. */ + bool differs_lot_from_p (profile_probability other) const; + + /* LTO streaming support. */ + static profile_probability stream_in (struct lto_input_block *); + void stream_out (struct output_block *); + void stream_out (struct lto_output_stream *); +}; + /* Main data type to hold profile counters in GCC. In most cases profile counts originate from profile feedback. They are 64bit integers representing number of executions during the train run. @@ -85,7 +501,7 @@ class GTY(()) profile_count static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1; uint64_t m_val : n_bits; - enum profile_count_quality m_quality : 2; + enum profile_quality m_quality : 2; /* Assume numbers smaller than this to multiply. This is set to make testsuite pass, in future we may implement precise multiplication in higer @@ -108,7 +524,7 @@ public: { profile_count c; c.m_val = uninitialized_count; - c.m_quality = count_guessed; + c.m_quality = profile_guessed; return c; } @@ -120,7 +536,7 @@ public: profile_count ret; gcc_checking_assert (v >= 0 && (uint64_t) v <= max_count); ret.m_val = v; - ret.m_quality = count_read; + ret.m_quality = profile_precise; return ret; } @@ -207,7 +623,7 @@ public: /* Return false if profile_count is bogus. */ bool verify () const { - return m_val != uninitialized_count || m_quality == count_guessed; + return m_val != uninitialized_count || m_quality == profile_guessed; } /* Comparsions are three-state and conservative. False is returned if @@ -237,7 +653,7 @@ public: } bool operator>= (const profile_count &other) const { - return initialized_p () && m_val >= other.m_val; + return initialized_p () && other.initialized_p () && m_val >= other.m_val; } bool operator<= (const gcov_type other) const { @@ -261,7 +677,23 @@ public: return profile_count::uninitialized (); profile_count ret; ret.m_val = RDIV (m_val * prob, REG_BR_PROB_BASE); - ret.m_quality = MIN (m_quality, count_adjusted); + ret.m_quality = MIN (m_quality, profile_adjusted); + return ret; + } + + /* Scale counter according to PROB. */ + profile_count apply_probability (profile_probability prob) const + { + if (*this == profile_count::zero ()) + return *this; + if (prob == profile_probability::never ()) + return profile_count::zero (); + if (!initialized_p ()) + return profile_count::uninitialized (); + profile_count ret; + ret.m_val = RDIV (m_val * prob.m_val, + profile_probability::max_probability); + ret.m_quality = MIN (m_quality, prob.m_quality); return ret; } /* Return *THIS * NUM / DEN. */ @@ -277,7 +709,7 @@ public: gcc_checking_assert ((num <= REG_BR_PROB_BASE || den <= REG_BR_PROB_BASE) || 1); ret.m_val = RDIV (m_val * num, den); - ret.m_quality = MIN (m_quality, count_adjusted); + ret.m_quality = MIN (m_quality, profile_adjusted); return ret; } profile_count apply_scale (profile_count num, profile_count den) const @@ -299,23 +731,27 @@ public: else ret.m_val = RDIV (m_val * RDIV (num.m_val * max_safe_multiplier, den.m_val), max_safe_multiplier); - ret.m_quality = MIN (m_quality, count_adjusted); + ret.m_quality = MIN (m_quality, profile_adjusted); return ret; } /* Return probability of event with counter THIS within event with counter OVERALL. */ - int probability_in (profile_count overall) + profile_probability probability_in (const profile_count overall) const { if (!m_val) - return 0; - if (!initialized_p () || !overall.initialized_p ()) - return REG_BR_PROB_BASE / 2; - if (overall < *this) - return REG_BR_PROB_BASE; - if (!overall.m_val) - return REG_BR_PROB_BASE / 2; - return RDIV (m_val * REG_BR_PROB_BASE, overall.m_val); + return profile_probability::never (); + if (!initialized_p () || !overall.initialized_p () + || !overall.m_val) + return profile_probability::uninitialized (); + profile_probability ret; + if (overall < m_val) + ret.m_val = profile_probability::max_probability; + else + ret.m_val = RDIV (m_val * profile_probability::max_probability, + overall.m_val); + ret.m_quality = MIN (m_quality, overall.m_quality); + return ret; } /* Output THIS to F. */ diff --git a/gcc/profile.c b/gcc/profile.c index 51ca248f5ad..6d40241a37b 100644 --- a/gcc/profile.c +++ b/gcc/profile.c @@ -768,8 +768,8 @@ compute_branch_probabilities (unsigned cfg_checksum, unsigned lineno_checksum) if (bb_gcov_count (bb)) { FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = GCOV_COMPUTE_SCALE (edge_gcov_count (e), - bb_gcov_count (bb)); + e->probability = profile_probability::probability_in_gcov_type + (edge_gcov_count (e), bb_gcov_count (bb)); if (bb->index >= NUM_FIXED_BLOCKS && block_ends_with_condjump_p (bb) && EDGE_COUNT (bb->succs) >= 2) @@ -784,7 +784,7 @@ compute_branch_probabilities (unsigned cfg_checksum, unsigned lineno_checksum) if (!(e->flags & (EDGE_FAKE | EDGE_FALLTHRU))) break; - prob = e->probability; + prob = e->probability.to_reg_br_prob_base (); index = prob * 20 / REG_BR_PROB_BASE; if (index == 20) @@ -810,15 +810,17 @@ compute_branch_probabilities (unsigned cfg_checksum, unsigned lineno_checksum) { FOR_EACH_EDGE (e, ei, bb->succs) if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE))) - e->probability = REG_BR_PROB_BASE / total; + e->probability + = profile_probability::guessed_always ().apply_scale (1, total); else - e->probability = 0; + e->probability = profile_probability::never (); } else { total += EDGE_COUNT (bb->succs); FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = REG_BR_PROB_BASE / total; + e->probability + = profile_probability::guessed_always ().apply_scale (1, total); } if (bb->index >= NUM_FIXED_BLOCKS && block_ends_with_condjump_p (bb) diff --git a/gcc/recog.c b/gcc/recog.c index 63cdc237240..fd4e4605830 100644 --- a/gcc/recog.c +++ b/gcc/recog.c @@ -3460,8 +3460,7 @@ peep2_attempt (basic_block bb, rtx_insn *insn, int match_len, rtx_insn *attempt) flags); nehe->probability = eh_edge->probability; - nfte->probability - = REG_BR_PROB_BASE - nehe->probability; + nfte->probability = nehe->probability.invert (); peep2_do_cleanup_cfg |= purge_dead_edges (nfte->dest); bb = nfte->src; diff --git a/gcc/sched-ebb.c b/gcc/sched-ebb.c index 9123343fa0f..a0422f4b1ba 100644 --- a/gcc/sched-ebb.c +++ b/gcc/sched-ebb.c @@ -648,7 +648,8 @@ schedule_ebbs (void) e = find_fallthru_edge (bb->succs); if (! e) break; - if (e->probability <= probability_cutoff) + if (e->probability.initialized_p () + && e->probability.to_reg_br_prob_base () <= probability_cutoff) break; if (e->dest->flags & BB_DISABLE_SCHEDULE) break; diff --git a/gcc/sched-rgn.c b/gcc/sched-rgn.c index 3bb9356693e..492094e3624 100644 --- a/gcc/sched-rgn.c +++ b/gcc/sched-rgn.c @@ -507,7 +507,8 @@ find_single_block_region (bool ebbs_p) e = find_fallthru_edge (bb->succs); if (! e) break; - if (e->probability <= probability_cutoff) + if (e->probability.initialized_p () + && e->probability.to_reg_br_prob_base () <= probability_cutoff) break; } @@ -1441,7 +1442,11 @@ compute_dom_prob_ps (int bb) FOR_EACH_EDGE (out_edge, out_ei, in_edge->src->succs) bitmap_set_bit (pot_split[bb], EDGE_TO_BIT (out_edge)); - prob[bb] += combine_probabilities (prob[pred_bb], in_edge->probability); + prob[bb] += combine_probabilities + (prob[pred_bb], + in_edge->probability.initialized_p () + ? in_edge->probability.to_reg_br_prob_base () + : 0); // The rounding divide in combine_probabilities can result in an extra // probability increment propagating along 50-50 edges. Eventually when // the edges re-merge, the accumulated probability can go slightly above @@ -3171,8 +3176,10 @@ schedule_region (int rgn) sched_rgn_n_insns += sched_n_insns; realloc_bb_state_array (saved_last_basic_block); f = find_fallthru_edge (last_bb->succs); - if (f && f->probability * 100 / REG_BR_PROB_BASE >= - PARAM_VALUE (PARAM_SCHED_STATE_EDGE_PROB_CUTOFF)) + if (f + && (!f->probability.initialized_p () + || f->probability.to_reg_br_prob_base () * 100 / REG_BR_PROB_BASE >= + PARAM_VALUE (PARAM_SCHED_STATE_EDGE_PROB_CUTOFF))) { memcpy (bb_state[f->dest->index], curr_state, dfa_state_size); diff --git a/gcc/sel-sched-ir.c b/gcc/sel-sched-ir.c index c1e2c24d950..dd72828795d 100644 --- a/gcc/sel-sched-ir.c +++ b/gcc/sel-sched-ir.c @@ -4747,7 +4747,9 @@ compute_succs_info (insn_t insn, short flags) sinfo->probs_ok.safe_push ( /* FIXME: Improve calculation when skipping inner loop to exits. */ - si.bb_end ? si.e1->probability : REG_BR_PROB_BASE); + si.bb_end && si.e1->probability.initialized_p () + ? si.e1->probability.to_reg_br_prob_base () + : REG_BR_PROB_BASE); sinfo->succs_ok_n++; } else @@ -4756,8 +4758,8 @@ compute_succs_info (insn_t insn, short flags) /* Compute all_prob. */ if (!si.bb_end) sinfo->all_prob = REG_BR_PROB_BASE; - else - sinfo->all_prob += si.e1->probability; + else if (si.e1->probability.initialized_p ()) + sinfo->all_prob += si.e1->probability.to_reg_br_prob_base (); sinfo->all_succs_n++; } diff --git a/gcc/stmt.c b/gcc/stmt.c index fdf29a5c837..10d394eee69 100644 --- a/gcc/stmt.c +++ b/gcc/stmt.c @@ -93,9 +93,9 @@ struct case_node tree low; /* Lowest index value for this label */ tree high; /* Highest index value for this label */ tree code_label; /* Label to jump to when node matches */ - int prob; /* Probability of taking this case. */ + profile_probability prob; /* Probability of taking this case. */ /* Probability of reaching subtree rooted at this node */ - int subtree_prob; + profile_probability subtree_prob; }; typedef struct case_node *case_node_ptr; @@ -108,7 +108,8 @@ static void balance_case_nodes (case_node_ptr *, case_node_ptr); static int node_has_low_bound (case_node_ptr, tree); static int node_has_high_bound (case_node_ptr, tree); static int node_is_bounded (case_node_ptr, tree); -static void emit_case_nodes (rtx, case_node_ptr, rtx_code_label *, int, tree); +static void emit_case_nodes (rtx, case_node_ptr, rtx_code_label *, + profile_probability, tree); /* Return the rtx-label that corresponds to a LABEL_DECL, creating it if necessary. */ @@ -704,9 +705,8 @@ expand_naked_return (void) is the probability of jumping to LABEL. */ static void do_jump_if_equal (machine_mode mode, rtx op0, rtx op1, rtx_code_label *label, - int unsignedp, int prob) + int unsignedp, profile_probability prob) { - gcc_assert (prob <= REG_BR_PROB_BASE); do_compare_rtx_and_jump (op0, op1, EQ, unsignedp, mode, NULL_RTX, NULL, label, prob); } @@ -722,7 +722,7 @@ do_jump_if_equal (machine_mode mode, rtx op0, rtx op1, rtx_code_label *label, static struct case_node * add_case_node (struct case_node *head, tree low, tree high, - tree label, int prob, + tree label, profile_probability prob, object_allocator<case_node> &case_node_pool) { struct case_node *r; @@ -859,7 +859,7 @@ expand_switch_as_decision_tree_p (tree range, static void emit_case_decision_tree (tree index_expr, tree index_type, case_node_ptr case_list, rtx_code_label *default_label, - int default_prob) + profile_probability default_prob) { rtx index = expand_normal (index_expr); @@ -902,14 +902,14 @@ emit_case_decision_tree (tree index_expr, tree index_type, /* Return the sum of probabilities of outgoing edges of basic block BB. */ -static int +static profile_probability get_outgoing_edge_probs (basic_block bb) { edge e; edge_iterator ei; - int prob_sum = 0; + profile_probability prob_sum = profile_probability::never (); if (!bb) - return 0; + return profile_probability::never (); FOR_EACH_EDGE (e, ei, bb->succs) prob_sum += e->probability; return prob_sum; @@ -922,16 +922,11 @@ get_outgoing_edge_probs (basic_block bb) BASE_PROB is the probability of reaching the branch instruction relative to the same basic block BB. */ -static inline int -conditional_probability (int target_prob, int base_prob) +static inline profile_probability +conditional_probability (profile_probability target_prob, + profile_probability base_prob) { - if (base_prob > 0) - { - gcc_assert (target_prob >= 0); - gcc_assert (target_prob <= base_prob); - return GCOV_COMPUTE_SCALE (target_prob, base_prob); - } - return -1; + return target_prob / base_prob; } /* Generate a dispatch tabler, switching on INDEX_EXPR and jumping to @@ -960,12 +955,13 @@ emit_case_dispatch_table (tree index_expr, tree index_type, rtx_code_label *table_label = gen_label_rtx (); bool has_gaps = false; edge default_edge = stmt_bb ? EDGE_SUCC (stmt_bb, 0) : NULL; - int default_prob = default_edge ? default_edge->probability : 0; - int base = get_outgoing_edge_probs (stmt_bb); + profile_probability default_prob = default_edge ? default_edge->probability + : profile_probability::never (); + profile_probability base = get_outgoing_edge_probs (stmt_bb); bool try_with_tablejump = false; - int new_default_prob = conditional_probability (default_prob, - base); + profile_probability new_default_prob = conditional_probability (default_prob, + base); if (! try_casesi (index_type, index_expr, minval, range, table_label, default_label, fallback_label, @@ -1030,15 +1026,16 @@ emit_case_dispatch_table (tree index_expr, tree index_type, through the indirect jump or the direct conditional jump before that. Split the probability of reaching the default label among these two jumps. */ - new_default_prob = conditional_probability (default_prob/2, + new_default_prob = conditional_probability (default_prob.apply_scale + (1, 2), base); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); base -= default_prob; } else { base -= default_prob; - default_prob = 0; + default_prob = profile_probability::never (); } if (default_edge) @@ -1047,12 +1044,12 @@ emit_case_dispatch_table (tree index_expr, tree index_type, /* We have altered the probability of the default edge. So the probabilities of all other edges need to be adjusted so that it sums up to REG_BR_PROB_BASE. */ - if (base) + if (base > profile_probability::never ()) { edge e; edge_iterator ei; FOR_EACH_EDGE (e, ei, stmt_bb->succs) - e->probability = GCOV_COMPUTE_SCALE (e->probability, base); + e->probability /= base; } if (try_with_tablejump) @@ -1153,7 +1150,7 @@ expand_case (gswitch *stmt) default_label = jump_target_rtx (CASE_LABEL (gimple_switch_default_label (stmt))); edge default_edge = EDGE_SUCC (bb, 0); - int default_prob = default_edge->probability; + profile_probability default_prob = default_edge->probability; /* Get upper and lower bounds of case values. */ elt = gimple_switch_label (stmt, 1); @@ -1216,7 +1213,7 @@ expand_case (gswitch *stmt) edge case_edge = find_edge (bb, case_bb); case_list = add_case_node ( case_list, low, high, lab, - case_edge->probability / (intptr_t)(case_edge->aux), + case_edge->probability.apply_scale (1, (intptr_t)(case_edge->aux)), case_node_pool); } reset_out_edges_aux (bb); @@ -1313,7 +1310,8 @@ expand_sjlj_dispatch_table (rtx dispatch_index, { tree elt = dispatch_table[i]; rtx_code_label *lab = jump_target_rtx (CASE_LABEL (elt)); - do_jump_if_equal (index_mode, index, zero, lab, 0, -1); + do_jump_if_equal (index_mode, index, zero, lab, 0, + profile_probability::uninitialized ()); force_expand_binop (index_mode, sub_optab, index, CONST1_RTX (index_mode), index, 0, OPTAB_DIRECT); @@ -1335,7 +1333,10 @@ expand_sjlj_dispatch_table (rtx dispatch_index, tree elt = dispatch_table[i]; tree low = CASE_LOW (elt); tree lab = CASE_LABEL (elt); - case_list = add_case_node (case_list, low, low, lab, 0, case_node_pool); + case_list = add_case_node (case_list, low, low, lab, + profile_probability::guessed_always () + .apply_scale (1, ncases), + case_node_pool); } emit_case_dispatch_table (index_expr, index_type, @@ -1579,12 +1580,12 @@ node_is_bounded (case_node_ptr node, tree index_type) static void emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, - int default_prob, tree index_type) + profile_probability default_prob, tree index_type) { /* If INDEX has an unsigned type, we must make unsigned branches. */ int unsignedp = TYPE_UNSIGNED (index_type); - int probability; - int prob = node->prob, subtree_prob = node->subtree_prob; + profile_probability probability; + profile_probability prob = node->prob, subtree_prob = node->subtree_prob; machine_mode mode = GET_MODE (index); machine_mode imode = TYPE_MODE (index_type); @@ -1704,7 +1705,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, subtree or the left subtree. Divide the probability equally. */ probability = conditional_probability ( - node->right->subtree_prob + default_prob/2, + node->right->subtree_prob + default_prob.apply_scale (1, 2), subtree_prob + default_prob); /* See if the value is on the right. */ emit_cmp_and_jump_insns (index, @@ -1715,7 +1716,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, GT, NULL_RTX, mode, unsignedp, label_rtx (test_label), probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); /* Value must be on the left. Handle the left-hand subtree. */ @@ -1746,7 +1747,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, if (!node_has_low_bound (node, index_type)) { probability = conditional_probability ( - default_prob/2, + default_prob.apply_scale (1, 2), subtree_prob + default_prob); emit_cmp_and_jump_insns (index, convert_modes @@ -1756,7 +1757,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, LT, NULL_RTX, mode, unsignedp, default_label, probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); } emit_case_nodes (index, node->right, default_label, default_prob, index_type); @@ -1788,7 +1789,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, if (!node_has_high_bound (node, index_type)) { probability = conditional_probability ( - default_prob/2, + default_prob.apply_scale (1, 2), subtree_prob + default_prob); emit_cmp_and_jump_insns (index, convert_modes @@ -1798,7 +1799,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, GT, NULL_RTX, mode, unsignedp, default_label, probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); } emit_case_nodes (index, node->left, default_label, @@ -1861,7 +1862,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, test_label = build_decl (curr_insn_location (), LABEL_DECL, NULL_TREE, void_type_node); probability = conditional_probability ( - node->right->subtree_prob + default_prob/2, + node->right->subtree_prob + default_prob.apply_scale (1, 2), subtree_prob + default_prob); emit_cmp_and_jump_insns (index, convert_modes @@ -1871,7 +1872,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, GT, NULL_RTX, mode, unsignedp, label_rtx (test_label), probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); } /* Value belongs to this node or to the left-hand subtree. */ @@ -1912,7 +1913,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, if (!node_has_low_bound (node, index_type)) { probability = conditional_probability ( - default_prob/2, + default_prob.apply_scale (1, 2), subtree_prob + default_prob); emit_cmp_and_jump_insns (index, convert_modes @@ -1922,7 +1923,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, LT, NULL_RTX, mode, unsignedp, default_label, probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); } /* Value belongs to this node or to the right-hand subtree. */ @@ -1949,7 +1950,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, if (!node_has_high_bound (node, index_type)) { probability = conditional_probability ( - default_prob/2, + default_prob.apply_scale (1, 2), subtree_prob + default_prob); emit_cmp_and_jump_insns (index, convert_modes @@ -1959,7 +1960,7 @@ emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label, GT, NULL_RTX, mode, unsignedp, default_label, probability); - default_prob /= 2; + default_prob = default_prob.apply_scale (1, 2); } /* Value belongs to this node or to the left-hand subtree. */ diff --git a/gcc/targhooks.c b/gcc/targhooks.c index 1cdec068ed8..889189c5d82 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -60,6 +60,7 @@ along with GCC; see the file COPYING3. If not see #include "stringpool.h" #include "tree-vrp.h" #include "tree-ssanames.h" +#include "profile-count.h" #include "optabs.h" #include "regs.h" #include "recog.h" diff --git a/gcc/tracer.c b/gcc/tracer.c index 0b7f4da0d81..dd071c1650c 100644 --- a/gcc/tracer.c +++ b/gcc/tracer.c @@ -135,10 +135,8 @@ better_p (const_edge e1, const_edge e2) if (e1->count.initialized_p () && e2->count.initialized_p () && !(e1->count == e2->count)) return e1->count > e2->count; - if (e1->src->frequency * e1->probability != - e2->src->frequency * e2->probability) - return (e1->src->frequency * e1->probability - > e2->src->frequency * e2->probability); + if (EDGE_FREQUENCY (e1) != EDGE_FREQUENCY (e2)) + return EDGE_FREQUENCY (e1) > EDGE_FREQUENCY (e2); /* This is needed to avoid changes in the decision after CFG is modified. */ if (e1->src != e2->src) @@ -160,7 +158,8 @@ find_best_successor (basic_block bb) best = e; if (!best || ignore_bb_p (best->dest)) return NULL; - if (best->probability <= probability_cutoff) + if (best->probability.initialized_p () + && best->probability.to_reg_br_prob_base () <= probability_cutoff) return NULL; return best; } diff --git a/gcc/trans-mem.c b/gcc/trans-mem.c index f66abfc8835..f747e05bd10 100644 --- a/gcc/trans-mem.c +++ b/gcc/trans-mem.c @@ -2934,9 +2934,9 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED) join_bb->frequency = test_bb->frequency = transaction_bb->frequency; join_bb->count = test_bb->count = transaction_bb->count; - ei->probability = PROB_ALWAYS; - et->probability = PROB_LIKELY; - ef->probability = PROB_UNLIKELY; + ei->probability = profile_probability::always (); + et->probability = profile_probability::likely (); + ef->probability = profile_probability::unlikely (); et->count = test_bb->count.apply_probability (et->probability); ef->count = test_bb->count.apply_probability (ef->probability); @@ -2967,20 +2967,20 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED) edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU); test_bb->frequency = transaction_bb->frequency; test_bb->count = transaction_bb->count; - ei->probability = PROB_ALWAYS; + ei->probability = profile_probability::always (); // Not abort edge. If both are live, chose one at random as we'll // we'll be fixing that up below. redirect_edge_pred (fallthru_edge, test_bb); fallthru_edge->flags = EDGE_FALSE_VALUE; - fallthru_edge->probability = PROB_VERY_LIKELY; + fallthru_edge->probability = profile_probability::very_likely (); fallthru_edge->count = test_bb->count.apply_probability (fallthru_edge->probability); // Abort/over edge. redirect_edge_pred (abort_edge, test_bb); abort_edge->flags = EDGE_TRUE_VALUE; - abort_edge->probability = PROB_VERY_UNLIKELY; + abort_edge->probability = profile_probability::unlikely (); abort_edge->count = test_bb->count.apply_probability (abort_edge->probability); @@ -3020,13 +3020,13 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED) // use the uninst path when falling back to serial mode. redirect_edge_pred (inst_edge, test_bb); inst_edge->flags = EDGE_FALSE_VALUE; - inst_edge->probability = REG_BR_PROB_BASE / 2; + inst_edge->probability = profile_probability::even (); inst_edge->count = test_bb->count.apply_probability (inst_edge->probability); redirect_edge_pred (uninst_edge, test_bb); uninst_edge->flags = EDGE_TRUE_VALUE; - uninst_edge->probability = REG_BR_PROB_BASE / 2; + uninst_edge->probability = profile_probability::even (); uninst_edge->count = test_bb->count.apply_probability (uninst_edge->probability); } diff --git a/gcc/tree-call-cdce.c b/gcc/tree-call-cdce.c index 69d3207696d..862d1a6421d 100644 --- a/gcc/tree-call-cdce.c +++ b/gcc/tree-call-cdce.c @@ -752,10 +752,6 @@ gen_shrink_wrap_conditions (gcall *bi_call, vec<gimple *> conds, return; } - -/* Probability of the branch (to the call) is taken. */ -#define ERR_PROB 0.01 - /* Shrink-wrap BI_CALL so that it is only called when one of the NCONDS conditions in CONDS is false. */ @@ -916,14 +912,15 @@ shrink_wrap_one_built_in_call_with_conds (gcall *bi_call, vec <gimple *> conds, basic_block src_bb = call_edge->src; gcc_assert (src_bb == nocall_edge->src); - call_edge->probability = REG_BR_PROB_BASE * ERR_PROB; + call_edge->probability = profile_probability::very_unlikely (); call_edge->count = src_bb->count.apply_probability (call_edge->probability); - nocall_edge->probability = inverse_probability (call_edge->probability); + nocall_edge->probability = profile_probability::always () + - call_edge->probability; nocall_edge->count = src_bb->count - call_edge->count; - unsigned int call_frequency = apply_probability (src_bb->frequency, - call_edge->probability); + unsigned int call_frequency + = call_edge->probability.apply (src_bb->frequency); bi_call_bb->count += call_edge->count; bi_call_bb->frequency += call_frequency; diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c index e82f95b48dd..5af677ba652 100644 --- a/gcc/tree-cfg.c +++ b/gcc/tree-cfg.c @@ -2843,9 +2843,7 @@ gimple_split_edge (edge edge_in) new_bb = create_empty_bb (after_bb); new_bb->frequency = EDGE_FREQUENCY (edge_in); new_bb->count = edge_in->count; - new_edge = make_edge (new_bb, dest, EDGE_FALLTHRU); - new_edge->probability = REG_BR_PROB_BASE; - new_edge->count = edge_in->count; + new_edge = make_single_succ_edge (new_bb, dest, EDGE_FALLTHRU); e = redirect_edge_and_branch (edge_in, new_bb); gcc_assert (e == edge_in); @@ -7250,7 +7248,7 @@ move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb, basic_block after, bb, *entry_pred, *exit_succ, abb; struct function *saved_cfun = cfun; int *entry_flag, *exit_flag; - unsigned *entry_prob, *exit_prob; + profile_probability *entry_prob, *exit_prob; unsigned i, num_entry_edges, num_exit_edges, num_nodes; edge e; edge_iterator ei; @@ -7288,7 +7286,7 @@ move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb, num_entry_edges = EDGE_COUNT (entry_bb->preds); entry_pred = XNEWVEC (basic_block, num_entry_edges); entry_flag = XNEWVEC (int, num_entry_edges); - entry_prob = XNEWVEC (unsigned, num_entry_edges); + entry_prob = XNEWVEC (profile_probability, num_entry_edges); i = 0; for (ei = ei_start (entry_bb->preds); (e = ei_safe_edge (ei)) != NULL;) { @@ -7303,7 +7301,7 @@ move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb, num_exit_edges = EDGE_COUNT (exit_bb->succs); exit_succ = XNEWVEC (basic_block, num_exit_edges); exit_flag = XNEWVEC (int, num_exit_edges); - exit_prob = XNEWVEC (unsigned, num_exit_edges); + exit_prob = XNEWVEC (profile_probability, num_exit_edges); i = 0; for (ei = ei_start (exit_bb->succs); (e = ei_safe_edge (ei)) != NULL;) { diff --git a/gcc/tree-cfgcleanup.c b/gcc/tree-cfgcleanup.c index c904d0cf57f..b4127f968ff 100644 --- a/gcc/tree-cfgcleanup.c +++ b/gcc/tree-cfgcleanup.c @@ -155,8 +155,6 @@ cleanup_control_expr_graph (basic_block bb, gimple_stmt_iterator gsi, } if (!warned) fold_undefer_and_ignore_overflow_warnings (); - if (taken_edge->probability > REG_BR_PROB_BASE) - taken_edge->probability = REG_BR_PROB_BASE; } else taken_edge = single_succ_edge (bb); diff --git a/gcc/tree-eh.c b/gcc/tree-eh.c index fc016d795b7..34c223d661d 100644 --- a/gcc/tree-eh.c +++ b/gcc/tree-eh.c @@ -3244,9 +3244,7 @@ lower_resx (basic_block bb, gresx *stmt, } gcc_assert (EDGE_COUNT (bb->succs) == 0); - e = make_edge (bb, new_bb, EDGE_FALLTHRU); - e->count = bb->count; - e->probability = REG_BR_PROB_BASE; + e = make_single_succ_edge (bb, new_bb, EDGE_FALLTHRU); } else { @@ -3262,7 +3260,7 @@ lower_resx (basic_block bb, gresx *stmt, e = single_succ_edge (bb); gcc_assert (e->flags & EDGE_EH); e->flags = (e->flags & ~EDGE_EH) | EDGE_FALLTHRU; - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = bb->count; /* If there are no more EH users of the landing pad, delete it. */ @@ -4283,7 +4281,7 @@ cleanup_empty_eh_move_lp (basic_block bb, edge e_out, /* Clean up E_OUT for the fallthru. */ e_out->flags = (e_out->flags & ~EDGE_EH) | EDGE_FALLTHRU; - e_out->probability = REG_BR_PROB_BASE; + e_out->probability = profile_probability::always (); e_out->count = e_out->src->count; } diff --git a/gcc/tree-if-conv.c b/gcc/tree-if-conv.c index eb637952e22..b1f06a83fb2 100644 --- a/gcc/tree-if-conv.c +++ b/gcc/tree-if-conv.c @@ -2564,7 +2564,8 @@ version_loop_for_if_conversion (struct loop *loop) /* At this point we invalidate porfile confistency until IFN_LOOP_VECTORIZED is re-merged in the vectorizer. */ new_loop = loop_version (loop, cond, &cond_bb, - REG_BR_PROB_BASE, REG_BR_PROB_BASE, + profile_probability::always (), + profile_probability::always (), REG_BR_PROB_BASE, REG_BR_PROB_BASE, true); free_original_copy_tables (); diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index aea8a79da14..9306c3fbdf7 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -2291,10 +2291,44 @@ copy_edges_for_bb (basic_block bb, profile_count num, profile_count den, } } + bool update_probs = false; + if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH) - make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt)); + { + make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt)); + update_probs = true; + } else if (can_throw) - make_eh_edges (copy_stmt); + { + make_eh_edges (copy_stmt); + update_probs = true; + } + + /* EH edges may not match old edges. Copy as much as possible. */ + if (update_probs) + { + edge e; + edge_iterator ei; + basic_block copy_stmt_bb = gimple_bb (copy_stmt); + + FOR_EACH_EDGE (old_edge, ei, bb->succs) + if ((old_edge->flags & EDGE_EH) + && (e = find_edge (copy_stmt_bb, + (basic_block) old_edge->dest->aux)) + && (e->flags & EDGE_EH)) + { + e->probability = old_edge->probability; + e->count = old_edge->count; + } + + FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs) + if ((e->flags & EDGE_EH) && !e->probability.initialized_p ()) + { + e->probability = profile_probability::never (); + e->count = profile_count::zero (); + } + } + /* If the call we inline cannot make abnormal goto do not add additional abnormal edges but only retain those already present @@ -2317,7 +2351,8 @@ copy_edges_for_bb (basic_block bb, profile_count num, profile_count den, && gimple_call_arg (copy_stmt, 0) == boolean_true_node) nonlocal_goto = false; else - make_edge (copy_stmt_bb, abnormal_goto_dest, EDGE_ABNORMAL); + make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest, + EDGE_ABNORMAL); } if ((can_throw || nonlocal_goto) @@ -2789,7 +2824,7 @@ copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale, if (new_entry) { edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = incoming_count; } diff --git a/gcc/tree-parloops.c b/gcc/tree-parloops.c index 79d616b760a..a47ea5c149e 100644 --- a/gcc/tree-parloops.c +++ b/gcc/tree-parloops.c @@ -2115,10 +2115,12 @@ create_parallel_loop (struct loop *loop, tree loop_fn, tree data, gcc_assert (exit == single_dom_exit (loop)); guard = make_edge (for_bb, ex_bb, 0); + /* FIXME: What is the probability? */ + guard->probability = profile_probability::guessed_never (); /* Split the latch edge, so LOOPS_HAVE_SIMPLE_LATCHES is still valid. */ loop->latch = split_edge (single_succ_edge (loop->latch)); single_pred_edge (loop->latch)->flags = 0; - end = make_edge (single_pred (loop->latch), ex_bb, EDGE_FALLTHRU); + end = make_single_succ_edge (single_pred (loop->latch), ex_bb, EDGE_FALLTHRU); rescan_loop_exit (end, true, false); for (gphi_iterator gpi = gsi_start_phis (ex_bb); @@ -2358,7 +2360,9 @@ gen_parallel_loop (struct loop *loop, /* We assume that the loop usually iterates a lot. */ prob = 4 * REG_BR_PROB_BASE / 5; loop_version (loop, many_iterations_cond, NULL, - prob, REG_BR_PROB_BASE - prob, + profile_probability::from_reg_br_prob_base (prob), + profile_probability::from_reg_br_prob_base + (REG_BR_PROB_BASE - prob), prob, REG_BR_PROB_BASE - prob, true); update_ssa (TODO_update_ssa); free_original_copy_tables (); @@ -3132,6 +3136,8 @@ oacc_entry_exit_single_gang (bitmap in_loop_bbs, vec<basic_block> region_bbs, gsi_insert_after (&gsi2, cond, GSI_NEW_STMT); edge e3 = make_edge (bb, bb3, EDGE_FALSE_VALUE); + /* FIXME: What is the probability? */ + e3->probability = profile_probability::guessed_never (); e->flags = EDGE_TRUE_VALUE; tree vdef = gimple_vdef (stmt); diff --git a/gcc/tree-profile.c b/gcc/tree-profile.c index 2ae4b69b68e..b1ee7f2b3f5 100644 --- a/gcc/tree-profile.c +++ b/gcc/tree-profile.c @@ -436,16 +436,16 @@ gimple_gen_ic_func_profiler (void) edge true_edge = single_succ_edge (cond_bb); true_edge->flags = EDGE_TRUE_VALUE; - int probability; + profile_probability probability; if (DECL_VIRTUAL_P (current_function_decl)) - probability = PROB_VERY_LIKELY; + probability = profile_probability::very_likely (); else - probability = PROB_UNLIKELY; + probability = profile_probability::unlikely (); true_edge->probability = probability; edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE); - e->probability = REG_BR_PROB_BASE - true_edge->probability; + e->probability = true_edge->probability.invert (); /* Insert code: @@ -497,10 +497,10 @@ gimple_gen_time_profiler (unsigned tag, unsigned base) edge true_edge = single_succ_edge (cond_bb); true_edge->flags = EDGE_TRUE_VALUE; - true_edge->probability = PROB_UNLIKELY; + true_edge->probability = profile_probability::unlikely (); edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE); - e->probability = REG_BR_PROB_BASE - true_edge->probability; + e->probability = true_edge->probability.invert (); gimple_stmt_iterator gsi = gsi_start_bb (cond_bb); tree original_ref = tree_coverage_counter_ref (tag, base); diff --git a/gcc/tree-ssa-dce.c b/gcc/tree-ssa-dce.c index 934324990e8..e62afad1bea 100644 --- a/gcc/tree-ssa-dce.c +++ b/gcc/tree-ssa-dce.c @@ -1054,7 +1054,7 @@ remove_dead_stmt (gimple_stmt_iterator *i, basic_block bb) e = e2; } gcc_assert (e); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = bb->count; /* The edge is no longer associated with a conditional, so it does diff --git a/gcc/tree-ssa-ifcombine.c b/gcc/tree-ssa-ifcombine.c index 904befa9808..c564ea104bc 100644 --- a/gcc/tree-ssa-ifcombine.c +++ b/gcc/tree-ssa-ifcombine.c @@ -363,18 +363,14 @@ update_profile_after_ifcombine (basic_block inner_cond_bb, inner_taken->count += outer2->count; outer2->count = profile_count::zero (); - inner_taken->probability = outer2->probability - + RDIV (outer_to_inner->probability - * inner_taken->probability, - REG_BR_PROB_BASE); - if (inner_taken->probability > REG_BR_PROB_BASE) - inner_taken->probability = REG_BR_PROB_BASE; - inner_not_taken->probability = REG_BR_PROB_BASE + inner_taken->probability = outer2->probability + outer_to_inner->probability + * inner_taken->probability; + inner_not_taken->probability = profile_probability::always () - inner_taken->probability; - outer_to_inner->probability = REG_BR_PROB_BASE; + outer_to_inner->probability = profile_probability::always (); inner_cond_bb->frequency = outer_cond_bb->frequency; - outer2->probability = 0; + outer2->probability = profile_probability::never (); } /* If-convert on a and pattern with a common else block. The inner diff --git a/gcc/tree-ssa-loop-im.c b/gcc/tree-ssa-loop-im.c index e92eaa6e2d7..2fbaa62c142 100644 --- a/gcc/tree-ssa-loop-im.c +++ b/gcc/tree-ssa-loop-im.c @@ -1801,7 +1801,7 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag, int freq_sum = 0; profile_count count_sum = profile_count::zero (); int nbbs = 0, ncount = 0; - int flag_probability = -1; + profile_probability flag_probability = profile_probability::uninitialized (); /* Flag is set in FLAG_BBS. Determine probability that flag will be true at loop exit. @@ -1824,27 +1824,29 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag, if ((*it)->count.initialized_p ()) count_sum += (*it)->count, ncount ++; if (dominated_by_p (CDI_DOMINATORS, ex->src, *it)) - flag_probability = REG_BR_PROB_BASE; + flag_probability = profile_probability::always (); nbbs++; } - if (flag_probability != -1) + profile_probability cap = profile_probability::always ().apply_scale (2, 3); + + if (flag_probability.initialized_p ()) ; else if (ncount == nbbs && count_sum > 0 && preheader->count >= count_sum) { flag_probability = count_sum.probability_in (preheader->count); - if (flag_probability > REG_BR_PROB_BASE * 2 / 3) - flag_probability = REG_BR_PROB_BASE * 2 / 3; + if (flag_probability > cap) + flag_probability = cap; } else if (freq_sum > 0 && EDGE_FREQUENCY (preheader) >= freq_sum) { - flag_probability = GCOV_COMPUTE_SCALE (freq_sum, - EDGE_FREQUENCY (preheader)); - if (flag_probability > REG_BR_PROB_BASE * 2 / 3) - flag_probability = REG_BR_PROB_BASE * 2 / 3; + flag_probability = profile_probability::from_reg_br_prob_base + (GCOV_COMPUTE_SCALE (freq_sum, EDGE_FREQUENCY (preheader))); + if (flag_probability > cap) + flag_probability = cap; } else - flag_probability = REG_BR_PROB_BASE * 2 / 3; + flag_probability = cap; /* ?? Insert store after previous store if applicable. See note below. */ @@ -1876,7 +1878,7 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag, old_dest = ex->dest; new_bb = split_edge (ex); then_bb = create_empty_bb (new_bb); - then_bb->frequency = apply_probability (new_bb->frequency, flag_probability); + then_bb->frequency = flag_probability.apply (new_bb->frequency); then_bb->count = new_bb->count.apply_probability (flag_probability); if (irr) then_bb->flags = BB_IRREDUCIBLE_LOOP; @@ -1901,13 +1903,11 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag, e1->flags |= EDGE_FALSE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0); e1->flags &= ~EDGE_FALLTHRU; - e1->probability = REG_BR_PROB_BASE - flag_probability; + e1->probability = flag_probability.invert (); e1->count = new_bb->count - then_bb->count; - then_old_edge = make_edge (then_bb, old_dest, + then_old_edge = make_single_succ_edge (then_bb, old_dest, EDGE_FALLTHRU | (irr ? EDGE_IRREDUCIBLE_LOOP : 0)); - then_old_edge->probability = REG_BR_PROB_BASE; - then_old_edge->count = then_bb->count; set_immediate_dominator (CDI_DOMINATORS, then_bb, new_bb); diff --git a/gcc/tree-ssa-loop-ivcanon.c b/gcc/tree-ssa-loop-ivcanon.c index b6ac765aac6..4e828931f53 100644 --- a/gcc/tree-ssa-loop-ivcanon.c +++ b/gcc/tree-ssa-loop-ivcanon.c @@ -529,7 +529,7 @@ remove_exits_and_undefined_stmts (struct loop *loop, unsigned int npeeled) } if (!loop_exit_edge_p (loop, exit_edge)) exit_edge = EDGE_SUCC (bb, 1); - exit_edge->probability = REG_BR_PROB_BASE; + exit_edge->probability = profile_probability::always (); exit_edge->count = exit_edge->src->count; gcc_checking_assert (loop_exit_edge_p (loop, exit_edge)); gcond *cond_stmt = as_a <gcond *> (elt->stmt); @@ -642,7 +642,7 @@ unloop_loops (bitmap loop_closed_ssa_invalidated, it in. */ stmt = gimple_build_call (builtin_decl_implicit (BUILT_IN_UNREACHABLE), 0); latch_edge = make_edge (latch, create_basic_block (NULL, NULL, latch), flags); - latch_edge->probability = 0; + latch_edge->probability = profile_probability::never (); latch_edge->count = profile_count::zero (); latch_edge->flags |= flags; latch_edge->goto_locus = locus; @@ -1106,7 +1106,7 @@ try_peel_loop (struct loop *loop, } int scale = 1; if (loop->header->count > 0) - scale = entry_count.probability_in (loop->header->count); + scale = entry_count.probability_in (loop->header->count).to_reg_br_prob_base (); else if (loop->header->frequency) scale = RDIV (entry_freq * REG_BR_PROB_BASE, loop->header->frequency); scale_loop_profile (loop, scale, 0); diff --git a/gcc/tree-ssa-loop-manip.c b/gcc/tree-ssa-loop-manip.c index 18ed4f62686..b2c1ccdd6f4 100644 --- a/gcc/tree-ssa-loop-manip.c +++ b/gcc/tree-ssa-loop-manip.c @@ -1244,7 +1244,10 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor, scale_rest = REG_BR_PROB_BASE; new_loop = loop_version (loop, enter_main_cond, NULL, - prob_entry, REG_BR_PROB_BASE - prob_entry, + profile_probability::from_reg_br_prob_base + (prob_entry), + profile_probability::from_reg_br_prob_base + (REG_BR_PROB_BASE - prob_entry), scale_unrolled, scale_rest, true); gcc_assert (new_loop != NULL); update_ssa (TODO_update_ssa); @@ -1259,9 +1262,11 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor, /* Since the exit edge will be removed, the frequency of all the blocks in the loop that are dominated by it must be scaled by 1 / (1 - exit->probability). */ - scale_dominated_blocks_in_loop (loop, exit->src, - REG_BR_PROB_BASE, - REG_BR_PROB_BASE - exit->probability); + if (exit->probability.initialized_p ()) + scale_dominated_blocks_in_loop (loop, exit->src, + REG_BR_PROB_BASE, + REG_BR_PROB_BASE + - exit->probability.to_reg_br_prob_base ()); bsi = gsi_last_bb (exit_bb); exit_if = gimple_build_cond (EQ_EXPR, integer_zero_node, @@ -1278,11 +1283,13 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor, new_exit->count = exit->count; new_exit->probability = exit->probability; new_nonexit = single_pred_edge (loop->latch); - new_nonexit->probability = REG_BR_PROB_BASE - exit->probability; + new_nonexit->probability = exit->probability.invert (); new_nonexit->flags = EDGE_TRUE_VALUE; new_nonexit->count -= exit->count; - scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability, - REG_BR_PROB_BASE); + if (new_nonexit->probability.initialized_p ()) + scale_bbs_frequencies_int (&loop->latch, 1, + new_nonexit->probability.to_reg_br_prob_base (), + REG_BR_PROB_BASE); old_entry = loop_preheader_edge (loop); new_entry = loop_preheader_edge (new_loop); @@ -1368,24 +1375,29 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor, if (freq_e == profile_count::zero ()) freq_e = profile_count::from_gcov_type (1); /* This should not overflow. */ - scale = freq_e.probability_in (freq_h); + scale = freq_e.probability_in (freq_h).to_reg_br_prob_base (); scale_loop_frequencies (loop, scale, REG_BR_PROB_BASE); } exit_bb = single_pred (loop->latch); new_exit = find_edge (exit_bb, rest); new_exit->count = loop_preheader_edge (loop)->count; - new_exit->probability = REG_BR_PROB_BASE / (new_est_niter + 1); + new_exit->probability = profile_probability::always () + .apply_scale (1, new_est_niter + 1); rest->count += new_exit->count; rest->frequency += EDGE_FREQUENCY (new_exit); new_nonexit = single_pred_edge (loop->latch); - prob = new_nonexit->probability; - new_nonexit->probability = REG_BR_PROB_BASE - new_exit->probability; + if (new_nonexit->probability.initialized_p ()) + prob = new_nonexit->probability.to_reg_br_prob_base (); + else + prob = 0; + new_nonexit->probability = new_exit->probability.invert (); new_nonexit->count = exit_bb->count - new_exit->count; if (prob > 0) - scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability, + scale_bbs_frequencies_int (&loop->latch, 1, + new_nonexit->probability.to_reg_br_prob_base (), prob); /* Finally create the new counter for number of iterations and add the new diff --git a/gcc/tree-ssa-loop-split.c b/gcc/tree-ssa-loop-split.c index e77f2bfd1b5..2d57957dcfd 100644 --- a/gcc/tree-ssa-loop-split.c +++ b/gcc/tree-ssa-loop-split.c @@ -354,10 +354,10 @@ connect_loops (struct loop *loop1, struct loop *loop2) } new_e->count = skip_bb->count; - new_e->probability = PROB_LIKELY; + new_e->probability = profile_probability::likely (); new_e->count = skip_e->count.apply_probability (PROB_LIKELY); skip_e->count -= new_e->count; - skip_e->probability = inverse_probability (PROB_LIKELY); + skip_e->probability = profile_probability::unlikely (); return new_e; } @@ -559,8 +559,11 @@ split_loop (struct loop *loop1, struct tree_niter_desc *niter) them, and fix up SSA form for that. */ initialize_original_copy_tables (); basic_block cond_bb; + + /* FIXME: probabilities seems wrong here. */ struct loop *loop2 = loop_version (loop1, cond, &cond_bb, - REG_BR_PROB_BASE, REG_BR_PROB_BASE, + profile_probability::always (), + profile_probability::always (), REG_BR_PROB_BASE, REG_BR_PROB_BASE, true); gcc_assert (loop2); diff --git a/gcc/tree-ssa-loop-unswitch.c b/gcc/tree-ssa-loop-unswitch.c index 12bf640d3f8..6ad5741827b 100644 --- a/gcc/tree-ssa-loop-unswitch.c +++ b/gcc/tree-ssa-loop-unswitch.c @@ -480,7 +480,7 @@ static struct loop * tree_unswitch_loop (struct loop *loop, basic_block unswitch_on, tree cond) { - unsigned prob_true; + profile_probability prob_true; edge edge_true, edge_false; /* Some sanity checking. */ @@ -490,9 +490,13 @@ tree_unswitch_loop (struct loop *loop, extract_true_false_edges_from_block (unswitch_on, &edge_true, &edge_false); prob_true = edge_true->probability; + int p = prob_true.initialized_p () ? prob_true.to_reg_br_prob_base () + : REG_BR_PROB_BASE / 2; return loop_version (loop, unshare_expr (cond), - NULL, prob_true, REG_BR_PROB_BASE - prob_true, prob_true, - REG_BR_PROB_BASE - prob_true, false); + NULL, prob_true, + prob_true.invert (), + p, REG_BR_PROB_BASE - p, + false); } /* Unswitch outer loops by hoisting invariant guard on @@ -818,10 +822,13 @@ hoist_guard (struct loop *loop, edge guard) /* Create new loop pre-header. */ e = split_block (pre_header, last_stmt (pre_header)); if (dump_file && (dump_flags & TDF_DETAILS)) - fprintf (dump_file, " Moving guard %i->%i (prob %i) to bb %i, " - "new preheader is %i\n", - guard->src->index, guard->dest->index, guard->probability, - e->src->index, e->dest->index); + { + fprintf (dump_file, " Moving guard %i->%i (prob ", + guard->src->index, guard->dest->index); + guard->probability.dump (dump_file); + fprintf (dump_file, ") to bb %i, new preheader is %i\n", + e->src->index, e->dest->index); + } gcc_assert (loop_preheader_edge (loop)->src == e->dest); @@ -854,23 +861,26 @@ hoist_guard (struct loop *loop, edge guard) } new_edge->count = skip_count; if (dump_file && (dump_flags & TDF_DETAILS)) - fprintf (dump_file, " Estimated probability of skipping loop is %i\n", - new_edge->probability); + { + fprintf (dump_file, " Estimated probability of skipping loop is "); + new_edge->probability.dump (dump_file); + fprintf (dump_file, "\n"); + } /* Update profile after the transform: First decrease count of path from newly hoisted loop guard to loop header... */ e->count -= skip_count; - e->probability = REG_BR_PROB_BASE - new_edge->probability; + e->probability = new_edge->probability.invert (); e->dest->count = e->count; e->dest->frequency = EDGE_FREQUENCY (e); /* ... now update profile to represent that original guard will be optimized away ... */ - guard->probability = 0; + guard->probability = profile_probability::never (); guard->count = profile_count::zero (); - not_guard->probability = REG_BR_PROB_BASE; + not_guard->probability = profile_probability::always (); /* This count is wrong (frequency of not_guard does not change), but will be scaled later. */ not_guard->count = guard->src->count; @@ -888,7 +898,10 @@ hoist_guard (struct loop *loop, edge guard) { if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, " %i", bb->index); - scale_bbs_frequencies_int (&bb, 1, e->probability, REG_BR_PROB_BASE); + if (e->probability.initialized_p ()) + scale_bbs_frequencies_int (&bb, 1, + e->probability.to_reg_br_prob_base (), + REG_BR_PROB_BASE); } } diff --git a/gcc/tree-ssa-phionlycprop.c b/gcc/tree-ssa-phionlycprop.c index 9fa427ff2f7..65af44834df 100644 --- a/gcc/tree-ssa-phionlycprop.c +++ b/gcc/tree-ssa-phionlycprop.c @@ -313,8 +313,6 @@ propagate_rhs_into_lhs (gimple *stmt, tree lhs, tree rhs, te->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); te->flags &= ~EDGE_ABNORMAL; te->flags |= EDGE_FALLTHRU; - if (te->probability > REG_BR_PROB_BASE) - te->probability = REG_BR_PROB_BASE; } } } diff --git a/gcc/tree-ssa-phiopt.c b/gcc/tree-ssa-phiopt.c index b652361b255..f5c07dc27f1 100644 --- a/gcc/tree-ssa-phiopt.c +++ b/gcc/tree-ssa-phiopt.c @@ -374,7 +374,7 @@ replace_phi_edge_with_variable (basic_block cond_block, { EDGE_SUCC (cond_block, 0)->flags |= EDGE_FALLTHRU; EDGE_SUCC (cond_block, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); - EDGE_SUCC (cond_block, 0)->probability = REG_BR_PROB_BASE; + EDGE_SUCC (cond_block, 0)->probability = profile_probability::always (); EDGE_SUCC (cond_block, 0)->count += EDGE_SUCC (cond_block, 1)->count; block_to_remove = EDGE_SUCC (cond_block, 1)->dest; @@ -384,7 +384,7 @@ replace_phi_edge_with_variable (basic_block cond_block, EDGE_SUCC (cond_block, 1)->flags |= EDGE_FALLTHRU; EDGE_SUCC (cond_block, 1)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); - EDGE_SUCC (cond_block, 1)->probability = REG_BR_PROB_BASE; + EDGE_SUCC (cond_block, 1)->probability = profile_probability::always (); EDGE_SUCC (cond_block, 1)->count += EDGE_SUCC (cond_block, 0)->count; block_to_remove = EDGE_SUCC (cond_block, 0)->dest; @@ -1017,7 +1017,7 @@ value_replacement (basic_block cond_bb, basic_block middle_bb, if (optimize_bb_for_speed_p (cond_bb) /* The special case is useless if it has a low probability. */ && profile_status_for_fn (cfun) != PROFILE_ABSENT - && EDGE_PRED (middle_bb, 0)->probability < PROB_EVEN + && EDGE_PRED (middle_bb, 0)->probability < profile_probability::even () /* If assign is cheap, there is no point avoiding it. */ && estimate_num_insns (assign, &eni_time_weights) >= 3 * estimate_num_insns (cond, &eni_time_weights)) diff --git a/gcc/tree-ssa-reassoc.c b/gcc/tree-ssa-reassoc.c index 982ab790cb3..2f49ba13553 100644 --- a/gcc/tree-ssa-reassoc.c +++ b/gcc/tree-ssa-reassoc.c @@ -5843,7 +5843,7 @@ branch_fixup (void) gsi_insert_after (&gsi, g, GSI_NEW_STMT); edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE); - etrue->probability = REG_BR_PROB_BASE / 2; + etrue->probability = profile_probability::even (); etrue->count = cond_bb->count.apply_scale (1, 2); edge efalse = find_edge (cond_bb, then_bb); efalse->flags = EDGE_FALSE_VALUE; diff --git a/gcc/tree-ssa-tail-merge.c b/gcc/tree-ssa-tail-merge.c index f6c9878a0a3..db836f7187f 100644 --- a/gcc/tree-ssa-tail-merge.c +++ b/gcc/tree-ssa-tail-merge.c @@ -1592,9 +1592,10 @@ replace_block_by (basic_block bb1, basic_block bb2) else if (bb2->frequency && !bb1->frequency) ; else if (out_freq_sum) - e2->probability = GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1) - + EDGE_FREQUENCY (e2), - out_freq_sum); + e2->probability = profile_probability::from_reg_br_prob_base + (GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1) + + EDGE_FREQUENCY (e2), + out_freq_sum)); out_sum += e2->count; } bb2->frequency += bb1->frequency; diff --git a/gcc/tree-ssa-threadupdate.c b/gcc/tree-ssa-threadupdate.c index df4c6def78a..f5bc95f092f 100644 --- a/gcc/tree-ssa-threadupdate.c +++ b/gcc/tree-ssa-threadupdate.c @@ -302,7 +302,7 @@ remove_ctrl_stmt_and_useless_edges (basic_block bb, basic_block dest_bb) } else { - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = bb->count; ei_next (&ei); } @@ -546,11 +546,9 @@ static void create_edge_and_update_destination_phis (struct redirection_data *rd, basic_block bb, int idx) { - edge e = make_edge (bb, rd->path->last ()->e->dest, EDGE_FALLTHRU); + edge e = make_single_succ_edge (bb, rd->path->last ()->e->dest, EDGE_FALLTHRU); rescan_loop_exit (e, true, false); - e->probability = REG_BR_PROB_BASE; - e->count = bb->count; /* We used to copy the thread path here. That was added in 2007 and dutifully updated through the representation changes in 2013. @@ -765,7 +763,8 @@ compute_path_counts (struct redirection_data *rd, /* Handle incoming profile insanities. */ if (total_count < path_in_count) path_in_count = total_count; - int onpath_scale = path_in_count.probability_in (total_count); + int onpath_scale + = path_in_count.probability_in (total_count).to_reg_br_prob_base (); /* Walk the entire path to do some more computation in order to estimate how much of the path_in_count will flow out of the duplicated threading @@ -919,7 +918,7 @@ recompute_probabilities (basic_block bb) get a flow verification error. Not much we can do to make counts/freqs sane without redoing the profile estimation. */ - esucc->probability = REG_BR_PROB_BASE; + esucc->probability = profile_probability::guessed_always (); } } @@ -978,7 +977,8 @@ update_joiner_offpath_counts (edge epath, basic_block dup_bb, among the duplicated off-path edges based on their original ratio to the full off-path count (total_orig_off_path_count). */ - int scale = enonpath->count.probability_in (total_orig_off_path_count); + int scale = enonpath->count.probability_in (total_orig_off_path_count) + .to_reg_br_prob_base (); /* Give the duplicated offpath edge a portion of the duplicated total. */ enonpathdup->count = total_dup_off_path_count.apply_probability (scale); @@ -1048,9 +1048,13 @@ freqs_to_counts_path (struct redirection_data *rd) /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding errors applying the probability when the frequencies are very small. */ - ein->count = profile_count::from_gcov_type - (apply_probability (ein->src->frequency * REG_BR_PROB_BASE, - ein->probability)); + if (ein->probability.initialized_p ()) + ein->count = profile_count::from_gcov_type + (apply_probability (ein->src->frequency * REG_BR_PROB_BASE, + ein->probability.to_reg_br_prob_base ())); + else + /* FIXME: this is hack; we should track uninitialized values. */ + ein->count = profile_count::zero (); } for (unsigned int i = 1; i < path->length (); i++) @@ -2358,7 +2362,7 @@ duplicate_thread_path (edge entry, edge exit, if (e) { rescan_loop_exit (e, true, false); - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); e->count = region_copy[n_region - 1]->count; } diff --git a/gcc/tree-switch-conversion.c b/gcc/tree-switch-conversion.c index 72e53af773f..e5b5cb9a0be 100644 --- a/gcc/tree-switch-conversion.c +++ b/gcc/tree-switch-conversion.c @@ -103,7 +103,7 @@ hoist_edge_and_branch_if_true (gimple_stmt_iterator *gsip, e_false->flags &= ~EDGE_FALLTHRU; e_false->flags |= EDGE_FALSE_VALUE; - e_false->probability = REG_BR_PROB_BASE - e_true->probability; + e_false->probability = e_true->probability.invert (); e_false->count = split_bb->count - e_true->count; new_bb->count = e_false->count; @@ -556,7 +556,7 @@ struct switch_conv_info basic_block final_bb; /* The probability of the default edge in the replaced switch. */ - int default_prob; + profile_probability default_prob; /* The count of the default edge in the replaced switch. */ profile_count default_count; @@ -1422,7 +1422,7 @@ gen_inbound_check (gswitch *swtch, struct switch_conv_info *info) /* flags and profiles of the edge for in-range values */ if (!info->default_case_nonstandard) e01 = make_edge (bb0, bb1, EDGE_TRUE_VALUE); - e01->probability = REG_BR_PROB_BASE - info->default_prob; + e01->probability = info->default_prob.invert (); e01->count = info->other_count; /* flags and profiles of the edge taking care of out-of-range values */ @@ -1434,7 +1434,7 @@ gen_inbound_check (gswitch *swtch, struct switch_conv_info *info) bbf = info->final_bb; e1f = make_edge (bb1, bbf, EDGE_FALLTHRU); - e1f->probability = REG_BR_PROB_BASE; + e1f->probability = profile_probability::always (); e1f->count = info->other_count; if (info->default_case_nonstandard) @@ -1442,7 +1442,7 @@ gen_inbound_check (gswitch *swtch, struct switch_conv_info *info) else { e2f = make_edge (bb2, bbf, EDGE_FALLTHRU); - e2f->probability = REG_BR_PROB_BASE; + e2f->probability = profile_probability::always (); e2f->count = info->default_count; } diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index d60b84e60ef..e41372b0235 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -540,7 +540,7 @@ slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *loop, static edge slpeel_add_loop_guard (basic_block guard_bb, tree cond, basic_block guard_to, basic_block dom_bb, - int probability, bool irreducible_p) + profile_probability probability, bool irreducible_p) { gimple_stmt_iterator gsi; edge new_e, enter_e; @@ -571,7 +571,7 @@ slpeel_add_loop_guard (basic_block guard_bb, tree cond, new_e->flags |= EDGE_IRREDUCIBLE_LOOP; enter_e->count -= new_e->count; - enter_e->probability = inverse_probability (probability); + enter_e->probability = probability.invert (); set_immediate_dominator (CDI_DOMINATORS, guard_to, dom_bb); /* Split enter_e to preserve LOOPS_HAVE_PREHEADERS. */ @@ -1660,7 +1660,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, edge e, guard_e; tree type = TREE_TYPE (niters), guard_cond; basic_block guard_bb, guard_to; - int prob_prolog, prob_vector, prob_epilog; + profile_probability prob_prolog, prob_vector, prob_epilog; int bound_prolog = 0, bound_scalar = 0, bound = 0; int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); int prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo); @@ -1670,10 +1670,11 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, if (!prolog_peeling && !epilog_peeling) return NULL; - prob_vector = 9 * REG_BR_PROB_BASE / 10; + prob_vector = profile_probability::guessed_always ().apply_scale (9, 10); if ((vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo)) == 2) vf = 3; - prob_prolog = prob_epilog = (vf - 1) * REG_BR_PROB_BASE / vf; + prob_prolog = prob_epilog = profile_probability::guessed_always () + .apply_scale (vf - 1, vf); vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); struct loop *prolog, *epilog = NULL, *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -1718,9 +1719,11 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, separately. Note in this case, the probability of epilog loop needs to be scaled back later. */ basic_block bb_before_loop = loop_preheader_edge (loop)->src; - scale_bbs_frequencies_int (&bb_before_loop, 1, prob_vector, + if (prob_vector.initialized_p ()) + scale_bbs_frequencies_int (&bb_before_loop, 1, + prob_vector.to_reg_br_prob_base (), REG_BR_PROB_BASE); - scale_loop_profile (loop, prob_vector, bound); + scale_loop_profile (loop, prob_vector.to_reg_br_prob_base (), bound); } tree niters_prolog = build_int_cst (type, 0); @@ -1762,15 +1765,17 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, guard_to = split_edge (loop_preheader_edge (loop)); guard_e = slpeel_add_loop_guard (guard_bb, guard_cond, guard_to, guard_bb, - inverse_probability (prob_prolog), + prob_prolog.invert (), irred_flag); e = EDGE_PRED (guard_to, 0); e = (e != guard_e ? e : EDGE_PRED (guard_to, 1)); slpeel_update_phi_nodes_for_guard1 (prolog, loop, guard_e, e); - scale_bbs_frequencies_int (&bb_after_prolog, 1, prob_prolog, + scale_bbs_frequencies_int (&bb_after_prolog, 1, + prob_prolog.to_reg_br_prob_base (), REG_BR_PROB_BASE); - scale_loop_profile (prolog, prob_prolog, bound_prolog); + scale_loop_profile (prolog, prob_prolog.to_reg_br_prob_base (), + bound_prolog); } /* Update init address of DRs. */ vect_update_inits_of_drs (loop_vinfo, niters_prolog); @@ -1834,7 +1839,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, guard_to = split_edge (loop_preheader_edge (epilog)); guard_e = slpeel_add_loop_guard (guard_bb, guard_cond, guard_to, guard_bb, - inverse_probability (prob_vector), + prob_vector.invert (), irred_flag); e = EDGE_PRED (guard_to, 0); e = (e != guard_e ? e : EDGE_PRED (guard_to, 1)); @@ -1846,7 +1851,8 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, guard_to->count = guard_bb->count; single_succ_edge (guard_to)->count = guard_to->count; /* Scale probability of epilog loop back. */ - int scale_up = REG_BR_PROB_BASE * REG_BR_PROB_BASE / prob_vector; + int scale_up = REG_BR_PROB_BASE * REG_BR_PROB_BASE + / prob_vector.to_reg_br_prob_base (); scale_loop_frequencies (epilog, scale_up, REG_BR_PROB_BASE); } @@ -1875,7 +1881,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, guard_to = split_edge (single_exit (epilog)); guard_e = slpeel_add_loop_guard (guard_bb, guard_cond, guard_to, skip_vector ? anchor : guard_bb, - inverse_probability (prob_epilog), + prob_epilog.invert (), irred_flag); slpeel_update_phi_nodes_for_guard2 (loop, epilog, guard_e, single_exit (epilog)); @@ -1883,13 +1889,13 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, the guard_bb, which is the case when skip_vector is true. */ if (guard_bb != bb_before_epilog) { - prob_epilog = (combine_probabilities (prob_vector, prob_epilog) - + inverse_probability (prob_vector)); + prob_epilog = prob_vector * prob_epilog + prob_vector.invert (); - scale_bbs_frequencies_int (&bb_before_epilog, 1, prob_epilog, + scale_bbs_frequencies_int (&bb_before_epilog, 1, + prob_epilog.to_reg_br_prob_base (), REG_BR_PROB_BASE); } - scale_loop_profile (epilog, prob_epilog, bound); + scale_loop_profile (epilog, prob_epilog.to_reg_br_prob_base (), bound); } else slpeel_update_phi_nodes_for_lcssa (epilog); @@ -2171,7 +2177,10 @@ vect_loop_versioning (loop_vec_info loop_vinfo, /* We don't want to scale SCALAR_LOOP's frequencies, we need to scale LOOP's frequencies instead. */ nloop = loop_version (scalar_loop, cond_expr, &condition_bb, - prob, REG_BR_PROB_BASE - prob, + profile_probability::guessed_always ().apply_scale + (prob, REG_BR_PROB_BASE), + profile_probability::guessed_always ().apply_scale + (REG_BR_PROB_BASE - prob, REG_BR_PROB_BASE), REG_BR_PROB_BASE, REG_BR_PROB_BASE - prob, true); scale_loop_frequencies (loop, prob, REG_BR_PROB_BASE); /* CONDITION_BB was created above SCALAR_LOOP's preheader, @@ -2200,7 +2209,10 @@ vect_loop_versioning (loop_vec_info loop_vinfo, } else nloop = loop_version (loop, cond_expr, &condition_bb, - prob, REG_BR_PROB_BASE - prob, + profile_probability::guessed_always ().apply_scale + (prob, REG_BR_PROB_BASE), + profile_probability::guessed_always ().apply_scale + (REG_BR_PROB_BASE - prob, REG_BR_PROB_BASE), prob, REG_BR_PROB_BASE - prob, true); if (version_niter) diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 32a5caa391e..576a42fc42f 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -7123,21 +7123,25 @@ scale_profile_for_vect_loop (struct loop *loop, unsigned vf) if (!(freq_e > profile_count::from_gcov_type (1))) freq_e = profile_count::from_gcov_type (1); /* This should not overflow. */ - scale = freq_e.apply_scale (new_est_niter + 1, 1).probability_in (freq_h); + scale = freq_e.apply_scale (new_est_niter + 1, 1).probability_in (freq_h) + .to_reg_br_prob_base (); scale_loop_frequencies (loop, scale, REG_BR_PROB_BASE); } basic_block exit_bb = single_pred (loop->latch); edge exit_e = single_exit (loop); exit_e->count = loop_preheader_edge (loop)->count; - exit_e->probability = REG_BR_PROB_BASE / (new_est_niter + 1); + exit_e->probability = profile_probability::always () + .apply_scale (1, new_est_niter + 1); edge exit_l = single_pred_edge (loop->latch); - int prob = exit_l->probability; - exit_l->probability = REG_BR_PROB_BASE - exit_e->probability; + int prob = exit_l->probability.initialized_p () + ? exit_l->probability.to_reg_br_prob_base () : 0; + exit_l->probability = exit_e->probability.invert (); exit_l->count = exit_bb->count - exit_e->count; if (prob > 0) - scale_bbs_frequencies_int (&loop->latch, 1, exit_l->probability, prob); + scale_bbs_frequencies_int (&loop->latch, 1, + exit_l->probability.to_reg_br_prob_base (), prob); } /* Function vect_transform_loop. @@ -7660,7 +7664,7 @@ optimize_mask_stores (struct loop *loop) e->flags = EDGE_TRUE_VALUE; efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE); /* Put STORE_BB to likely part. */ - efalse->probability = PROB_UNLIKELY; + efalse->probability = profile_probability::unlikely (); store_bb->frequency = PROB_ALWAYS - EDGE_FREQUENCY (efalse); make_edge (store_bb, join_bb, EDGE_FALLTHRU); if (dom_info_available_p (CDI_DOMINATORS)) diff --git a/gcc/ubsan.c b/gcc/ubsan.c index bd0588b89ba..8ea352a69d4 100644 --- a/gcc/ubsan.c +++ b/gcc/ubsan.c @@ -799,7 +799,7 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip) /* Make an edge coming from the 'cond block' into the 'then block'; this edge is unlikely taken, so set up the probability accordingly. */ e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); - e->probability = PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_unlikely (); /* Connect 'then block' with the 'else block'. This is needed as the ubsan routines we call in the 'then block' are not noreturn. @@ -810,7 +810,7 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip) e = find_edge (cond_bb, fallthru_bb); e->flags = EDGE_FALSE_VALUE; e->count = cond_bb->count; - e->probability = REG_BR_PROB_BASE - PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_likely (); /* Update dominance info for the newly created then_bb; note that fallthru_bb's dominance info has already been updated by @@ -873,13 +873,13 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip) this edge is unlikely taken, so set up the probability accordingly. */ e = make_edge (cond1_bb, then_bb, EDGE_TRUE_VALUE); - e->probability = PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_unlikely (); /* Set up the fallthrough basic block. */ e = find_edge (cond1_bb, cond2_bb); e->flags = EDGE_FALSE_VALUE; e->count = cond1_bb->count; - e->probability = REG_BR_PROB_BASE - PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_likely (); /* Update dominance info. */ if (dom_info_available_p (CDI_DOMINATORS)) diff --git a/gcc/value-prof.c b/gcc/value-prof.c index 2fed338f01a..23b8dc26471 100644 --- a/gcc/value-prof.c +++ b/gcc/value-prof.c @@ -104,11 +104,6 @@ along with GCC; see the file COPYING3. If not see and gimple_value_profile_transformations table-driven, perhaps... */ -static tree gimple_divmod_fixed_value (gassign *, tree, int, gcov_type, - gcov_type); -static tree gimple_mod_pow2 (gassign *, int, gcov_type, gcov_type); -static tree gimple_mod_subtract (gassign *, int, int, int, gcov_type, - gcov_type, gcov_type); static bool gimple_divmod_fixed_value_transform (gimple_stmt_iterator *); static bool gimple_mod_pow2_value_transform (gimple_stmt_iterator *); static bool gimple_mod_subtract_transform (gimple_stmt_iterator *); @@ -693,7 +688,7 @@ gimple_value_profile_transformations (void) alter the original STMT. */ static tree -gimple_divmod_fixed_value (gassign *stmt, tree value, int prob, +gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob, gcov_type count, gcov_type all) { gassign *stmt1, *stmt2; @@ -753,16 +748,16 @@ gimple_divmod_fixed_value (gassign *stmt, tree value, int prob, e12->count = profile_count::from_gcov_type (count); e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE); - e13->probability = REG_BR_PROB_BASE - prob; + e13->probability = prob.invert (); e13->count = profile_count::from_gcov_type (all - count); remove_edge (e23); e24 = make_edge (bb2, bb4, EDGE_FALLTHRU); - e24->probability = REG_BR_PROB_BASE; + e24->probability = profile_probability::always (); e24->count = profile_count::from_gcov_type (count); - e34->probability = REG_BR_PROB_BASE; + e34->probability = profile_probability::always (); e34->count = profile_count::from_gcov_type (all - count); return tmp2; @@ -777,7 +772,7 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) enum tree_code code; gcov_type val, count, all; tree result, value, tree_val; - gcov_type prob; + profile_probability prob; gassign *stmt; stmt = dyn_cast <gassign *> (gsi_stmt (*si)); @@ -816,9 +811,9 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) /* Compute probability of taking the optimal path. */ if (all > 0) - prob = GCOV_COMPUTE_SCALE (count, all); + prob = profile_probability::probability_in_gcov_type (count, all); else - prob = 0; + prob = profile_probability::never (); if (sizeof (gcov_type) == sizeof (HOST_WIDE_INT)) tree_val = build_int_cst (get_gcov_type (), val); @@ -855,7 +850,7 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) the temp; it does not replace or alter the original STMT. */ static tree -gimple_mod_pow2 (gassign *stmt, int prob, gcov_type count, gcov_type all) +gimple_mod_pow2 (gassign *stmt, profile_probability prob, gcov_type count, gcov_type all) { gassign *stmt1, *stmt2, *stmt3; gcond *stmt4; @@ -918,16 +913,16 @@ gimple_mod_pow2 (gassign *stmt, int prob, gcov_type count, gcov_type all) e12->count = profile_count::from_gcov_type (count); e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE); - e13->probability = REG_BR_PROB_BASE - prob; + e13->probability = prob.invert (); e13->count = profile_count::from_gcov_type (all - count); remove_edge (e23); e24 = make_edge (bb2, bb4, EDGE_FALLTHRU); - e24->probability = REG_BR_PROB_BASE; + e24->probability = profile_probability::always (); e24->count = profile_count::from_gcov_type (count); - e34->probability = REG_BR_PROB_BASE; + e34->probability = profile_probability::always (); e34->count = profile_count::from_gcov_type (all - count); return result; @@ -942,7 +937,7 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) enum tree_code code; gcov_type count, wrong_values, all; tree lhs_type, result, value; - gcov_type prob; + profile_probability prob; gassign *stmt; stmt = dyn_cast <gassign *> (gsi_stmt (*si)); @@ -987,9 +982,9 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) return false; if (all > 0) - prob = GCOV_COMPUTE_SCALE (count, all); + prob = profile_probability::probability_in_gcov_type (count, all); else - prob = 0; + prob = profile_probability::never (); result = gimple_mod_pow2 (stmt, prob, count, all); @@ -1009,7 +1004,8 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) /* FIXME: Generalize the interface to handle NCOUNTS > 1. */ static tree -gimple_mod_subtract (gassign *stmt, int prob1, int prob2, int ncounts, +gimple_mod_subtract (gassign *stmt, profile_probability prob1, + profile_probability prob2, int ncounts, gcov_type count1, gcov_type count2, gcov_type all) { gassign *stmt1; @@ -1079,7 +1075,7 @@ gimple_mod_subtract (gassign *stmt, int prob1, int prob2, int ncounts, e12->flags &= ~EDGE_FALLTHRU; e12->flags |= EDGE_FALSE_VALUE; - e12->probability = REG_BR_PROB_BASE - prob1; + e12->probability = prob1.invert (); e12->count = profile_count::from_gcov_type (all - count1); e14 = make_edge (bb, bb4, EDGE_TRUE_VALUE); @@ -1091,14 +1087,14 @@ gimple_mod_subtract (gassign *stmt, int prob1, int prob2, int ncounts, e23->flags &= ~EDGE_FALLTHRU; e23->flags |= EDGE_FALSE_VALUE; e23->count = profile_count::from_gcov_type (all - count1 - count2); - e23->probability = REG_BR_PROB_BASE - prob2; + e23->probability = prob2.invert (); e24 = make_edge (bb2, bb4, EDGE_TRUE_VALUE); e24->probability = prob2; e24->count = profile_count::from_gcov_type (count2); } - e34->probability = REG_BR_PROB_BASE; + e34->probability = profile_probability::always (); e34->count = profile_count::from_gcov_type (all - count1 - count2); return result; @@ -1113,7 +1109,7 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) enum tree_code code; gcov_type count, wrong_values, all; tree lhs_type, result; - gcov_type prob1, prob2; + profile_probability prob1, prob2; unsigned int i, steps; gcov_type count1, count2; gassign *stmt; @@ -1181,12 +1177,12 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) /* Compute probability of taking the optimal path(s). */ if (all > 0) { - prob1 = GCOV_COMPUTE_SCALE (count1, all); - prob2 = GCOV_COMPUTE_SCALE (count2, all); + prob1 = profile_probability::probability_in_gcov_type (count1, all); + prob2 = profile_probability::probability_in_gcov_type (count2, all); } else { - prob1 = prob2 = 0; + prob1 = prob2 = profile_probability::never (); } /* In practice, "steps" is always 2. This interface reflects this, @@ -1316,7 +1312,7 @@ check_ic_target (gcall *call_stmt, struct cgraph_node *target) gcall * gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call, - int prob, profile_count count, profile_count all) + profile_probability prob, profile_count count, profile_count all) { gcall *dcall_stmt; gassign *load_stmt; @@ -1386,7 +1382,7 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call, /* The indirect call might be noreturn. */ if (e_ij != NULL) { - e_ij->probability = REG_BR_PROB_BASE; + e_ij->probability = profile_probability::always (); e_ij->count = all - count; e_ij = single_pred_edge (split_edge (e_ij)); } @@ -1402,7 +1398,7 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call, e_cd->count = count; e_ci = make_edge (cond_bb, icall_bb, EDGE_FALSE_VALUE); - e_ci->probability = REG_BR_PROB_BASE - prob; + e_ci->probability = prob.invert (); e_ci->count = all - count; remove_edge (e_di); @@ -1414,12 +1410,12 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call, else { e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU); - e_dj->probability = REG_BR_PROB_BASE; + e_dj->probability = profile_probability::always (); e_dj->count = count; e_ij->count = all - count; } - e_ij->probability = REG_BR_PROB_BASE; + e_ij->probability = profile_probability::always (); } /* Insert PHI node for the call result if necessary. */ @@ -1497,6 +1493,8 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call, if (e_eh->flags & (EDGE_EH | EDGE_ABNORMAL)) { e = make_edge (dcall_bb, e_eh->dest, e_eh->flags); + e->probability = e_eh->probability; + e->count = e_eh->count; for (gphi_iterator psi = gsi_start_phis (e_eh->dest); !gsi_end_p (psi); gsi_next (&psi)) { @@ -1645,7 +1643,7 @@ interesting_stringop_to_profile_p (gcall *call, int *size_arg) assuming we'll propagate a true constant into ICALL_SIZE later. */ static void -gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, int prob, +gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, profile_probability prob, gcov_type count, gcov_type all) { gassign *tmp_stmt; @@ -1709,16 +1707,16 @@ gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, int prob, e_ci->count = profile_count::from_gcov_type (count); e_cv = make_edge (cond_bb, vcall_bb, EDGE_FALSE_VALUE); - e_cv->probability = REG_BR_PROB_BASE - prob; + e_cv->probability = prob.invert (); e_cv->count = profile_count::from_gcov_type (all - count); remove_edge (e_iv); e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU); - e_ij->probability = REG_BR_PROB_BASE; + e_ij->probability = profile_probability::always (); e_ij->count = profile_count::from_gcov_type (count); - e_vj->probability = REG_BR_PROB_BASE; + e_vj->probability = profile_probability::always (); e_vj->count = profile_count::from_gcov_type (all - count); /* Insert PHI node for the call result if necessary. */ @@ -1753,7 +1751,7 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) gcov_type count, all, val; tree dest, src; unsigned int dest_align, src_align; - gcov_type prob; + profile_probability prob; tree tree_val; int size_arg; @@ -1788,9 +1786,9 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count)) return false; if (all > 0) - prob = GCOV_COMPUTE_SCALE (count, all); + prob = profile_probability::probability_in_gcov_type (count, all); else - prob = 0; + prob = profile_probability::never (); dest = gimple_call_arg (stmt, 0); dest_align = get_pointer_alignment (dest); diff --git a/gcc/value-prof.h b/gcc/value-prof.h index 92649ecd322..f72bb2d2241 100644 --- a/gcc/value-prof.h +++ b/gcc/value-prof.h @@ -90,8 +90,8 @@ void gimple_move_stmt_histograms (struct function *, gimple *, gimple *); void verify_histograms (void); void free_histograms (function *); void stringop_block_profile (gimple *, unsigned int *, HOST_WIDE_INT *); -gcall *gimple_ic (gcall *, struct cgraph_node *, int, profile_count, - profile_count); +gcall *gimple_ic (gcall *, struct cgraph_node *, profile_probability, + profile_count, profile_count); bool check_ic_target (gcall *, struct cgraph_node *); |