From 954a2d23af28ec77b97d9e4bbdb0de1beeb57ec6 Mon Sep 17 00:00:00 2001 From: jakub Date: Tue, 27 Jun 2017 09:13:53 +0000 Subject: * predict.c (test_prediction_value_range): Use -1U instead of -1 to avoid narrowing conversion warning. * dumpfile.c (dump_options): Wrap all value into dump_flags_t cast to avoid narrowing conversion warning. * opt-functions.awk (var_ref): Return (unsigned short) -1 instead of -1. * optc-gen.awk (END): Expect (unsigned short) -1 instead of -1. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@249683 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 44151bc2e6c..71d6d0a5bb1 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -4031,7 +4031,7 @@ test_prediction_value_range () { branch_predictor predictors[] = { #include "predict.def" - {NULL, -1} + {NULL, -1U} }; for (unsigned i = 0; predictors[i].name != NULL; i++) -- cgit v1.2.1 From 720cfc437994ab33ce3047b15eddcb727a26f2c1 Mon Sep 17 00:00:00 2001 From: hubicka Date: Thu, 29 Jun 2017 16:40:53 +0000 Subject: * asan.c (asan_emit_stack_protection): Update. (create_cond_insert_point): Update. * auto-profile.c (afdo_propagate_circuit): Update. * basic-block.h (struct edge_def): Turn probability to profile_probability. (EDGE_FREQUENCY): Update. * bb-reorder.c (find_traces_1_round): Update. (better_edge_p): Update. (sanitize_hot_paths): Update. * cfg.c (unchecked_make_edge): Initialize probability to uninitialized. (make_single_succ_edge): Update. (check_bb_profile): Update. (dump_edge_info): Update. (update_bb_profile_for_threading): Update. * cfganal.c (connect_infinite_loops_to_exit): Initialize new edge probabilitycount to 0. * cfgbuild.c (compute_outgoing_frequencies): Update. * cfgcleanup.c (try_forward_edges): Update. (outgoing_edges_match): Update. (try_crossjump_to_edge): Update. * cfgexpand.c (expand_gimple_cond): Update make_single_succ_edge. (expand_gimple_tailcall): Update. (construct_init_block): Use make_single_succ_edge. (construct_exit_block): Use make_single_succ_edge. * cfghooks.c (verify_flow_info): Update. (redirect_edge_succ_nodup): Update. (split_edge): Update. (account_profile_record): Update. * cfgloopanal.c (single_likely_exit): Update. * cfgloopmanip.c (scale_loop_profile): Update. (set_zero_probability): Remove. (duplicate_loop_to_header_edge): Update. * cfgloopmanip.h (loop_version): Update prototype. * cfgrtl.c (try_redirect_by_replacing_jump): Update. (force_nonfallthru_and_redirect): Update. (update_br_prob_note): Update. (rtl_verify_edges): Update. (purge_dead_edges): Update. (rtl_lv_add_condition_to_bb): Update. * cgraph.c: (cgraph_edge::redirect_call_stmt_to_calle): Update. * cgraphunit.c (init_lowered_empty_function): Update. (cgraph_node::expand_thunk): Update. * cilk-common.c: Include profile-count.h * dojump.c (inv): Remove. (jumpifnot): Update. (jumpifnot_1): Update. (do_jump_1): Update. (do_jump): Update. (do_jump_by_parts_greater_rtx): Update. (do_compare_rtx_and_jump): Update. * dojump.h (jumpifnot, jumpifnot_1, jumpif_1, jumpif, do_jump, do_jump_1. do_compare_rtx_and_jump): Update prototype. * dwarf2cfi.c: Include profile-count.h * except.c (dw2_build_landing_pads): Use make_single_succ_edge. (sjlj_emit_dispatch_table): Likewise. * explow.c: Include profile-count.h * expmed.c (emit_store_flag_force): Update. (do_cmp_and_jump): Update. * expr.c (compare_by_pieces_d::generate): Update. (compare_by_pieces_d::finish_mode): Update. (emit_block_move_via_loop): Update. (store_expr_with_bounds): Update. (store_constructor): Update. (expand_expr_real_2): Update. (expand_expr_real_1): Update. * expr.h (try_casesi, try_tablejump): Update prototypes. * gimple-pretty-print.c (dump_probability): Update. (dump_profile): New. (dump_gimple_label): Update. (dump_gimple_bb_header): Update. * graph.c (draw_cfg_node_succ_edges): Update. * hsa-gen.c (convert_switch_statements): Update. * ifcvt.c (cheap_bb_rtx_cost_p): Update. (find_if_case_1): Update. (find_if_case_2): Update. * internal-fn.c (expand_arith_overflow_result_store): Update. (expand_addsub_overflow): Update. (expand_neg_overflow): Update. (expand_mul_overflow): Update. (expand_vector_ubsan_overflow): Update. * ipa-cp.c (good_cloning_opportunity_p): Update. * ipa-split.c (split_function): Use make_single_succ_edge. * ipa-utils.c (ipa_merge_profiles): Update. * loop-doloop.c (add_test): Update. (doloop_modify): Update. * loop-unroll.c (compare_and_jump_seq): Update. (unroll_loop_runtime_iterations): Update. * lra-constraints.c (lra_inheritance): Update. * lto-streamer-in.c (input_cfg): Update. * lto-streamer-out.c (output_cfg): Update. * mcf.c (adjust_cfg_counts): Update. * modulo-sched.c (sms_schedule): Update. * omp-expand.c (expand_omp_for_init_counts): Update. (extract_omp_for_update_vars): Update. (expand_omp_ordered_sink): Update. (expand_omp_for_ordered_loops): Update. (expand_omp_for_generic): Update. (expand_omp_for_static_nochunk): Update. (expand_omp_for_static_chunk): Update. (expand_cilk_for): Update. (expand_omp_simd): Update. (expand_omp_taskloop_for_outer): Update. (expand_omp_taskloop_for_inner): Update. * omp-simd-clone.c (simd_clone_adjust): Update. * optabs.c (expand_doubleword_shift): Update. (expand_abs): Update. (emit_cmp_and_jump_insn_1): Update. (expand_compare_and_swap_loop): Update. * optabs.h (emit_cmp_and_jump_insns): Update prototype. * predict.c (predictable_edge_p): Update. (edge_probability_reliable_p): Update. (set_even_probabilities): Update. (combine_predictions_for_insn): Update. (combine_predictions_for_bb): Update. (propagate_freq): Update. (estimate_bb_frequencies): Update. (force_edge_cold): Update. * profile-count.c (profile_count::dump): Add missing space into dump. (profile_count::debug): Add newline. (profile_count::differs_from_p): Explicitly convert to unsigned. (profile_count::stream_in): Update. (profile_probability::dump): New member function. (profile_probability::debug): New member function. (profile_probability::differs_from_p): New member function. (profile_probability::differs_lot_from_p): New member function. (profile_probability::stream_in): New member function. (profile_probability::stream_out): New member function. * profile-count.h (profile_count_quality): Rename to ... (profile_quality): ... this one. (profile_probability): New. (profile_count): Update. * profile.c (compute_branch_probabilities): Update. * recog.c (peep2_attempt): Update. * sched-ebb.c (schedule_ebbs): Update. * sched-rgn.c (find_single_block_region): Update. (compute_dom_prob_ps): Update. (schedule_region): Update. * sel-sched-ir.c (compute_succs_info): Update. * stmt.c (struct case_node): Update. (do_jump_if_equal): Update. (get_outgoing_edge_probs): Update. (conditional_probability): Update. (emit_case_dispatch_table): Update. (expand_case): Update. (expand_sjlj_dispatch_table): Update. (emit_case_nodes): Update. * targhooks.c: Update. * tracer.c (better_p): Update. (find_best_successor): Update. * trans-mem.c (expand_transaction): Update. * tree-call-cdce.c: Update. * tree-cfg.c (gimple_split_edge): Upate. (move_sese_region_to_fn): Upate. * tree-cfgcleanup.c (cleanup_control_expr_graph): Upate. * tree-eh.c (lower_resx): Upate. (cleanup_empty_eh_move_lp): Upate. * tree-if-conv.c (version_loop_for_if_conversion): Update. * tree-inline.c (copy_edges_for_bb): Update. (copy_cfg_body): Update. * tree-parloops.c (gen_parallel_loop): Update. * tree-profile.c (gimple_gen_ic_func_profiler): Update. (gimple_gen_time_profiler): Update. * tree-ssa-dce.c (remove_dead_stmt): Update. * tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update. * tree-ssa-loop-im.c (execute_sm_if_changed): Update. * tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update. (unloop_loops): Update. (try_peel_loop): Update. * tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update. * tree-ssa-loop-split.c (connect_loops): Update. (split_loop): Update. * tree-ssa-loop-unswitch.c (tree_unswitch_loop): Update. (hoist_guard): Update. * tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update. * tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update. (value_replacement): Update. * tree-ssa-reassoc.c (branch_fixup): Update. * tree-ssa-tail-merge.c (replace_block_by): Update. * tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update. (create_edge_and_update_destination_phis): Update. (compute_path_counts): Update. (recompute_probabilities): Update. (update_joiner_offpath_counts): Update. (freqs_to_counts_path): Update. (duplicate_thread_path): Update. * tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update. (struct switch_conv_info): Update. (gen_inbound_check): Update. * tree-vect-loop-manip.c (slpeel_add_loop_guard): Update. (vect_do_peeling): Update. (vect_loop_versioning): Update. * tree-vect-loop.c (scale_profile_for_vect_loop): Update. (optimize_mask_stores): Update. * ubsan.c (ubsan_expand_null_ifn): Update. * value-prof.c (gimple_divmod_fixed_value): Update. (gimple_divmod_fixed_value_transform): Update. (gimple_mod_pow2): Update. (gimple_mod_pow2_value_transform): Update. (gimple_mod_subtract): Update. (gimple_mod_subtract_transform): Update. (gimple_ic): Update. (gimple_stringop_fixed_value): Update. (gimple_stringops_transform): Update. * value-prof.h: Update. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@249800 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 113 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 45 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 71d6d0a5bb1..4ee36ffc8c3 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -58,6 +58,7 @@ along with GCC; see the file COPYING3. If not see #include "ipa-utils.h" #include "gimple-pretty-print.h" #include "selftest.h" +#include "cfgrtl.h" /* Enum with reasons why a predictor is ignored. */ @@ -404,11 +405,11 @@ optimize_loop_nest_for_size_p (struct loop *loop) bool predictable_edge_p (edge e) { - if (profile_status_for_fn (cfun) == PROFILE_ABSENT) + if (!e->probability.initialized_p ()) return false; - if ((e->probability + if ((e->probability.to_reg_br_prob_base () <= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100) - || (REG_BR_PROB_BASE - e->probability + || (REG_BR_PROB_BASE - e->probability.to_reg_br_prob_base () <= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100)) return true; return false; @@ -539,7 +540,7 @@ probability_reliable_p (int prob) bool edge_probability_reliable_p (const_edge e) { - return probability_reliable_p (e->probability); + return e->probability.reliable_p (); } /* Same predicate as edge_probability_reliable_p, working on notes. */ @@ -859,12 +860,13 @@ set_even_probabilities (basic_block bb, if (!unlikely_executed_edge_p (e)) { if (unlikely_edges != NULL && unlikely_edges->contains (e)) - e->probability = PROB_VERY_UNLIKELY; + e->probability = profile_probability::very_unlikely (); else - e->probability = (REG_BR_PROB_BASE + c / 2) / c; + e->probability = profile_probability::guessed_always () + .apply_scale (1, c); } else - e->probability = 0; + e->probability = profile_probability::never (); } /* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB @@ -971,20 +973,23 @@ combine_predictions_for_insn (rtx_insn *insn, basic_block bb) conditional jump. */ if (!single_succ_p (bb)) { - BRANCH_EDGE (bb)->probability = combined_probability; + BRANCH_EDGE (bb)->probability + = profile_probability::from_reg_br_prob_base (combined_probability); FALLTHRU_EDGE (bb)->probability - = REG_BR_PROB_BASE - combined_probability; + = BRANCH_EDGE (bb)->probability.invert (); } } else if (!single_succ_p (bb)) { int prob = XINT (prob_note, 0); - BRANCH_EDGE (bb)->probability = prob; - FALLTHRU_EDGE (bb)->probability = REG_BR_PROB_BASE - prob; + BRANCH_EDGE (bb)->probability + = profile_probability::from_reg_br_prob_base (prob); + FALLTHRU_EDGE (bb)->probability + = BRANCH_EDGE (bb)->probability.invert (); } else - single_succ_edge (bb)->probability = REG_BR_PROB_BASE; + single_succ_edge (bb)->probability = profile_probability::always (); } /* Edge prediction hash traits. */ @@ -1129,6 +1134,8 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) if (!first) first = e; } + else if (!e->probability.initialized_p ()) + e->probability = profile_probability::never (); /* When there is no successor or only one choice, prediction is easy. @@ -1173,8 +1180,8 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) nedges, bb->index); FOR_EACH_EDGE (e, ei, bb->succs) if (!unlikely_executed_edge_p (e)) - dump_prediction (dump_file, PRED_COMBINED, e->probability, - bb, REASON_NONE, e); + dump_prediction (dump_file, PRED_COMBINED, + e->probability.to_reg_br_prob_base (), bb, REASON_NONE, e); } } return; @@ -1284,8 +1291,9 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) if (!bb->count.initialized_p () && !dry_run) { - first->probability = combined_probability; - second->probability = REG_BR_PROB_BASE - combined_probability; + first->probability + = profile_probability::from_reg_br_prob_base (combined_probability); + second->probability = first->probability.invert (); } } @@ -3042,7 +3050,7 @@ propagate_freq (basic_block head, bitmap tovisit) * BLOCK_INFO (e->src)->frequency / REG_BR_PROB_BASE); */ - sreal tmp = e->probability; + sreal tmp = e->probability.to_reg_br_prob_base (); tmp *= BLOCK_INFO (e->src)->frequency; tmp *= real_inv_br_prob_base; frequency += tmp; @@ -3074,7 +3082,7 @@ propagate_freq (basic_block head, bitmap tovisit) = ((e->probability * BLOCK_INFO (bb)->frequency) / REG_BR_PROB_BASE); */ - sreal tmp = e->probability; + sreal tmp = e->probability.to_reg_br_prob_base (); tmp *= BLOCK_INFO (bb)->frequency; EDGE_INFO (e)->back_edge_prob = tmp * real_inv_br_prob_base; } @@ -3534,7 +3542,7 @@ estimate_bb_frequencies (bool force) mark_dfs_back_edges (); single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))->probability = - REG_BR_PROB_BASE; + profile_probability::always (); /* Set up block info for each basic block. */ alloc_aux_for_blocks (sizeof (block_info)); @@ -3546,7 +3554,8 @@ estimate_bb_frequencies (bool force) FOR_EACH_EDGE (e, ei, bb->succs) { - EDGE_INFO (e)->back_edge_prob = e->probability; + EDGE_INFO (e)->back_edge_prob + = e->probability.to_reg_br_prob_base (); EDGE_INFO (e)->back_edge_prob *= real_inv_br_prob_base; } } @@ -3898,16 +3907,18 @@ void force_edge_cold (edge e, bool impossible) { profile_count count_sum = profile_count::zero (); - int prob_sum = 0; + profile_probability prob_sum = profile_probability::never (); edge_iterator ei; edge e2; profile_count old_count = e->count; - int old_probability = e->probability; - int prob_scale = REG_BR_PROB_BASE; + profile_probability old_probability = e->probability; bool uninitialized_exit = false; + profile_probability goal = (impossible ? profile_probability::never () + : profile_probability::very_unlikely ()); + /* If edge is already improbably or cold, just return. */ - if (e->probability <= (impossible ? PROB_VERY_UNLIKELY : 0) + if (e->probability <= goal && (!impossible || e->count == profile_count::zero ())) return; FOR_EACH_EDGE (e2, ei, e->src->succs) @@ -3917,24 +3928,26 @@ force_edge_cold (edge e, bool impossible) count_sum += e2->count; else uninitialized_exit = true; - prob_sum += e2->probability; + if (e2->probability.initialized_p ()) + prob_sum += e2->probability; } /* If there are other edges out of e->src, redistribute probabilitity there. */ - if (prob_sum) + if (prob_sum > profile_probability::never ()) { - e->probability - = MIN (e->probability, impossible ? 0 : PROB_VERY_UNLIKELY); + if (!(e->probability < goal)) + e->probability = goal; if (impossible) e->count = profile_count::zero (); - else if (old_probability) - e->count = e->count.apply_scale (e->probability, old_probability); + else if (old_probability > profile_probability::never ()) + e->count = e->count.apply_probability (e->probability + / old_probability); else e->count = e->count.apply_scale (1, REG_BR_PROB_BASE); - prob_scale = RDIV ((REG_BR_PROB_BASE - e->probability) * REG_BR_PROB_BASE, - prob_sum); + profile_probability prob_comp = prob_sum / e->probability.invert (); + if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, "Making edge %i->%i %s by redistributing " "probability to other edges.\n", @@ -3946,30 +3959,39 @@ force_edge_cold (edge e, bool impossible) { if (count_sum > 0) e2->count.apply_scale (count_sum2, count_sum); - e2->probability = RDIV (e2->probability * prob_scale, - REG_BR_PROB_BASE); + e2->probability /= prob_comp; } + if (current_ir_type () != IR_GIMPLE) + update_br_prob_note (e->src); } /* If all edges out of e->src are unlikely, the basic block itself is unlikely. */ else { - e->probability = REG_BR_PROB_BASE; + e->probability = profile_probability::always (); + if (current_ir_type () != IR_GIMPLE) + update_br_prob_note (e->src); if (e->src->count == profile_count::zero ()) return; if (count_sum == profile_count::zero () && !uninitialized_exit && impossible) { bool found = false; - for (gimple_stmt_iterator gsi = gsi_start_bb (e->src); - !gsi_end_p (gsi); gsi_next (&gsi)) - { - if (stmt_can_terminate_bb_p (gsi_stmt (gsi))) - { - found = true; - break; - } - } + if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun)) + ; + else if (current_ir_type () == IR_GIMPLE) + for (gimple_stmt_iterator gsi = gsi_start_bb (e->src); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + if (stmt_can_terminate_bb_p (gsi_stmt (gsi))) + { + found = true; + break; + } + } + /* FIXME: Implement RTL path. */ + else + found = true; if (!found) { if (dump_file && (dump_flags & TDF_DETAILS)) @@ -3989,7 +4011,8 @@ force_edge_cold (edge e, bool impossible) This in general is difficult task to do, but handle special case when BB has only one predecestor. This is common case when we are updating after loop transforms. */ - if (!prob_sum && count_sum == profile_count::zero () + if (!(prob_sum > profile_probability::never ()) + && count_sum == profile_count::zero () && single_pred_p (e->src) && e->src->frequency > (impossible ? 0 : 1)) { int old_frequency = e->src->frequency; -- cgit v1.2.1 From ca69b0696432d6230a0cac12b1f9fc939ec1eae1 Mon Sep 17 00:00:00 2001 From: hubicka Date: Sat, 1 Jul 2017 20:46:40 +0000 Subject: * cfg.c (scale_bbs_frequencies): New function. * cfg.h (scale_bbs_frequencies): Declare it. * cfgloopanal.c (single_likely_exit): Cleanup. * cfgloopmanip.c (scale_loop_frequencies): Take profile_probability as parameter. (scale_loop_profile): Likewise. (loop_version): Likewise. (create_empty_loop_on_edge): Update. * cfgloopmanip.h (scale_loop_frequencies, scale_loop_profile, scale_loop_frequencies, scale_loop_profile, loopify, loop_version): Update prototypes. * modulo-sched.c (sms_schedule): Update. * predict.c (unlikely_executed_edge_p): Also check probability. (probably_never_executed_edge_p): Fix typo. * tree-if-conv.c (version_loop_for_if_conversion): Update. * tree-parloops.c (gen_parallel_loop): Update. * tree-ssa-loop-ivcanon.c (try_peel_loop): Update. * tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update. * tree-ssa-loop-split.c (split_loop): Update. * tree-ssa-loop-unswitch.c (tree_unswitch_loop): Update. * tree-vect-loop-manip.c (vect_do_peeling): Update. (vect_loop_versioning): Update. * tree-vect-loop.c (scale_profile_for_vect_loop): Update. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@249872 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 4ee36ffc8c3..4d01bf357ef 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -245,7 +245,8 @@ probably_never_executed_bb_p (struct function *fun, const_basic_block bb) static bool unlikely_executed_edge_p (edge e) { - return e->count == profile_count::zero () + return (e->count == profile_count::zero () + || e->probability == profile_probability::never ()) || (e->flags & (EDGE_EH | EDGE_FAKE)); } @@ -254,8 +255,8 @@ unlikely_executed_edge_p (edge e) bool probably_never_executed_edge_p (struct function *fun, edge e) { - if (e->count.initialized_p ()) - unlikely_executed_edge_p (e); + if (unlikely_executed_edge_p (e)) + return true; return probably_never_executed (fun, e->count, EDGE_FREQUENCY (e)); } -- cgit v1.2.1 From 4bb697cd1dc105710e976b4bb45bcac242c71a6f Mon Sep 17 00:00:00 2001 From: hubicka Date: Mon, 3 Jul 2017 14:40:46 +0000 Subject: PR middle-end/81290 * predict.c (force_edge_cold): Be more careful about propagation backward. * profile-count.h (profile_probability::guessed, profile_probability::fdo, profile_count::guessed, profile_count::fdo): New. * tree-ssa-threadupdate.c (recompute_probabilities): Result is guessed. * gcc.c-torture/compile/pr81290.c: New. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@249924 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 4d01bf357ef..2a7dbfa4d9b 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -3962,15 +3962,26 @@ force_edge_cold (edge e, bool impossible) e2->count.apply_scale (count_sum2, count_sum); e2->probability /= prob_comp; } - if (current_ir_type () != IR_GIMPLE) + if (current_ir_type () != IR_GIMPLE + && e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)) update_br_prob_note (e->src); } /* If all edges out of e->src are unlikely, the basic block itself is unlikely. */ else { - e->probability = profile_probability::always (); - if (current_ir_type () != IR_GIMPLE) + if (prob_sum == profile_probability::never ()) + e->probability = profile_probability::always (); + else + { + if (impossible) + e->probability = profile_probability::never (); + /* If BB has some edges out that are not impossible, we can not + assume that BB itself is. */ + impossible = false; + } + if (current_ir_type () != IR_GIMPLE + && e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)) update_br_prob_note (e->src); if (e->src->count == profile_count::zero ()) return; -- cgit v1.2.1 From 61cb18166c6a1507af78ce73a4e3c8a21b0cb8ab Mon Sep 17 00:00:00 2001 From: hubicka Date: Sun, 16 Jul 2017 16:10:58 +0000 Subject: * profile-count.h (profile_probability::from_reg_br_prob_note, profile_probability::to_reg_br_prob_note): New functions. * doc/rtl.texi (REG_BR_PROB_NOTE): Update documentation. * reg-notes.h (REG_BR_PROB, REG_BR_PRED): Update docs. * predict.c (probability_reliable_p): Update. (edge_probability_reliable_p): Update. (br_prob_note_reliable_p): Update. (invert_br_probabilities): Update. (add_reg_br_prob_note): New function. (combine_predictions_for_insn): Update. * asan.c (asan_clear_shadow): Update. * cfgbuild.c (compute_outgoing_frequencies): Update. * cfgrtl.c (force_nonfallthru_and_redirect): Update. (update_br_prob_note): Update. (rtl_verify_edges): Update. (purge_dead_edges): Update. (fixup_reorder_chain): Update. * emit-rtl.c (try_split): Update. * ifcvt.c (cond_exec_process_insns): Update. (cond_exec_process_if_block): Update. (dead_or_predicable): Update. * internal-fn.c (expand_addsub_overflow): Update. (expand_neg_overflow): Update. (expand_mul_overflow): Update. * loop-doloop.c (doloop_modify): Update. * loop-unroll.c (compare_and_jump_seq): Update. * optabs.c (emit_cmp_and_jump_insn_1): Update. * predict.h: Update. * reorg.c (mostly_true_jump): Update. * rtl.h: Update. * config/aarch64/aarch64.c (aarch64_emit_unlikely_jump): Update. * config/alpha/alpha.c (emit_unlikely_jump): Update. * config/arc/arc.c: (emit_unlikely_jump): Update. * config/arm/arm.c: (emit_unlikely_jump): Update. * config/bfin/bfin.c (cbranch_predicted_taken_p): Update. * config/frv/frv.c (frv_print_operand_jump_hint): Update. * config/i386/i386.c (ix86_expand_split_stack_prologue): Update. (ix86_print_operand): Update. (ix86_split_fp_branch): Update. (predict_jump): Update. * config/ia64/ia64.c (ia64_print_operand): Update. * config/mmix/mmix.c (mmix_print_operand): Update. * config/powerpcspe/powerpcspe.c (output_cbranch): Update. (rs6000_expand_split_stack_prologue): Update. * config/rs6000/rs6000.c: Update. * config/s390/s390.c (s390_expand_vec_strlen): Update. (s390_expand_vec_movstr): Update. (s390_expand_cs_tdsi): Update. (s390_expand_split_stack_prologue): Update. * config/sh/sh.c (sh_print_operand): Update. (expand_cbranchsi4): Update. (expand_cbranchdi4): Update. * config/sparc/sparc.c (output_v9branch): Update. * config/spu/spu.c (get_branch_target): Update. (ea_load_store_inline): Update. * config/tilegx/tilegx.c (cbranch_predicted_p): Update. * config/tilepro/tilepro.c: Update. * gcc.dg/predict-8.c: Update. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@250239 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 57 ++++++++++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 2a7dbfa4d9b..310d9b0aced 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -513,35 +513,11 @@ edge_predicted_by_p (edge e, enum br_predictor predictor, bool taken) return false; } -/* Return true when the probability of edge is reliable. - - The profile guessing code is good at predicting branch outcome (ie. - taken/not taken), that is predicted right slightly over 75% of time. - It is however notoriously poor on predicting the probability itself. - In general the profile appear a lot flatter (with probabilities closer - to 50%) than the reality so it is bad idea to use it to drive optimization - such as those disabling dynamic branch prediction for well predictable - branches. - - There are two exceptions - edges leading to noreturn edges and edges - predicted by number of iterations heuristics are predicted well. This macro - should be able to distinguish those, but at the moment it simply check for - noreturn heuristic that is only one giving probability over 99% or bellow - 1%. In future we might want to propagate reliability information across the - CFG if we find this information useful on multiple places. */ -static bool -probability_reliable_p (int prob) -{ - return (profile_status_for_fn (cfun) == PROFILE_READ - || (profile_status_for_fn (cfun) == PROFILE_GUESSED - && (prob <= HITRATE (1) || prob >= HITRATE (99)))); -} - /* Same predicate as above, working on edges. */ bool edge_probability_reliable_p (const_edge e) { - return e->probability.reliable_p (); + return e->probability.probably_reliable_p (); } /* Same predicate as edge_probability_reliable_p, working on notes. */ @@ -549,7 +525,8 @@ bool br_prob_note_reliable_p (const_rtx note) { gcc_assert (REG_NOTE_KIND (note) == REG_BR_PROB); - return probability_reliable_p (XINT (note, 0)); + return profile_probability::from_reg_br_prob_note + (XINT (note, 0)).probably_reliable_p (); } static void @@ -723,7 +700,8 @@ invert_br_probabilities (rtx insn) for (note = REG_NOTES (insn); note; note = XEXP (note, 1)) if (REG_NOTE_KIND (note) == REG_BR_PROB) - XINT (note, 0) = REG_BR_PROB_BASE - XINT (note, 0); + XINT (note, 0) = profile_probability::from_reg_br_prob_note + (XINT (note, 0)).invert ().to_reg_br_prob_note (); else if (REG_NOTE_KIND (note) == REG_BR_PRED) XEXP (XEXP (note, 0), 1) = GEN_INT (REG_BR_PROB_BASE - INTVAL (XEXP (XEXP (note, 0), 1))); @@ -870,6 +848,15 @@ set_even_probabilities (basic_block bb, e->probability = profile_probability::never (); } +/* Add REG_BR_PROB note to JUMP with PROB. */ + +void +add_reg_br_prob_note (rtx_insn *jump, profile_probability prob) +{ + gcc_checking_assert (JUMP_P (jump) && !find_reg_note (jump, REG_BR_PROB, 0)); + add_int_reg_note (jump, REG_BR_PROB, prob.to_reg_br_prob_note ()); +} + /* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB note if not already present. Remove now useless REG_BR_PRED notes. */ @@ -968,26 +955,26 @@ combine_predictions_for_insn (rtx_insn *insn, basic_block bb) if (!prob_note) { - add_int_reg_note (insn, REG_BR_PROB, combined_probability); + profile_probability p + = profile_probability::from_reg_br_prob_base (combined_probability); + add_reg_br_prob_note (insn, p); /* Save the prediction into CFG in case we are seeing non-degenerated conditional jump. */ if (!single_succ_p (bb)) { - BRANCH_EDGE (bb)->probability - = profile_probability::from_reg_br_prob_base (combined_probability); + BRANCH_EDGE (bb)->probability = p; FALLTHRU_EDGE (bb)->probability = BRANCH_EDGE (bb)->probability.invert (); } } else if (!single_succ_p (bb)) { - int prob = XINT (prob_note, 0); + profile_probability prob = profile_probability::from_reg_br_prob_note + (XINT (prob_note, 0)); - BRANCH_EDGE (bb)->probability - = profile_probability::from_reg_br_prob_base (prob); - FALLTHRU_EDGE (bb)->probability - = BRANCH_EDGE (bb)->probability.invert (); + BRANCH_EDGE (bb)->probability = prob; + FALLTHRU_EDGE (bb)->probability = prob.invert (); } else single_succ_edge (bb)->probability = profile_probability::always (); -- cgit v1.2.1 From c7a0aa224dbb257d57d7b68305b5936a189b2a74 Mon Sep 17 00:00:00 2001 From: hubicka Date: Tue, 18 Jul 2017 13:49:30 +0000 Subject: PR middle-end/81462 * predict.c (set_even_probabilities): Cleanup; do not affect probabilities that are already known. (combine_predictions_for_bb): Call even when count is set. * g++.dg/torture/pr81462.C: New testcase. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@250310 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 310d9b0aced..1e2e11d71f0 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -817,16 +817,25 @@ static void set_even_probabilities (basic_block bb, hash_set *unlikely_edges = NULL) { - unsigned nedges = 0; + unsigned nedges = 0, unlikely_count = 0; edge e = NULL; edge_iterator ei; + profile_probability all = profile_probability::always (); FOR_EACH_EDGE (e, ei, bb->succs) - if (!unlikely_executed_edge_p (e)) - nedges ++; + if (e->probability.initialized_p ()) + all -= e->probability; + else if (!unlikely_executed_edge_p (e)) + { + nedges ++; + if (unlikely_edges != NULL && unlikely_edges->contains (e)) + { + all -= profile_probability::very_unlikely (); + unlikely_count++; + } + } /* Make the distribution even if all edges are unlikely. */ - unsigned unlikely_count = unlikely_edges ? unlikely_edges->elements () : 0; if (unlikely_count == nedges) { unlikely_edges = NULL; @@ -836,13 +845,14 @@ set_even_probabilities (basic_block bb, unsigned c = nedges - unlikely_count; FOR_EACH_EDGE (e, ei, bb->succs) - if (!unlikely_executed_edge_p (e)) + if (e->probability.initialized_p ()) + ; + else if (!unlikely_executed_edge_p (e)) { if (unlikely_edges != NULL && unlikely_edges->contains (e)) e->probability = profile_probability::very_unlikely (); else - e->probability = profile_probability::guessed_always () - .apply_scale (1, c); + e->probability = all.apply_scale (1, c).guessed (); } else e->probability = profile_probability::never (); @@ -1151,7 +1161,7 @@ combine_predictions_for_bb (basic_block bb, bool dry_run) if (pred->ep_probability <= PROB_VERY_UNLIKELY) unlikely_edges.add (pred->ep_edge); - if (!bb->count.initialized_p () && !dry_run) + if (!dry_run) set_even_probabilities (bb, &unlikely_edges); clear_bb_predictions (bb); if (dump_file) -- cgit v1.2.1 From 84242aee9f3921d2487451bc5e5c3fcabceff28e Mon Sep 17 00:00:00 2001 From: hubicka Date: Wed, 19 Jul 2017 18:08:53 +0000 Subject: * predict.c (propagate_unlikely_bbs_forward): Break out from ... (determine_unlikely_bbs): ... here. * predict.h (propagate_unlikely_bbs_forward): Declare. * cfgexpand.c (pass_expand::execute): Use it. * bb-reorder.c (sanitize_hot_paths): Do not consider known to be unlikely edges. (find_rarely_executed_basic_blocks_and_crossing_edges): Use propagate_unlikely_bbs_forward. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@250359 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/predict.c | 86 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 37 deletions(-) (limited to 'gcc/predict.c') diff --git a/gcc/predict.c b/gcc/predict.c index 1e2e11d71f0..609c099d7b5 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -3374,6 +3374,55 @@ expensive_function_p (int threshold) return false; } +/* All basic blocks that are reachable only from unlikely basic blocks are + unlikely. */ + +void +propagate_unlikely_bbs_forward (void) +{ + auto_vec worklist; + basic_block bb; + edge_iterator ei; + edge e; + + if (!(ENTRY_BLOCK_PTR_FOR_FN (cfun)->count == profile_count::zero ())) + { + ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(size_t) 1; + worklist.safe_push (ENTRY_BLOCK_PTR_FOR_FN (cfun)); + + while (worklist.length () > 0) + { + bb = worklist.pop (); + FOR_EACH_EDGE (e, ei, bb->succs) + if (!(e->count == profile_count::zero ()) + && !(e->dest->count == profile_count::zero ()) + && !e->dest->aux) + { + e->dest->aux = (void *)(size_t) 1; + worklist.safe_push (e->dest); + } + } + } + + FOR_ALL_BB_FN (bb, cfun) + { + if (!bb->aux) + { + if (!(bb->count == profile_count::zero ()) + && (dump_file && (dump_flags & TDF_DETAILS))) + fprintf (dump_file, + "Basic block %i is marked unlikely by forward prop\n", + bb->index); + bb->count = profile_count::zero (); + bb->frequency = 0; + FOR_EACH_EDGE (e, ei, bb->succs) + e->count = profile_count::zero (); + } + else + bb->aux = NULL; + } +} + /* Determine basic blocks/edges that are known to be unlikely executed and set their counters to zero. This is done with first identifying obviously unlikely BBs/edges and then @@ -3418,43 +3467,6 @@ determine_unlikely_bbs () gcc_checking_assert (!bb->aux); } - if (!(ENTRY_BLOCK_PTR_FOR_FN (cfun)->count == profile_count::zero ())) - { - ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(size_t) 1; - worklist.safe_push (ENTRY_BLOCK_PTR_FOR_FN (cfun)); - - while (worklist.length () > 0) - { - bb = worklist.pop (); - FOR_EACH_EDGE (e, ei, bb->succs) - if (!(e->count == profile_count::zero ()) - && !(e->dest->count == profile_count::zero ()) - && !e->dest->aux) - { - e->dest->aux = (void *)(size_t) 1; - worklist.safe_push (e->dest); - } - } - } - - FOR_ALL_BB_FN (bb, cfun) - { - if (!bb->aux) - { - if (!(bb->count == profile_count::zero ()) - && (dump_file && (dump_flags & TDF_DETAILS))) - fprintf (dump_file, - "Basic block %i is marked unlikely by forward prop\n", - bb->index); - bb->count = profile_count::zero (); - bb->frequency = 0; - FOR_EACH_EDGE (e, ei, bb->succs) - e->count = profile_count::zero (); - } - else - bb->aux = NULL; - } - auto_vec nsuccs; nsuccs.safe_grow_cleared (last_basic_block_for_fn (cfun)); FOR_ALL_BB_FN (bb, cfun) -- cgit v1.2.1